code.grnet.gr Git - ganeti-local/blob - qa/qa_cluster.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2010, 2011 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Cluster related QA tests.
  23
  24 """
  25
  26 import tempfile
  27 import os.path
  28
  29 from ganeti import constants
  30 from ganeti import compat
  31 from ganeti import utils
  32
  33 import qa_config
  34 import qa_utils
  35 import qa_error
  36
  37 from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
  38
  39
  40 #: cluster verify command
  41 _CLUSTER_VERIFY = ["gnt-cluster", "verify"]
  42
  43
  44 def _RemoveFileFromAllNodes(filename):
  45   """Removes a file from all nodes.
  46
  47   """
  48   for node in qa_config.get("nodes"):
  49     AssertCommand(["rm", "-f", filename], node=node)
  50
  51
  52 def _CheckFileOnAllNodes(filename, content):
  53   """Verifies the content of the given file on all nodes.
  54
  55   """
  56   cmd = utils.ShellQuoteArgs(["cat", filename])
  57   for node in qa_config.get("nodes"):
  58     AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content)
  59
  60
  61 def TestClusterInit(rapi_user, rapi_secret):
  62   """gnt-cluster init"""
  63   master = qa_config.GetMasterNode()
  64
  65   rapi_dir = os.path.dirname(constants.RAPI_USERS_FILE)
  66
  67   # First create the RAPI credentials
  68   fh = tempfile.NamedTemporaryFile()
  69   try:
  70     fh.write("%s %s write\n" % (rapi_user, rapi_secret))
  71     fh.flush()
  72
  73     tmpru = qa_utils.UploadFile(master["primary"], fh.name)
  74     try:
  75       AssertCommand(["mkdir", "-p", rapi_dir])
  76       AssertCommand(["mv", tmpru, constants.RAPI_USERS_FILE])
  77     finally:
  78       AssertCommand(["rm", "-f", tmpru])
  79   finally:
  80     fh.close()
  81
  82   # Initialize cluster
  83   cmd = ["gnt-cluster", "init"]
  84
  85   cmd.append("--primary-ip-version=%d" %
  86              qa_config.get("primary_ip_version", 4))
  87
  88   if master.get("secondary", None):
  89     cmd.append("--secondary-ip=%s" % master["secondary"])
  90
  91   bridge = qa_config.get("bridge", None)
  92   if bridge:
  93     cmd.append("--bridge=%s" % bridge)
  94     cmd.append("--master-netdev=%s" % bridge)
  95
  96   htype = qa_config.get("enabled-hypervisors", None)
  97   if htype:
  98     cmd.append("--enabled-hypervisors=%s" % htype)
  99
 100   cmd.append(qa_config.get("name"))
 101
 102   AssertCommand(cmd)
 103
 104   cmd = ["gnt-cluster", "modify"]
 105   # hypervisor parameter modifications
 106   hvp = qa_config.get("hypervisor-parameters", {})
 107   for k, v in hvp.items():
 108     cmd.extend(["-H", "%s:%s" % (k, v)])
 109   # backend parameter modifications
 110   bep = qa_config.get("backend-parameters", "")
 111   if bep:
 112     cmd.extend(["-B", bep])
 113
 114   if len(cmd) > 2:
 115     AssertCommand(cmd)
 116
 117   # OS parameters
 118   osp = qa_config.get("os-parameters", {})
 119   for k, v in osp.items():
 120     AssertCommand(["gnt-os", "modify", "-O", v, k])
 121
 122   # OS hypervisor parameters
 123   os_hvp = qa_config.get("os-hvp", {})
 124   for os_name in os_hvp:
 125     for hv, hvp in os_hvp[os_name].items():
 126       AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
 127
 128
 129 def TestClusterRename():
 130   """gnt-cluster rename"""
 131   cmd = ["gnt-cluster", "rename", "-f"]
 132
 133   original_name = qa_config.get("name")
 134   rename_target = qa_config.get("rename", None)
 135   if rename_target is None:
 136     print qa_utils.FormatError('"rename" entry is missing')
 137     return
 138
 139   for data in [
 140     cmd + [rename_target],
 141     _CLUSTER_VERIFY,
 142     cmd + [original_name],
 143     _CLUSTER_VERIFY,
 144     ]:
 145     AssertCommand(data)
 146
 147
 148 def TestClusterOob():
 149   """out-of-band framework"""
 150   oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
 151
 152   AssertCommand(_CLUSTER_VERIFY)
 153   AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 154                  "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
 155                  utils.NewUUID()])
 156
 157   AssertCommand(_CLUSTER_VERIFY, fail=True)
 158
 159   AssertCommand(["touch", oob_path_exists])
 160   AssertCommand(["chmod", "0400", oob_path_exists])
 161   AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
 162
 163   try:
 164     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 165                    "oob_program=%s" % oob_path_exists])
 166
 167     AssertCommand(_CLUSTER_VERIFY, fail=True)
 168
 169     AssertCommand(["chmod", "0500", oob_path_exists])
 170     AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
 171
 172     AssertCommand(_CLUSTER_VERIFY)
 173   finally:
 174     AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
 175
 176   AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 177                  "oob_program="])
 178
 179
 180 def TestClusterEpo():
 181   """gnt-cluster epo"""
 182   master = qa_config.GetMasterNode()
 183
 184   # Assert that OOB is unavailable for all nodes
 185   result_output = GetCommandOutput(master["primary"],
 186                                    "gnt-node list --verbose --no-header -o"
 187                                    " powered")
 188   AssertEqual(compat.all(powered == "(unavail)"
 189                          for powered in result_output.splitlines()), True)
 190
 191   # Conflicting
 192   AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
 193   # --all doesn't expect arguments
 194   AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
 195
 196   # Unless --all is given master is not allowed to be in the list
 197   AssertCommand(["gnt-cluster", "epo", "-f", master["primary"]], fail=True)
 198
 199   # This shouldn't fail
 200   AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
 201
 202   # All instances should have been stopped now
 203   result_output = GetCommandOutput(master["primary"],
 204                                    "gnt-instance list --no-header -o status")
 205   AssertEqual(compat.all(status == "ADMIN_down"
 206                          for status in result_output.splitlines()), True)
 207
 208   # Now start everything again
 209   AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
 210
 211   # All instances should have been started now
 212   result_output = GetCommandOutput(master["primary"],
 213                                    "gnt-instance list --no-header -o status")
 214   AssertEqual(compat.all(status == "running"
 215                          for status in result_output.splitlines()), True)
 216
 217
 218 def TestClusterVerify():
 219   """gnt-cluster verify"""
 220   AssertCommand(_CLUSTER_VERIFY)
 221   AssertCommand(["gnt-cluster", "verify-disks"])
 222
 223
 224 def TestJobqueue():
 225   """gnt-debug test-jobqueue"""
 226   AssertCommand(["gnt-debug", "test-jobqueue"])
 227
 228
 229 def TestClusterReservedLvs():
 230   """gnt-cluster reserved lvs"""
 231   for fail, cmd in [
 232     (False, _CLUSTER_VERIFY),
 233     (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
 234     (False, ["lvcreate", "-L1G", "-nqa-test", "xenvg"]),
 235     (True, _CLUSTER_VERIFY),
 236     (False, ["gnt-cluster", "modify", "--reserved-lvs",
 237              "xenvg/qa-test,.*/other-test"]),
 238     (False, _CLUSTER_VERIFY),
 239     (False, ["gnt-cluster", "modify", "--reserved-lvs", ".*/qa-.*"]),
 240     (False, _CLUSTER_VERIFY),
 241     (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
 242     (True, _CLUSTER_VERIFY),
 243     (False, ["lvremove", "-f", "xenvg/qa-test"]),
 244     (False, _CLUSTER_VERIFY),
 245     ]:
 246     AssertCommand(cmd, fail=fail)
 247
 248
 249 def TestClusterModifyBe():
 250   """gnt-cluster modify -B"""
 251   for fail, cmd in [
 252     # mem
 253     (False, ["gnt-cluster", "modify", "-B", "memory=256"]),
 254     (False, ["sh", "-c", "gnt-cluster info|grep '^ *memory: 256$'"]),
 255     (True, ["gnt-cluster", "modify", "-B", "memory=a"]),
 256     (False, ["gnt-cluster", "modify", "-B", "memory=128"]),
 257     (False, ["sh", "-c", "gnt-cluster info|grep '^ *memory: 128$'"]),
 258     # vcpus
 259     (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
 260     (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
 261     (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
 262     (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
 263     (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
 264     # auto_balance
 265     (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
 266     (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
 267     (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
 268     (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
 269     (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
 270     ]:
 271     AssertCommand(cmd, fail=fail)
 272
 273   # redo the original-requested BE parameters, if any
 274   bep = qa_config.get("backend-parameters", "")
 275   if bep:
 276     AssertCommand(["gnt-cluster", "modify", "-B", bep])
 277
 278
 279 def TestClusterInfo():
 280   """gnt-cluster info"""
 281   AssertCommand(["gnt-cluster", "info"])
 282
 283
 284 def TestClusterRedistConf():
 285   """gnt-cluster redist-conf"""
 286   AssertCommand(["gnt-cluster", "redist-conf"])
 287
 288
 289 def TestClusterGetmaster():
 290   """gnt-cluster getmaster"""
 291   AssertCommand(["gnt-cluster", "getmaster"])
 292
 293
 294 def TestClusterVersion():
 295   """gnt-cluster version"""
 296   AssertCommand(["gnt-cluster", "version"])
 297
 298
 299 def TestClusterRenewCrypto():
 300   """gnt-cluster renew-crypto"""
 301   master = qa_config.GetMasterNode()
 302
 303   # Conflicting options
 304   cmd = ["gnt-cluster", "renew-crypto", "--force",
 305          "--new-cluster-certificate", "--new-confd-hmac-key"]
 306   conflicting = [
 307     ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
 308     ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
 309     ]
 310   for i in conflicting:
 311     AssertCommand(cmd + i, fail=True)
 312
 313   # Invalid RAPI certificate
 314   cmd = ["gnt-cluster", "renew-crypto", "--force",
 315          "--rapi-certificate=/dev/null"]
 316   AssertCommand(cmd, fail=True)
 317
 318   rapi_cert_backup = qa_utils.BackupFile(master["primary"],
 319                                          constants.RAPI_CERT_FILE)
 320   try:
 321     # Custom RAPI certificate
 322     fh = tempfile.NamedTemporaryFile()
 323
 324     # Ensure certificate doesn't cause "gnt-cluster verify" to complain
 325     validity = constants.SSL_CERT_EXPIRATION_WARN * 3
 326
 327     utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
 328
 329     tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
 330     try:
 331       AssertCommand(["gnt-cluster", "renew-crypto", "--force",
 332                      "--rapi-certificate=%s" % tmpcert])
 333     finally:
 334       AssertCommand(["rm", "-f", tmpcert])
 335
 336     # Custom cluster domain secret
 337     cds_fh = tempfile.NamedTemporaryFile()
 338     cds_fh.write(utils.GenerateSecret())
 339     cds_fh.write("\n")
 340     cds_fh.flush()
 341
 342     tmpcds = qa_utils.UploadFile(master["primary"], cds_fh.name)
 343     try:
 344       AssertCommand(["gnt-cluster", "renew-crypto", "--force",
 345                      "--cluster-domain-secret=%s" % tmpcds])
 346     finally:
 347       AssertCommand(["rm", "-f", tmpcds])
 348
 349     # Normal case
 350     AssertCommand(["gnt-cluster", "renew-crypto", "--force",
 351                    "--new-cluster-certificate", "--new-confd-hmac-key",
 352                    "--new-rapi-certificate", "--new-cluster-domain-secret"])
 353
 354     # Restore RAPI certificate
 355     AssertCommand(["gnt-cluster", "renew-crypto", "--force",
 356                    "--rapi-certificate=%s" % rapi_cert_backup])
 357   finally:
 358     AssertCommand(["rm", "-f", rapi_cert_backup])
 359
 360
 361 def TestClusterBurnin():
 362   """Burnin"""
 363   master = qa_config.GetMasterNode()
 364
 365   options = qa_config.get("options", {})
 366   disk_template = options.get("burnin-disk-template", "drbd")
 367   parallel = options.get("burnin-in-parallel", False)
 368   check_inst = options.get("burnin-check-instances", False)
 369   do_rename = options.get("burnin-rename", "")
 370   do_reboot = options.get("burnin-reboot", True)
 371   reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
 372
 373   # Get as many instances as we need
 374   instances = []
 375   try:
 376     try:
 377       num = qa_config.get("options", {}).get("burnin-instances", 1)
 378       for _ in range(0, num):
 379         instances.append(qa_config.AcquireInstance())
 380     except qa_error.OutOfInstancesError:
 381       print "Not enough instances, continuing anyway."
 382
 383     if len(instances) < 1:
 384       raise qa_error.Error("Burnin needs at least one instance")
 385
 386     script = qa_utils.UploadFile(master["primary"], "../tools/burnin")
 387     try:
 388       # Run burnin
 389       cmd = [script,
 390              "--os=%s" % qa_config.get("os"),
 391              "--disk-size=%s" % ",".join(qa_config.get("disk")),
 392              "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
 393              "--disk-template=%s" % disk_template]
 394       if parallel:
 395         cmd.append("--parallel")
 396         cmd.append("--early-release")
 397       if check_inst:
 398         cmd.append("--http-check")
 399       if do_rename:
 400         cmd.append("--rename=%s" % do_rename)
 401       if not do_reboot:
 402         cmd.append("--no-reboot")
 403       else:
 404         cmd.append("--reboot-types=%s" % ",".join(reboot_types))
 405       cmd += [inst["name"] for inst in instances]
 406       AssertCommand(cmd)
 407     finally:
 408       AssertCommand(["rm", "-f", script])
 409
 410   finally:
 411     for inst in instances:
 412       qa_config.ReleaseInstance(inst)
 413
 414
 415 def TestClusterMasterFailover():
 416   """gnt-cluster master-failover"""
 417   master = qa_config.GetMasterNode()
 418   failovermaster = qa_config.AcquireNode(exclude=master)
 419
 420   cmd = ["gnt-cluster", "master-failover"]
 421   try:
 422     AssertCommand(cmd, node=failovermaster)
 423     # Back to original master node
 424     AssertCommand(cmd, node=master)
 425   finally:
 426     qa_config.ReleaseNode(failovermaster)
 427
 428
 429 def TestClusterMasterFailoverWithDrainedQueue():
 430   """gnt-cluster master-failover with drained queue"""
 431   drain_check = ["test", "-f", constants.JOB_QUEUE_DRAIN_FILE]
 432
 433   master = qa_config.GetMasterNode()
 434   failovermaster = qa_config.AcquireNode(exclude=master)
 435
 436   # Ensure queue is not drained
 437   for node in [master, failovermaster]:
 438     AssertCommand(drain_check, node=node, fail=True)
 439
 440   # Drain queue on failover master
 441   AssertCommand(["touch", constants.JOB_QUEUE_DRAIN_FILE], node=failovermaster)
 442
 443   cmd = ["gnt-cluster", "master-failover"]
 444   try:
 445     AssertCommand(drain_check, node=failovermaster)
 446     AssertCommand(cmd, node=failovermaster)
 447     AssertCommand(drain_check, fail=True)
 448     AssertCommand(drain_check, node=failovermaster, fail=True)
 449
 450     # Back to original master node
 451     AssertCommand(cmd, node=master)
 452   finally:
 453     qa_config.ReleaseNode(failovermaster)
 454
 455   AssertCommand(drain_check, fail=True)
 456   AssertCommand(drain_check, node=failovermaster, fail=True)
 457
 458
 459 def TestClusterCopyfile():
 460   """gnt-cluster copyfile"""
 461   master = qa_config.GetMasterNode()
 462
 463   uniqueid = utils.NewUUID()
 464
 465   # Create temporary file
 466   f = tempfile.NamedTemporaryFile()
 467   f.write(uniqueid)
 468   f.flush()
 469   f.seek(0)
 470
 471   # Upload file to master node
 472   testname = qa_utils.UploadFile(master["primary"], f.name)
 473   try:
 474     # Copy file to all nodes
 475     AssertCommand(["gnt-cluster", "copyfile", testname])
 476     _CheckFileOnAllNodes(testname, uniqueid)
 477   finally:
 478     _RemoveFileFromAllNodes(testname)
 479
 480
 481 def TestClusterCommand():
 482   """gnt-cluster command"""
 483   uniqueid = utils.NewUUID()
 484   rfile = "/tmp/gnt%s" % utils.NewUUID()
 485   rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
 486   cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
 487                               "%s >%s" % (rcmd, rfile)])
 488
 489   try:
 490     AssertCommand(cmd)
 491     _CheckFileOnAllNodes(rfile, uniqueid)
 492   finally:
 493     _RemoveFileFromAllNodes(rfile)
 494
 495
 496 def TestClusterDestroy():
 497   """gnt-cluster destroy"""
 498   AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
 499
 500
 501 def TestClusterRepairDiskSizes():
 502   """gnt-cluster repair-disk-sizes"""
 503   AssertCommand(["gnt-cluster", "repair-disk-sizes"])