code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.added:
  40     raise qa_error.Error("Node %s already in cluster" % node.primary)
  41   elif readd and not node.added:
  42     raise qa_error.Error("Node %s not yet in cluster" % node.primary)
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.secondary:
  46     cmd.append("--secondary-ip=%s" % node.secondary)
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node.primary)
  50
  51   AssertCommand(cmd)
  52
  53   if readd:
  54     assert node.added
  55   else:
  56     node.MarkAdded()
  57
  58
  59 def _NodeRemove(node):
  60   AssertCommand(["gnt-node", "remove", node.primary])
  61   node.MarkRemoved()
  62
  63
  64 def MakeNodeOffline(node, value):
  65   """gnt-node modify --offline=value"""
  66   # value in ["yes", "no"]
  67   AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
  68
  69
  70 def TestNodeAddAll():
  71   """Adding all nodes to cluster."""
  72   master = qa_config.GetMasterNode()
  73   for node in qa_config.get("nodes"):
  74     if node != master:
  75       _NodeAdd(node, readd=False)
  76
  77
  78 def MarkNodeAddedAll():
  79   """Mark all nodes as added.
  80
  81   This is useful if we don't create the cluster ourselves (in qa).
  82
  83   """
  84   master = qa_config.GetMasterNode()
  85   for node in qa_config.get("nodes"):
  86     if node != master:
  87       node.MarkAdded()
  88
  89
  90 def TestNodeRemoveAll():
  91   """Removing all nodes from cluster."""
  92   master = qa_config.GetMasterNode()
  93   for node in qa_config.get("nodes"):
  94     if node != master:
  95       _NodeRemove(node)
  96
  97
  98 def TestNodeReadd(node):
  99   """gnt-node add --readd"""
 100   _NodeAdd(node, readd=True)
 101
 102
 103 def TestNodeInfo():
 104   """gnt-node info"""
 105   AssertCommand(["gnt-node", "info"])
 106
 107
 108 def TestNodeVolumes():
 109   """gnt-node volumes"""
 110   AssertCommand(["gnt-node", "volumes"])
 111
 112
 113 def TestNodeStorage():
 114   """gnt-node storage"""
 115   master = qa_config.GetMasterNode()
 116
 117   # FIXME: test all storage_types in constants.STORAGE_TYPES
 118   # as soon as they are implemented.
 119   enabled_storage_types = qa_config.GetEnabledStorageTypes()
 120   testable_storage_types = list(set(enabled_storage_types).intersection(
 121       set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV])))
 122
 123   for storage_type in testable_storage_types:
 124
 125     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
 126
 127     # Test simple list
 128     AssertCommand(cmd)
 129
 130     # Test all storage fields
 131     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 132            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS))]
 133     AssertCommand(cmd)
 134
 135     # Get list of valid storage devices
 136     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 137            "--output=node,name,allocatable", "--separator=|",
 138            "--no-headers"]
 139     output = qa_utils.GetCommandOutput(master.primary,
 140                                        utils.ShellQuoteArgs(cmd))
 141
 142     # Test with up to two devices
 143     testdevcount = 2
 144
 145     for line in output.splitlines()[:testdevcount]:
 146       (node_name, st_name, st_allocatable) = line.split("|")
 147
 148       # Dummy modification without any changes
 149       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 150       AssertCommand(cmd)
 151
 152       # Make sure we end up with the same value as before
 153       if st_allocatable.lower() == "y":
 154         test_allocatable = ["no", "yes"]
 155       else:
 156         test_allocatable = ["yes", "no"]
 157
 158       fail = (constants.SF_ALLOCATABLE not in
 159               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 160
 161       for i in test_allocatable:
 162         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 163                        node_name, storage_type, st_name], fail=fail)
 164
 165         # Verify list output
 166         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 167                "--output=name,allocatable", "--separator=|",
 168                "--no-headers", node_name]
 169         listout = qa_utils.GetCommandOutput(master.primary,
 170                                             utils.ShellQuoteArgs(cmd))
 171         for line in listout.splitlines():
 172           (vfy_name, vfy_allocatable) = line.split("|")
 173           if vfy_name == st_name and not fail:
 174             AssertEqual(vfy_allocatable, i[0].upper())
 175           else:
 176             AssertEqual(vfy_allocatable, st_allocatable)
 177
 178       # Test repair functionality
 179       fail = (constants.SO_FIX_CONSISTENCY not in
 180               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 181       AssertCommand(["gnt-node", "repair-storage", node_name,
 182                      storage_type, st_name], fail=fail)
 183
 184
 185 def TestNodeFailover(node, node2):
 186   """gnt-node failover"""
 187   if qa_utils.GetNodeInstances(node2, secondaries=False):
 188     raise qa_error.UnusableNodeError("Secondary node has at least one"
 189                                      " primary instance. This test requires"
 190                                      " it to have no primary instances.")
 191
 192   # Fail over to secondary node
 193   AssertCommand(["gnt-node", "failover", "-f", node.primary])
 194
 195   # ... and back again.
 196   AssertCommand(["gnt-node", "failover", "-f", node2.primary])
 197
 198
 199 def TestNodeMigrate(node, node2):
 200   """gnt-node migrate"""
 201   if qa_utils.GetNodeInstances(node2, secondaries=False):
 202     raise qa_error.UnusableNodeError("Secondary node has at least one"
 203                                      " primary instance. This test requires"
 204                                      " it to have no primary instances.")
 205
 206   # Migrate to secondary node
 207   AssertCommand(["gnt-node", "migrate", "-f", node.primary])
 208
 209   # ... and back again.
 210   AssertCommand(["gnt-node", "migrate", "-f", node2.primary])
 211
 212
 213 def TestNodeEvacuate(node, node2):
 214   """gnt-node evacuate"""
 215   node3 = qa_config.AcquireNode(exclude=[node, node2])
 216   try:
 217     if qa_utils.GetNodeInstances(node3, secondaries=True):
 218       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 219                                        " secondary instance. This test requires"
 220                                        " it to have no secondary instances.")
 221
 222     # Evacuate all secondary instances
 223     AssertCommand(["gnt-node", "evacuate", "-f",
 224                    "--new-secondary=%s" % node3.primary, node2.primary])
 225
 226     # ... and back again.
 227     AssertCommand(["gnt-node", "evacuate", "-f",
 228                    "--new-secondary=%s" % node2.primary, node3.primary])
 229   finally:
 230     node3.Release()
 231
 232
 233 def TestNodeModify(node):
 234   """gnt-node modify"""
 235
 236   # make sure enough master candidates will be available by disabling the
 237   # master candidate role first with --auto-promote
 238   AssertCommand(["gnt-node", "modify", "--master-candidate=no",
 239                 "--auto-promote", node.primary])
 240
 241   # now it's save to force-remove the master candidate role
 242   for flag in ["master-candidate", "drained", "offline"]:
 243     for value in ["yes", "no"]:
 244       AssertCommand(["gnt-node", "modify", "--force",
 245                      "--%s=%s" % (flag, value), node.primary])
 246
 247   AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary])
 248
 249   # Test setting secondary IP address
 250   AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
 251                  node.primary])
 252
 253
 254 def _CreateOobScriptStructure():
 255   """Create a simple OOB handling script and its structure."""
 256   master = qa_config.GetMasterNode()
 257
 258   data_path = qa_utils.UploadData(master.primary, "")
 259   verify_path = qa_utils.UploadData(master.primary, "")
 260   exit_code_path = qa_utils.UploadData(master.primary, "")
 261
 262   oob_script = (("#!/bin/bash\n"
 263                  "echo \"$@\" > %s\n"
 264                  "cat %s\n"
 265                  "exit $(< %s)\n") %
 266                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 267                  utils.ShellQuote(exit_code_path)))
 268   oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 269
 270   return [oob_path, verify_path, data_path, exit_code_path]
 271
 272
 273 def _UpdateOobFile(path, data):
 274   """Updates the data file with data."""
 275   master = qa_config.GetMasterNode()
 276   qa_utils.UploadData(master.primary, data, filename=path)
 277
 278
 279 def _AssertOobCall(verify_path, expected_args):
 280   """Assert the OOB call was performed with expetected args."""
 281   master = qa_config.GetMasterNode()
 282
 283   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 284   output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
 285                                      tty=False)
 286
 287   AssertEqual(expected_args, output.strip())
 288
 289
 290 def TestOutOfBand():
 291   """gnt-node power"""
 292   master = qa_config.GetMasterNode()
 293
 294   node = qa_config.AcquireNode(exclude=master)
 295
 296   master_name = master.primary
 297   node_name = node.primary
 298   full_node_name = qa_utils.ResolveNodeName(node)
 299
 300   (oob_path, verify_path,
 301    data_path, exit_code_path) = _CreateOobScriptStructure()
 302
 303   try:
 304     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 305                    "oob_program=%s" % oob_path])
 306
 307     # No data, exit 0
 308     _UpdateOobFile(exit_code_path, "0")
 309
 310     AssertCommand(["gnt-node", "power", "on", node_name])
 311     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 312
 313     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 314     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 315
 316     # Power off on master without options should fail
 317     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 318     # With force master it should still fail
 319     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 320                    master_name],
 321                   fail=True)
 322
 323     # Verify we can't transform back to online when not yet powered on
 324     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 325                   fail=True)
 326     # Now reset state
 327     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 328                    node_name])
 329
 330     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 331     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 332
 333     # Those commands should fail as they expect output which isn't provided yet
 334     # But they should have called the oob helper nevermind
 335     AssertCommand(["gnt-node", "power", "status", node_name],
 336                   fail=True)
 337     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 338
 339     AssertCommand(["gnt-node", "health", node_name],
 340                   fail=True)
 341     _AssertOobCall(verify_path, "health %s" % full_node_name)
 342
 343     AssertCommand(["gnt-node", "health"], fail=True)
 344
 345     # Correct Data, exit 0
 346     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 347
 348     AssertCommand(["gnt-node", "power", "status", node_name])
 349     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 350
 351     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 352                                                    ["disk0", "CRITICAL"]]))
 353
 354     AssertCommand(["gnt-node", "health", node_name])
 355     _AssertOobCall(verify_path, "health %s" % full_node_name)
 356
 357     AssertCommand(["gnt-node", "health"])
 358
 359     # Those commands should fail as they expect no data regardless of exit 0
 360     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 361     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 362
 363     try:
 364       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 365       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 366     finally:
 367       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 368
 369     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 370     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 371
 372     # Data, exit 1 (all should fail)
 373     _UpdateOobFile(exit_code_path, "1")
 374
 375     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 376     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 377
 378     try:
 379       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 380       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 381     finally:
 382       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 383
 384     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 385     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 386
 387     AssertCommand(["gnt-node", "power", "status", node_name],
 388                   fail=True)
 389     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 390
 391     AssertCommand(["gnt-node", "health", node_name],
 392                   fail=True)
 393     _AssertOobCall(verify_path, "health %s" % full_node_name)
 394
 395     AssertCommand(["gnt-node", "health"], fail=True)
 396
 397     # No data, exit 1 (all should fail)
 398     _UpdateOobFile(data_path, "")
 399     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 400     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 401
 402     try:
 403       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 404       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 405     finally:
 406       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 407
 408     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 409     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 410
 411     AssertCommand(["gnt-node", "power", "status", node_name],
 412                   fail=True)
 413     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 414
 415     AssertCommand(["gnt-node", "health", node_name],
 416                   fail=True)
 417     _AssertOobCall(verify_path, "health %s" % full_node_name)
 418
 419     AssertCommand(["gnt-node", "health"], fail=True)
 420
 421     # Different OOB script for node
 422     verify_path2 = qa_utils.UploadData(master.primary, "")
 423     oob_script = ("#!/bin/sh\n"
 424                   "echo \"$@\" > %s\n") % verify_path2
 425     oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 426
 427     try:
 428       AssertCommand(["gnt-node", "modify", "--node-parameters",
 429                      "oob_program=%s" % oob_path2, node_name])
 430       AssertCommand(["gnt-node", "power", "on", node_name])
 431       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 432     finally:
 433       AssertCommand(["gnt-node", "modify", "--node-parameters",
 434                      "oob_program=default", node_name])
 435       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 436   finally:
 437     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 438                    "oob_program="])
 439     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 440                    exit_code_path])
 441
 442
 443 def TestNodeList():
 444   """gnt-node list"""
 445   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 446
 447
 448 def TestNodeListFields():
 449   """gnt-node list-fields"""
 450   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
 451
 452
 453 def TestNodeListDrbd(node):
 454   """gnt-node list-drbd"""
 455   AssertCommand(["gnt-node", "list-drbd", node.primary])
 456
 457
 458 def _BuildSetESCmd(action, value, node_name):
 459   cmd = ["gnt-node"]
 460   if action == "add":
 461     cmd.extend(["add", "--readd"])
 462   else:
 463     cmd.append("modify")
 464   cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
 465   return cmd
 466
 467
 468 def TestExclStorSingleNode(node):
 469   """gnt-node add/modify cannot change the exclusive_storage flag.
 470
 471   """
 472   for action in ["add", "modify"]:
 473     for value in (True, False, "default"):
 474       AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)