code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.added:
  40     raise qa_error.Error("Node %s already in cluster" % node.primary)
  41   elif readd and not node.added:
  42     raise qa_error.Error("Node %s not yet in cluster" % node.primary)
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.secondary:
  46     cmd.append("--secondary-ip=%s" % node.secondary)
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node.primary)
  50
  51   AssertCommand(cmd)
  52
  53   if readd:
  54     assert node.added
  55   else:
  56     node.MarkAdded()
  57
  58
  59 def _NodeRemove(node):
  60   AssertCommand(["gnt-node", "remove", node.primary])
  61   node.MarkRemoved()
  62
  63
  64 def MakeNodeOffline(node, value):
  65   """gnt-node modify --offline=value"""
  66   # value in ["yes", "no"]
  67   AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
  68
  69
  70 def TestNodeAddAll():
  71   """Adding all nodes to cluster."""
  72   master = qa_config.GetMasterNode()
  73   for node in qa_config.get("nodes"):
  74     if node != master:
  75       _NodeAdd(node, readd=False)
  76
  77
  78 def MarkNodeAddedAll():
  79   """Mark all nodes as added.
  80
  81   This is useful if we don't create the cluster ourselves (in qa).
  82
  83   """
  84   master = qa_config.GetMasterNode()
  85   for node in qa_config.get("nodes"):
  86     if node != master:
  87       node.MarkAdded()
  88
  89
  90 def TestNodeRemoveAll():
  91   """Removing all nodes from cluster."""
  92   master = qa_config.GetMasterNode()
  93   for node in qa_config.get("nodes"):
  94     if node != master:
  95       _NodeRemove(node)
  96
  97
  98 def TestNodeReadd(node):
  99   """gnt-node add --readd"""
 100   _NodeAdd(node, readd=True)
 101
 102
 103 def TestNodeInfo():
 104   """gnt-node info"""
 105   AssertCommand(["gnt-node", "info"])
 106
 107
 108 def TestNodeVolumes():
 109   """gnt-node volumes"""
 110   AssertCommand(["gnt-node", "volumes"])
 111
 112
 113 def TestNodeStorage():
 114   """gnt-node storage"""
 115   master = qa_config.GetMasterNode()
 116
 117   # FIXME: test all storage_types in constants.VALID_STORAGE_TYPES
 118   # as soon as they are implemented.
 119   for storage_type in [constants.ST_FILE, constants.ST_LVM_VG,
 120                        constants.ST_LVM_PV]:
 121
 122     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
 123
 124     # Skip file storage if not enabled, otherwise QA will fail; we
 125     # just test for basic failure, but otherwise skip the rest of the
 126     # tests
 127     if storage_type == constants.ST_FILE and not constants.ENABLE_FILE_STORAGE:
 128       AssertCommand(cmd, fail=True)
 129       continue
 130
 131     # Test simple list
 132     AssertCommand(cmd)
 133
 134     # Test all storage fields
 135     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 136            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 137                                     [constants.SF_NODE, constants.SF_TYPE])]
 138     AssertCommand(cmd)
 139
 140     # Get list of valid storage devices
 141     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 142            "--output=node,name,allocatable", "--separator=|",
 143            "--no-headers"]
 144     output = qa_utils.GetCommandOutput(master.primary,
 145                                        utils.ShellQuoteArgs(cmd))
 146
 147     # Test with up to two devices
 148     testdevcount = 2
 149
 150     for line in output.splitlines()[:testdevcount]:
 151       (node_name, st_name, st_allocatable) = line.split("|")
 152
 153       # Dummy modification without any changes
 154       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 155       AssertCommand(cmd)
 156
 157       # Make sure we end up with the same value as before
 158       if st_allocatable.lower() == "y":
 159         test_allocatable = ["no", "yes"]
 160       else:
 161         test_allocatable = ["yes", "no"]
 162
 163       fail = (constants.SF_ALLOCATABLE not in
 164               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 165
 166       for i in test_allocatable:
 167         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 168                        node_name, storage_type, st_name], fail=fail)
 169
 170         # Verify list output
 171         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 172                "--output=name,allocatable", "--separator=|",
 173                "--no-headers", node_name]
 174         listout = qa_utils.GetCommandOutput(master.primary,
 175                                             utils.ShellQuoteArgs(cmd))
 176         for line in listout.splitlines():
 177           (vfy_name, vfy_allocatable) = line.split("|")
 178           if vfy_name == st_name and not fail:
 179             AssertEqual(vfy_allocatable, i[0].upper())
 180           else:
 181             AssertEqual(vfy_allocatable, st_allocatable)
 182
 183       # Test repair functionality
 184       fail = (constants.SO_FIX_CONSISTENCY not in
 185               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 186       AssertCommand(["gnt-node", "repair-storage", node_name,
 187                      storage_type, st_name], fail=fail)
 188
 189
 190 def TestNodeFailover(node, node2):
 191   """gnt-node failover"""
 192   if qa_utils.GetNodeInstances(node2, secondaries=False):
 193     raise qa_error.UnusableNodeError("Secondary node has at least one"
 194                                      " primary instance. This test requires"
 195                                      " it to have no primary instances.")
 196
 197   # Fail over to secondary node
 198   AssertCommand(["gnt-node", "failover", "-f", node.primary])
 199
 200   # ... and back again.
 201   AssertCommand(["gnt-node", "failover", "-f", node2.primary])
 202
 203
 204 def TestNodeMigrate(node, node2):
 205   """gnt-node migrate"""
 206   if qa_utils.GetNodeInstances(node2, secondaries=False):
 207     raise qa_error.UnusableNodeError("Secondary node has at least one"
 208                                      " primary instance. This test requires"
 209                                      " it to have no primary instances.")
 210
 211   # Migrate to secondary node
 212   AssertCommand(["gnt-node", "migrate", "-f", node.primary])
 213
 214   # ... and back again.
 215   AssertCommand(["gnt-node", "migrate", "-f", node2.primary])
 216
 217
 218 def TestNodeEvacuate(node, node2):
 219   """gnt-node evacuate"""
 220   node3 = qa_config.AcquireNode(exclude=[node, node2])
 221   try:
 222     if qa_utils.GetNodeInstances(node3, secondaries=True):
 223       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 224                                        " secondary instance. This test requires"
 225                                        " it to have no secondary instances.")
 226
 227     # Evacuate all secondary instances
 228     AssertCommand(["gnt-node", "evacuate", "-f",
 229                    "--new-secondary=%s" % node3.primary, node2.primary])
 230
 231     # ... and back again.
 232     AssertCommand(["gnt-node", "evacuate", "-f",
 233                    "--new-secondary=%s" % node2.primary, node3.primary])
 234   finally:
 235     node3.Release()
 236
 237
 238 def TestNodeModify(node):
 239   """gnt-node modify"""
 240
 241   # make sure enough master candidates will be available by disabling the
 242   # master candidate role first with --auto-promote
 243   AssertCommand(["gnt-node", "modify", "--master-candidate=no",
 244                 "--auto-promote", node.primary])
 245
 246   # now it's save to force-remove the master candidate role
 247   for flag in ["master-candidate", "drained", "offline"]:
 248     for value in ["yes", "no"]:
 249       AssertCommand(["gnt-node", "modify", "--force",
 250                      "--%s=%s" % (flag, value), node.primary])
 251
 252   AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary])
 253
 254   # Test setting secondary IP address
 255   AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
 256                  node.primary])
 257
 258
 259 def _CreateOobScriptStructure():
 260   """Create a simple OOB handling script and its structure."""
 261   master = qa_config.GetMasterNode()
 262
 263   data_path = qa_utils.UploadData(master.primary, "")
 264   verify_path = qa_utils.UploadData(master.primary, "")
 265   exit_code_path = qa_utils.UploadData(master.primary, "")
 266
 267   oob_script = (("#!/bin/bash\n"
 268                  "echo \"$@\" > %s\n"
 269                  "cat %s\n"
 270                  "exit $(< %s)\n") %
 271                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 272                  utils.ShellQuote(exit_code_path)))
 273   oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 274
 275   return [oob_path, verify_path, data_path, exit_code_path]
 276
 277
 278 def _UpdateOobFile(path, data):
 279   """Updates the data file with data."""
 280   master = qa_config.GetMasterNode()
 281   qa_utils.UploadData(master.primary, data, filename=path)
 282
 283
 284 def _AssertOobCall(verify_path, expected_args):
 285   """Assert the OOB call was performed with expetected args."""
 286   master = qa_config.GetMasterNode()
 287
 288   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 289   output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
 290                                      tty=False)
 291
 292   AssertEqual(expected_args, output.strip())
 293
 294
 295 def TestOutOfBand():
 296   """gnt-node power"""
 297   master = qa_config.GetMasterNode()
 298
 299   node = qa_config.AcquireNode(exclude=master)
 300
 301   master_name = master.primary
 302   node_name = node.primary
 303   full_node_name = qa_utils.ResolveNodeName(node)
 304
 305   (oob_path, verify_path,
 306    data_path, exit_code_path) = _CreateOobScriptStructure()
 307
 308   try:
 309     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 310                    "oob_program=%s" % oob_path])
 311
 312     # No data, exit 0
 313     _UpdateOobFile(exit_code_path, "0")
 314
 315     AssertCommand(["gnt-node", "power", "on", node_name])
 316     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 317
 318     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 319     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 320
 321     # Power off on master without options should fail
 322     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 323     # With force master it should still fail
 324     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 325                    master_name],
 326                   fail=True)
 327
 328     # Verify we can't transform back to online when not yet powered on
 329     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 330                   fail=True)
 331     # Now reset state
 332     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 333                    node_name])
 334
 335     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 336     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 337
 338     # Those commands should fail as they expect output which isn't provided yet
 339     # But they should have called the oob helper nevermind
 340     AssertCommand(["gnt-node", "power", "status", node_name],
 341                   fail=True)
 342     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 343
 344     AssertCommand(["gnt-node", "health", node_name],
 345                   fail=True)
 346     _AssertOobCall(verify_path, "health %s" % full_node_name)
 347
 348     AssertCommand(["gnt-node", "health"], fail=True)
 349
 350     # Correct Data, exit 0
 351     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 352
 353     AssertCommand(["gnt-node", "power", "status", node_name])
 354     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 355
 356     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 357                                                    ["disk0", "CRITICAL"]]))
 358
 359     AssertCommand(["gnt-node", "health", node_name])
 360     _AssertOobCall(verify_path, "health %s" % full_node_name)
 361
 362     AssertCommand(["gnt-node", "health"])
 363
 364     # Those commands should fail as they expect no data regardless of exit 0
 365     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 366     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 367
 368     try:
 369       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 370       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 371     finally:
 372       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 373
 374     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 375     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 376
 377     # Data, exit 1 (all should fail)
 378     _UpdateOobFile(exit_code_path, "1")
 379
 380     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 381     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 382
 383     try:
 384       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 385       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 386     finally:
 387       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 388
 389     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 390     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 391
 392     AssertCommand(["gnt-node", "power", "status", node_name],
 393                   fail=True)
 394     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 395
 396     AssertCommand(["gnt-node", "health", node_name],
 397                   fail=True)
 398     _AssertOobCall(verify_path, "health %s" % full_node_name)
 399
 400     AssertCommand(["gnt-node", "health"], fail=True)
 401
 402     # No data, exit 1 (all should fail)
 403     _UpdateOobFile(data_path, "")
 404     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 405     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 406
 407     try:
 408       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 409       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 410     finally:
 411       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 412
 413     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 414     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 415
 416     AssertCommand(["gnt-node", "power", "status", node_name],
 417                   fail=True)
 418     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 419
 420     AssertCommand(["gnt-node", "health", node_name],
 421                   fail=True)
 422     _AssertOobCall(verify_path, "health %s" % full_node_name)
 423
 424     AssertCommand(["gnt-node", "health"], fail=True)
 425
 426     # Different OOB script for node
 427     verify_path2 = qa_utils.UploadData(master.primary, "")
 428     oob_script = ("#!/bin/sh\n"
 429                   "echo \"$@\" > %s\n") % verify_path2
 430     oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 431
 432     try:
 433       AssertCommand(["gnt-node", "modify", "--node-parameters",
 434                      "oob_program=%s" % oob_path2, node_name])
 435       AssertCommand(["gnt-node", "power", "on", node_name])
 436       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 437     finally:
 438       AssertCommand(["gnt-node", "modify", "--node-parameters",
 439                      "oob_program=default", node_name])
 440       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 441   finally:
 442     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 443                    "oob_program="])
 444     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 445                    exit_code_path])
 446
 447
 448 def TestNodeList():
 449   """gnt-node list"""
 450   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 451
 452
 453 def TestNodeListFields():
 454   """gnt-node list-fields"""
 455   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
 456
 457
 458 def TestNodeListDrbd(node):
 459   """gnt-node list-drbd"""
 460   AssertCommand(["gnt-node", "list-drbd", node.primary])
 461
 462
 463 def _BuildSetESCmd(action, value, node_name):
 464   cmd = ["gnt-node"]
 465   if action == "add":
 466     cmd.extend(["add", "--readd"])
 467   else:
 468     cmd.append("modify")
 469   cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
 470   return cmd
 471
 472
 473 def TestExclStorSingleNode(node):
 474   """gnt-node add/modify cannot change the exclusive_storage flag.
 475
 476   """
 477   for action in ["add", "modify"]:
 478     for value in (True, False, "default"):
 479       AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)