code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.added:
  40     raise qa_error.Error("Node %s already in cluster" % node.primary)
  41   elif readd and not node.added:
  42     raise qa_error.Error("Node %s not yet in cluster" % node.primary)
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.secondary:
  46     cmd.append("--secondary-ip=%s" % node.secondary)
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node.primary)
  50
  51   AssertCommand(cmd)
  52
  53   if readd:
  54     assert node.added
  55   else:
  56     node.MarkAdded()
  57
  58
  59 def _NodeRemove(node):
  60   AssertCommand(["gnt-node", "remove", node.primary])
  61   node.MarkRemoved()
  62
  63
  64 def MakeNodeOffline(node, value):
  65   """gnt-node modify --offline=value"""
  66   # value in ["yes", "no"]
  67   AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
  68
  69
  70 def TestNodeAddAll():
  71   """Adding all nodes to cluster."""
  72   master = qa_config.GetMasterNode()
  73   for node in qa_config.get("nodes"):
  74     if node != master:
  75       _NodeAdd(node, readd=False)
  76
  77
  78 def MarkNodeAddedAll():
  79   """Mark all nodes as added.
  80
  81   This is useful if we don't create the cluster ourselves (in qa).
  82
  83   """
  84   master = qa_config.GetMasterNode()
  85   for node in qa_config.get("nodes"):
  86     if node != master:
  87       node.MarkAdded()
  88
  89
  90 def TestNodeRemoveAll():
  91   """Removing all nodes from cluster."""
  92   master = qa_config.GetMasterNode()
  93   for node in qa_config.get("nodes"):
  94     if node != master:
  95       _NodeRemove(node)
  96
  97
  98 def TestNodeReadd(node):
  99   """gnt-node add --readd"""
 100   _NodeAdd(node, readd=True)
 101
 102
 103 def TestNodeInfo():
 104   """gnt-node info"""
 105   AssertCommand(["gnt-node", "info"])
 106
 107
 108 def TestNodeVolumes():
 109   """gnt-node volumes"""
 110   AssertCommand(["gnt-node", "volumes"])
 111
 112
 113 def TestNodeStorage():
 114   """gnt-node storage"""
 115   master = qa_config.GetMasterNode()
 116
 117   # FIXME: test all storage_types in constants.VALID_STORAGE_TYPES
 118   # as soon as they are implemented.
 119   for storage_type in [constants.ST_FILE, constants.ST_LVM_VG,
 120                        constants.ST_LVM_PV]:
 121
 122     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
 123
 124     # Skip file storage if not enabled, otherwise QA will fail; we
 125     # just test for basic failure, but otherwise skip the rest of the
 126     # tests
 127     if storage_type == constants.ST_FILE and not constants.ENABLE_FILE_STORAGE:
 128       AssertCommand(cmd, fail=True)
 129       continue
 130
 131     # Test simple list
 132     AssertCommand(cmd)
 133
 134     # Test all storage fields
 135     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 136            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 137                                     [constants.SF_NODE, constants.SF_TYPE])]
 138     AssertCommand(cmd)
 139
 140     # Get list of valid storage devices
 141     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 142            "--output=node,name,allocatable", "--separator=|",
 143            "--no-headers"]
 144     output = qa_utils.GetCommandOutput(master.primary,
 145                                        utils.ShellQuoteArgs(cmd))
 146
 147     # Test with up to two devices
 148     testdevcount = 2
 149
 150     for line in output.splitlines()[:testdevcount]:
 151       (node_name, st_name, st_allocatable) = line.split("|")
 152
 153       # Dummy modification without any changes
 154       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 155       AssertCommand(cmd)
 156
 157       # Make sure we end up with the same value as before
 158       if st_allocatable.lower() == "y":
 159         test_allocatable = ["no", "yes"]
 160       else:
 161         test_allocatable = ["yes", "no"]
 162
 163       fail = (constants.SF_ALLOCATABLE not in
 164               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 165
 166       for i in test_allocatable:
 167         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 168                        node_name, storage_type, st_name], fail=fail)
 169
 170         # Verify list output
 171         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 172                "--output=name,allocatable", "--separator=|",
 173                "--no-headers", node_name]
 174         listout = qa_utils.GetCommandOutput(master.primary,
 175                                             utils.ShellQuoteArgs(cmd))
 176         for line in listout.splitlines():
 177           (vfy_name, vfy_allocatable) = line.split("|")
 178           if vfy_name == st_name and not fail:
 179             AssertEqual(vfy_allocatable, i[0].upper())
 180           else:
 181             AssertEqual(vfy_allocatable, st_allocatable)
 182
 183       # Test repair functionality
 184       fail = (constants.SO_FIX_CONSISTENCY not in
 185               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 186       AssertCommand(["gnt-node", "repair-storage", node_name,
 187                      storage_type, st_name], fail=fail)
 188
 189
 190 def TestNodeFailover(node, node2):
 191   """gnt-node failover"""
 192   if qa_utils.GetNodeInstances(node2, secondaries=False):
 193     raise qa_error.UnusableNodeError("Secondary node has at least one"
 194                                      " primary instance. This test requires"
 195                                      " it to have no primary instances.")
 196
 197   # Fail over to secondary node
 198   AssertCommand(["gnt-node", "failover", "-f", node.primary])
 199
 200   # ... and back again.
 201   AssertCommand(["gnt-node", "failover", "-f", node2.primary])
 202
 203
 204 def TestNodeEvacuate(node, node2):
 205   """gnt-node evacuate"""
 206   node3 = qa_config.AcquireNode(exclude=[node, node2])
 207   try:
 208     if qa_utils.GetNodeInstances(node3, secondaries=True):
 209       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 210                                        " secondary instance. This test requires"
 211                                        " it to have no secondary instances.")
 212
 213     # Evacuate all secondary instances
 214     AssertCommand(["gnt-node", "evacuate", "-f",
 215                    "--new-secondary=%s" % node3.primary, node2.primary])
 216
 217     # ... and back again.
 218     AssertCommand(["gnt-node", "evacuate", "-f",
 219                    "--new-secondary=%s" % node2.primary, node3.primary])
 220   finally:
 221     node3.Release()
 222
 223
 224 def TestNodeModify(node):
 225   """gnt-node modify"""
 226   for flag in ["master-candidate", "drained", "offline"]:
 227     for value in ["yes", "no"]:
 228       AssertCommand(["gnt-node", "modify", "--force",
 229                      "--%s=%s" % (flag, value), node.primary])
 230
 231   AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
 232                  "--auto-promote", node.primary])
 233
 234   # Test setting secondary IP address
 235   AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
 236                  node.primary])
 237
 238
 239 def _CreateOobScriptStructure():
 240   """Create a simple OOB handling script and its structure."""
 241   master = qa_config.GetMasterNode()
 242
 243   data_path = qa_utils.UploadData(master.primary, "")
 244   verify_path = qa_utils.UploadData(master.primary, "")
 245   exit_code_path = qa_utils.UploadData(master.primary, "")
 246
 247   oob_script = (("#!/bin/bash\n"
 248                  "echo \"$@\" > %s\n"
 249                  "cat %s\n"
 250                  "exit $(< %s)\n") %
 251                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 252                  utils.ShellQuote(exit_code_path)))
 253   oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 254
 255   return [oob_path, verify_path, data_path, exit_code_path]
 256
 257
 258 def _UpdateOobFile(path, data):
 259   """Updates the data file with data."""
 260   master = qa_config.GetMasterNode()
 261   qa_utils.UploadData(master.primary, data, filename=path)
 262
 263
 264 def _AssertOobCall(verify_path, expected_args):
 265   """Assert the OOB call was performed with expetected args."""
 266   master = qa_config.GetMasterNode()
 267
 268   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 269   output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
 270                                      tty=False)
 271
 272   AssertEqual(expected_args, output.strip())
 273
 274
 275 def TestOutOfBand():
 276   """gnt-node power"""
 277   master = qa_config.GetMasterNode()
 278
 279   node = qa_config.AcquireNode(exclude=master)
 280
 281   master_name = master.primary
 282   node_name = node.primary
 283   full_node_name = qa_utils.ResolveNodeName(node)
 284
 285   (oob_path, verify_path,
 286    data_path, exit_code_path) = _CreateOobScriptStructure()
 287
 288   try:
 289     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 290                    "oob_program=%s" % oob_path])
 291
 292     # No data, exit 0
 293     _UpdateOobFile(exit_code_path, "0")
 294
 295     AssertCommand(["gnt-node", "power", "on", node_name])
 296     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 297
 298     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 299     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 300
 301     # Power off on master without options should fail
 302     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 303     # With force master it should still fail
 304     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 305                    master_name],
 306                   fail=True)
 307
 308     # Verify we can't transform back to online when not yet powered on
 309     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 310                   fail=True)
 311     # Now reset state
 312     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 313                    node_name])
 314
 315     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 316     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 317
 318     # Those commands should fail as they expect output which isn't provided yet
 319     # But they should have called the oob helper nevermind
 320     AssertCommand(["gnt-node", "power", "status", node_name],
 321                   fail=True)
 322     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 323
 324     AssertCommand(["gnt-node", "health", node_name],
 325                   fail=True)
 326     _AssertOobCall(verify_path, "health %s" % full_node_name)
 327
 328     AssertCommand(["gnt-node", "health"], fail=True)
 329
 330     # Correct Data, exit 0
 331     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 332
 333     AssertCommand(["gnt-node", "power", "status", node_name])
 334     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 335
 336     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 337                                                    ["disk0", "CRITICAL"]]))
 338
 339     AssertCommand(["gnt-node", "health", node_name])
 340     _AssertOobCall(verify_path, "health %s" % full_node_name)
 341
 342     AssertCommand(["gnt-node", "health"])
 343
 344     # Those commands should fail as they expect no data regardless of exit 0
 345     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 346     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 347
 348     try:
 349       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 350       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 351     finally:
 352       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 353
 354     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 355     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 356
 357     # Data, exit 1 (all should fail)
 358     _UpdateOobFile(exit_code_path, "1")
 359
 360     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 361     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 362
 363     try:
 364       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 365       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 366     finally:
 367       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 368
 369     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 370     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 371
 372     AssertCommand(["gnt-node", "power", "status", node_name],
 373                   fail=True)
 374     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 375
 376     AssertCommand(["gnt-node", "health", node_name],
 377                   fail=True)
 378     _AssertOobCall(verify_path, "health %s" % full_node_name)
 379
 380     AssertCommand(["gnt-node", "health"], fail=True)
 381
 382     # No data, exit 1 (all should fail)
 383     _UpdateOobFile(data_path, "")
 384     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 385     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 386
 387     try:
 388       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 389       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 390     finally:
 391       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 392
 393     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 394     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 395
 396     AssertCommand(["gnt-node", "power", "status", node_name],
 397                   fail=True)
 398     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 399
 400     AssertCommand(["gnt-node", "health", node_name],
 401                   fail=True)
 402     _AssertOobCall(verify_path, "health %s" % full_node_name)
 403
 404     AssertCommand(["gnt-node", "health"], fail=True)
 405
 406     # Different OOB script for node
 407     verify_path2 = qa_utils.UploadData(master.primary, "")
 408     oob_script = ("#!/bin/sh\n"
 409                   "echo \"$@\" > %s\n") % verify_path2
 410     oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 411
 412     try:
 413       AssertCommand(["gnt-node", "modify", "--node-parameters",
 414                      "oob_program=%s" % oob_path2, node_name])
 415       AssertCommand(["gnt-node", "power", "on", node_name])
 416       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 417     finally:
 418       AssertCommand(["gnt-node", "modify", "--node-parameters",
 419                      "oob_program=default", node_name])
 420       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 421   finally:
 422     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 423                    "oob_program="])
 424     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 425                    exit_code_path])
 426
 427
 428 def TestNodeList():
 429   """gnt-node list"""
 430   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 431
 432
 433 def TestNodeListFields():
 434   """gnt-node list-fields"""
 435   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
 436
 437
 438 def TestNodeListDrbd(node):
 439   """gnt-node list-drbd"""
 440   AssertCommand(["gnt-node", "list-drbd", node.primary])
 441
 442
 443 def _BuildSetESCmd(action, value, node_name):
 444   cmd = ["gnt-node"]
 445   if action == "add":
 446     cmd.extend(["add", "--readd"])
 447   else:
 448     cmd.append("modify")
 449   cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
 450   return cmd
 451
 452
 453 def TestExclStorSingleNode(node):
 454   """gnt-node add/modify cannot change the exclusive_storage flag.
 455
 456   """
 457   for action in ["add", "modify"]:
 458     for value in (True, False, "default"):
 459       AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)