code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.added:
  40     raise qa_error.Error("Node %s already in cluster" % node.primary)
  41   elif readd and not node.added:
  42     raise qa_error.Error("Node %s not yet in cluster" % node.primary)
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.secondary:
  46     cmd.append("--secondary-ip=%s" % node.secondary)
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node.primary)
  50
  51   AssertCommand(cmd)
  52
  53   if readd:
  54     assert node.added
  55   else:
  56     node.MarkAdded()
  57
  58
  59 def _NodeRemove(node):
  60   AssertCommand(["gnt-node", "remove", node.primary])
  61   node.MarkRemoved()
  62
  63
  64 def MakeNodeOffline(node, value):
  65   """gnt-node modify --offline=value"""
  66   # value in ["yes", "no"]
  67   AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
  68
  69
  70 def TestNodeAddAll():
  71   """Adding all nodes to cluster."""
  72   master = qa_config.GetMasterNode()
  73   for node in qa_config.get("nodes"):
  74     if node != master:
  75       _NodeAdd(node, readd=False)
  76
  77
  78 def MarkNodeAddedAll():
  79   """Mark all nodes as added.
  80
  81   This is useful if we don't create the cluster ourselves (in qa).
  82
  83   """
  84   master = qa_config.GetMasterNode()
  85   for node in qa_config.get("nodes"):
  86     if node != master:
  87       node.MarkAdded()
  88
  89
  90 def TestNodeRemoveAll():
  91   """Removing all nodes from cluster."""
  92   master = qa_config.GetMasterNode()
  93   for node in qa_config.get("nodes"):
  94     if node != master:
  95       _NodeRemove(node)
  96
  97
  98 def TestNodeReadd(node):
  99   """gnt-node add --readd"""
 100   _NodeAdd(node, readd=True)
 101
 102
 103 def TestNodeInfo():
 104   """gnt-node info"""
 105   AssertCommand(["gnt-node", "info"])
 106
 107
 108 def TestNodeVolumes():
 109   """gnt-node volumes"""
 110   AssertCommand(["gnt-node", "volumes"])
 111
 112
 113 def TestNodeStorage():
 114   """gnt-node storage"""
 115   master = qa_config.GetMasterNode()
 116
 117   # FIXME: test all storage_types in constants.STORAGE_TYPES
 118   # as soon as they are implemented.
 119   enabled_storage_types = qa_config.GetEnabledStorageTypes()
 120   testable_storage_types = list(set(enabled_storage_types).intersection(
 121       set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV])))
 122
 123   for storage_type in testable_storage_types:
 124
 125     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
 126
 127     # Test simple list
 128     AssertCommand(cmd)
 129
 130     # Test all storage fields
 131     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 132            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 133                                     [constants.SF_NODE, constants.SF_TYPE])]
 134     AssertCommand(cmd)
 135
 136     # Get list of valid storage devices
 137     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 138            "--output=node,name,allocatable", "--separator=|",
 139            "--no-headers"]
 140     output = qa_utils.GetCommandOutput(master.primary,
 141                                        utils.ShellQuoteArgs(cmd))
 142
 143     # Test with up to two devices
 144     testdevcount = 2
 145
 146     for line in output.splitlines()[:testdevcount]:
 147       (node_name, st_name, st_allocatable) = line.split("|")
 148
 149       # Dummy modification without any changes
 150       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 151       AssertCommand(cmd)
 152
 153       # Make sure we end up with the same value as before
 154       if st_allocatable.lower() == "y":
 155         test_allocatable = ["no", "yes"]
 156       else:
 157         test_allocatable = ["yes", "no"]
 158
 159       fail = (constants.SF_ALLOCATABLE not in
 160               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 161
 162       for i in test_allocatable:
 163         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 164                        node_name, storage_type, st_name], fail=fail)
 165
 166         # Verify list output
 167         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 168                "--output=name,allocatable", "--separator=|",
 169                "--no-headers", node_name]
 170         listout = qa_utils.GetCommandOutput(master.primary,
 171                                             utils.ShellQuoteArgs(cmd))
 172         for line in listout.splitlines():
 173           (vfy_name, vfy_allocatable) = line.split("|")
 174           if vfy_name == st_name and not fail:
 175             AssertEqual(vfy_allocatable, i[0].upper())
 176           else:
 177             AssertEqual(vfy_allocatable, st_allocatable)
 178
 179       # Test repair functionality
 180       fail = (constants.SO_FIX_CONSISTENCY not in
 181               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 182       AssertCommand(["gnt-node", "repair-storage", node_name,
 183                      storage_type, st_name], fail=fail)
 184
 185
 186 def TestNodeFailover(node, node2):
 187   """gnt-node failover"""
 188   if qa_utils.GetNodeInstances(node2, secondaries=False):
 189     raise qa_error.UnusableNodeError("Secondary node has at least one"
 190                                      " primary instance. This test requires"
 191                                      " it to have no primary instances.")
 192
 193   # Fail over to secondary node
 194   AssertCommand(["gnt-node", "failover", "-f", node.primary])
 195
 196   # ... and back again.
 197   AssertCommand(["gnt-node", "failover", "-f", node2.primary])
 198
 199
 200 def TestNodeMigrate(node, node2):
 201   """gnt-node migrate"""
 202   if qa_utils.GetNodeInstances(node2, secondaries=False):
 203     raise qa_error.UnusableNodeError("Secondary node has at least one"
 204                                      " primary instance. This test requires"
 205                                      " it to have no primary instances.")
 206
 207   # Migrate to secondary node
 208   AssertCommand(["gnt-node", "migrate", "-f", node.primary])
 209
 210   # ... and back again.
 211   AssertCommand(["gnt-node", "migrate", "-f", node2.primary])
 212
 213
 214 def TestNodeEvacuate(node, node2):
 215   """gnt-node evacuate"""
 216   node3 = qa_config.AcquireNode(exclude=[node, node2])
 217   try:
 218     if qa_utils.GetNodeInstances(node3, secondaries=True):
 219       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 220                                        " secondary instance. This test requires"
 221                                        " it to have no secondary instances.")
 222
 223     # Evacuate all secondary instances
 224     AssertCommand(["gnt-node", "evacuate", "-f",
 225                    "--new-secondary=%s" % node3.primary, node2.primary])
 226
 227     # ... and back again.
 228     AssertCommand(["gnt-node", "evacuate", "-f",
 229                    "--new-secondary=%s" % node2.primary, node3.primary])
 230   finally:
 231     node3.Release()
 232
 233
 234 def TestNodeModify(node):
 235   """gnt-node modify"""
 236
 237   # make sure enough master candidates will be available by disabling the
 238   # master candidate role first with --auto-promote
 239   AssertCommand(["gnt-node", "modify", "--master-candidate=no",
 240                 "--auto-promote", node.primary])
 241
 242   # now it's save to force-remove the master candidate role
 243   for flag in ["master-candidate", "drained", "offline"]:
 244     for value in ["yes", "no"]:
 245       AssertCommand(["gnt-node", "modify", "--force",
 246                      "--%s=%s" % (flag, value), node.primary])
 247
 248   AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary])
 249
 250   # Test setting secondary IP address
 251   AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
 252                  node.primary])
 253
 254
 255 def _CreateOobScriptStructure():
 256   """Create a simple OOB handling script and its structure."""
 257   master = qa_config.GetMasterNode()
 258
 259   data_path = qa_utils.UploadData(master.primary, "")
 260   verify_path = qa_utils.UploadData(master.primary, "")
 261   exit_code_path = qa_utils.UploadData(master.primary, "")
 262
 263   oob_script = (("#!/bin/bash\n"
 264                  "echo \"$@\" > %s\n"
 265                  "cat %s\n"
 266                  "exit $(< %s)\n") %
 267                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 268                  utils.ShellQuote(exit_code_path)))
 269   oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 270
 271   return [oob_path, verify_path, data_path, exit_code_path]
 272
 273
 274 def _UpdateOobFile(path, data):
 275   """Updates the data file with data."""
 276   master = qa_config.GetMasterNode()
 277   qa_utils.UploadData(master.primary, data, filename=path)
 278
 279
 280 def _AssertOobCall(verify_path, expected_args):
 281   """Assert the OOB call was performed with expetected args."""
 282   master = qa_config.GetMasterNode()
 283
 284   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 285   output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
 286                                      tty=False)
 287
 288   AssertEqual(expected_args, output.strip())
 289
 290
 291 def TestOutOfBand():
 292   """gnt-node power"""
 293   master = qa_config.GetMasterNode()
 294
 295   node = qa_config.AcquireNode(exclude=master)
 296
 297   master_name = master.primary
 298   node_name = node.primary
 299   full_node_name = qa_utils.ResolveNodeName(node)
 300
 301   (oob_path, verify_path,
 302    data_path, exit_code_path) = _CreateOobScriptStructure()
 303
 304   try:
 305     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 306                    "oob_program=%s" % oob_path])
 307
 308     # No data, exit 0
 309     _UpdateOobFile(exit_code_path, "0")
 310
 311     AssertCommand(["gnt-node", "power", "on", node_name])
 312     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 313
 314     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 315     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 316
 317     # Power off on master without options should fail
 318     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 319     # With force master it should still fail
 320     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 321                    master_name],
 322                   fail=True)
 323
 324     # Verify we can't transform back to online when not yet powered on
 325     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 326                   fail=True)
 327     # Now reset state
 328     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 329                    node_name])
 330
 331     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 332     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 333
 334     # Those commands should fail as they expect output which isn't provided yet
 335     # But they should have called the oob helper nevermind
 336     AssertCommand(["gnt-node", "power", "status", node_name],
 337                   fail=True)
 338     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 339
 340     AssertCommand(["gnt-node", "health", node_name],
 341                   fail=True)
 342     _AssertOobCall(verify_path, "health %s" % full_node_name)
 343
 344     AssertCommand(["gnt-node", "health"], fail=True)
 345
 346     # Correct Data, exit 0
 347     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 348
 349     AssertCommand(["gnt-node", "power", "status", node_name])
 350     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 351
 352     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 353                                                    ["disk0", "CRITICAL"]]))
 354
 355     AssertCommand(["gnt-node", "health", node_name])
 356     _AssertOobCall(verify_path, "health %s" % full_node_name)
 357
 358     AssertCommand(["gnt-node", "health"])
 359
 360     # Those commands should fail as they expect no data regardless of exit 0
 361     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 362     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 363
 364     try:
 365       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 366       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 367     finally:
 368       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 369
 370     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 371     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 372
 373     # Data, exit 1 (all should fail)
 374     _UpdateOobFile(exit_code_path, "1")
 375
 376     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 377     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 378
 379     try:
 380       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 381       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 382     finally:
 383       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 384
 385     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 386     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 387
 388     AssertCommand(["gnt-node", "power", "status", node_name],
 389                   fail=True)
 390     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 391
 392     AssertCommand(["gnt-node", "health", node_name],
 393                   fail=True)
 394     _AssertOobCall(verify_path, "health %s" % full_node_name)
 395
 396     AssertCommand(["gnt-node", "health"], fail=True)
 397
 398     # No data, exit 1 (all should fail)
 399     _UpdateOobFile(data_path, "")
 400     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 401     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 402
 403     try:
 404       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 405       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 406     finally:
 407       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 408
 409     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 410     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 411
 412     AssertCommand(["gnt-node", "power", "status", node_name],
 413                   fail=True)
 414     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 415
 416     AssertCommand(["gnt-node", "health", node_name],
 417                   fail=True)
 418     _AssertOobCall(verify_path, "health %s" % full_node_name)
 419
 420     AssertCommand(["gnt-node", "health"], fail=True)
 421
 422     # Different OOB script for node
 423     verify_path2 = qa_utils.UploadData(master.primary, "")
 424     oob_script = ("#!/bin/sh\n"
 425                   "echo \"$@\" > %s\n") % verify_path2
 426     oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
 427
 428     try:
 429       AssertCommand(["gnt-node", "modify", "--node-parameters",
 430                      "oob_program=%s" % oob_path2, node_name])
 431       AssertCommand(["gnt-node", "power", "on", node_name])
 432       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 433     finally:
 434       AssertCommand(["gnt-node", "modify", "--node-parameters",
 435                      "oob_program=default", node_name])
 436       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 437   finally:
 438     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 439                    "oob_program="])
 440     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 441                    exit_code_path])
 442
 443
 444 def TestNodeList():
 445   """gnt-node list"""
 446   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 447
 448
 449 def TestNodeListFields():
 450   """gnt-node list-fields"""
 451   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
 452
 453
 454 def TestNodeListDrbd(node):
 455   """gnt-node list-drbd"""
 456   AssertCommand(["gnt-node", "list-drbd", node.primary])
 457
 458
 459 def _BuildSetESCmd(action, value, node_name):
 460   cmd = ["gnt-node"]
 461   if action == "add":
 462     cmd.extend(["add", "--readd"])
 463   else:
 464     cmd.append("modify")
 465   cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
 466   return cmd
 467
 468
 469 def TestExclStorSingleNode(node):
 470   """gnt-node add/modify cannot change the exclusive_storage flag.
 471
 472   """
 473   for action in ["add", "modify"]:
 474     for value in (True, False, "default"):
 475       AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)