code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011, 2012 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.get("_added", False):
  40     raise qa_error.Error("Node %s already in cluster" % node["primary"])
  41   elif readd and not node.get("_added", False):
  42     raise qa_error.Error("Node %s not yet in cluster" % node["primary"])
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.get("secondary", None):
  46     cmd.append("--secondary-ip=%s" % node["secondary"])
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node["primary"])
  50
  51   AssertCommand(cmd)
  52
  53   node["_added"] = True
  54
  55
  56 def _NodeRemove(node):
  57   AssertCommand(["gnt-node", "remove", node["primary"]])
  58   node["_added"] = False
  59
  60
  61 def MakeNodeOffline(node, value):
  62   """gnt-node modify --offline=value"""
  63   # value in ["yes", "no"]
  64   AssertCommand(["gnt-node", "modify", "--offline", value, node["primary"]])
  65
  66
  67 def TestNodeAddAll():
  68   """Adding all nodes to cluster."""
  69   master = qa_config.GetMasterNode()
  70   for node in qa_config.get("nodes"):
  71     if node != master:
  72       _NodeAdd(node, readd=False)
  73
  74
  75 def MarkNodeAddedAll():
  76   """Mark all nodes as added.
  77
  78   This is useful if we don't create the cluster ourselves (in qa).
  79
  80   """
  81   master = qa_config.GetMasterNode()
  82   for node in qa_config.get("nodes"):
  83     if node != master:
  84       node["_added"] = True
  85
  86
  87 def TestNodeRemoveAll():
  88   """Removing all nodes from cluster."""
  89   master = qa_config.GetMasterNode()
  90   for node in qa_config.get("nodes"):
  91     if node != master:
  92       _NodeRemove(node)
  93
  94
  95 def TestNodeReadd(node):
  96   """gnt-node add --readd"""
  97   _NodeAdd(node, readd=True)
  98
  99
 100 def TestNodeInfo():
 101   """gnt-node info"""
 102   AssertCommand(["gnt-node", "info"])
 103
 104
 105 def TestNodeVolumes():
 106   """gnt-node volumes"""
 107   AssertCommand(["gnt-node", "volumes"])
 108
 109
 110 def TestNodeStorage():
 111   """gnt-node storage"""
 112   master = qa_config.GetMasterNode()
 113
 114   for storage_type in constants.VALID_STORAGE_TYPES:
 115     # Test simple list
 116     AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
 117
 118     # Test all storage fields
 119     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 120            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 121                                     [constants.SF_NODE, constants.SF_TYPE])]
 122     AssertCommand(cmd)
 123
 124     # Get list of valid storage devices
 125     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 126            "--output=node,name,allocatable", "--separator=|",
 127            "--no-headers"]
 128     output = qa_utils.GetCommandOutput(master["primary"],
 129                                        utils.ShellQuoteArgs(cmd))
 130
 131     # Test with up to two devices
 132     testdevcount = 2
 133
 134     for line in output.splitlines()[:testdevcount]:
 135       (node_name, st_name, st_allocatable) = line.split("|")
 136
 137       # Dummy modification without any changes
 138       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 139       AssertCommand(cmd)
 140
 141       # Make sure we end up with the same value as before
 142       if st_allocatable.lower() == "y":
 143         test_allocatable = ["no", "yes"]
 144       else:
 145         test_allocatable = ["yes", "no"]
 146
 147       fail = (constants.SF_ALLOCATABLE not in
 148               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 149
 150       for i in test_allocatable:
 151         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 152                        node_name, storage_type, st_name], fail=fail)
 153
 154         # Verify list output
 155         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 156                "--output=name,allocatable", "--separator=|",
 157                "--no-headers", node_name]
 158         listout = qa_utils.GetCommandOutput(master["primary"],
 159                                             utils.ShellQuoteArgs(cmd))
 160         for line in listout.splitlines():
 161           (vfy_name, vfy_allocatable) = line.split("|")
 162           if vfy_name == st_name and not fail:
 163             AssertEqual(vfy_allocatable, i[0].upper())
 164           else:
 165             AssertEqual(vfy_allocatable, st_allocatable)
 166
 167       # Test repair functionality
 168       fail = (constants.SO_FIX_CONSISTENCY not in
 169               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 170       AssertCommand(["gnt-node", "repair-storage", node_name,
 171                      storage_type, st_name], fail=fail)
 172
 173
 174 def TestNodeFailover(node, node2):
 175   """gnt-node failover"""
 176   if qa_utils.GetNodeInstances(node2, secondaries=False):
 177     raise qa_error.UnusableNodeError("Secondary node has at least one"
 178                                      " primary instance. This test requires"
 179                                      " it to have no primary instances.")
 180
 181   # Fail over to secondary node
 182   AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
 183
 184   # ... and back again.
 185   AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
 186
 187
 188 def TestNodeEvacuate(node, node2):
 189   """gnt-node evacuate"""
 190   node3 = qa_config.AcquireNode(exclude=[node, node2])
 191   try:
 192     if qa_utils.GetNodeInstances(node3, secondaries=True):
 193       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 194                                        " secondary instance. This test requires"
 195                                        " it to have no secondary instances.")
 196
 197     # Evacuate all secondary instances
 198     AssertCommand(["gnt-node", "evacuate", "-f",
 199                    "--new-secondary=%s" % node3["primary"], node2["primary"]])
 200
 201     # ... and back again.
 202     AssertCommand(["gnt-node", "evacuate", "-f",
 203                    "--new-secondary=%s" % node2["primary"], node3["primary"]])
 204   finally:
 205     qa_config.ReleaseNode(node3)
 206
 207
 208 def TestNodeModify(node):
 209   """gnt-node modify"""
 210   for flag in ["master-candidate", "drained", "offline"]:
 211     for value in ["yes", "no"]:
 212       AssertCommand(["gnt-node", "modify", "--force",
 213                      "--%s=%s" % (flag, value), node["primary"]])
 214
 215   AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
 216                  "--auto-promote", node["primary"]])
 217
 218   # Test setting secondary IP address
 219   AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node["secondary"],
 220                  node["primary"]])
 221
 222
 223 def _CreateOobScriptStructure():
 224   """Create a simple OOB handling script and its structure."""
 225   master = qa_config.GetMasterNode()
 226
 227   data_path = qa_utils.UploadData(master["primary"], "")
 228   verify_path = qa_utils.UploadData(master["primary"], "")
 229   exit_code_path = qa_utils.UploadData(master["primary"], "")
 230
 231   oob_script = (("#!/bin/bash\n"
 232                  "echo \"$@\" > %s\n"
 233                  "cat %s\n"
 234                  "exit $(< %s)\n") %
 235                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 236                  utils.ShellQuote(exit_code_path)))
 237   oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 238
 239   return [oob_path, verify_path, data_path, exit_code_path]
 240
 241
 242 def _UpdateOobFile(path, data):
 243   """Updates the data file with data."""
 244   master = qa_config.GetMasterNode()
 245   qa_utils.UploadData(master["primary"], data, filename=path)
 246
 247
 248 def _AssertOobCall(verify_path, expected_args):
 249   """Assert the OOB call was performed with expetected args."""
 250   master = qa_config.GetMasterNode()
 251
 252   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 253   output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd,
 254                                      tty=False)
 255
 256   AssertEqual(expected_args, output.strip())
 257
 258
 259 def TestOutOfBand():
 260   """gnt-node power"""
 261   master = qa_config.GetMasterNode()
 262
 263   node = qa_config.AcquireNode(exclude=master)
 264
 265   master_name = master["primary"]
 266   node_name = node["primary"]
 267   full_node_name = qa_utils.ResolveNodeName(node)
 268
 269   (oob_path, verify_path,
 270    data_path, exit_code_path) = _CreateOobScriptStructure()
 271
 272   try:
 273     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 274                    "oob_program=%s" % oob_path])
 275
 276     # No data, exit 0
 277     _UpdateOobFile(exit_code_path, "0")
 278
 279     AssertCommand(["gnt-node", "power", "on", node_name])
 280     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 281
 282     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 283     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 284
 285     # Power off on master without options should fail
 286     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 287     # With force master it should still fail
 288     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 289                    master_name],
 290                   fail=True)
 291
 292     # Verify we can't transform back to online when not yet powered on
 293     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 294                   fail=True)
 295     # Now reset state
 296     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 297                    node_name])
 298
 299     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 300     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 301
 302     # Those commands should fail as they expect output which isn't provided yet
 303     # But they should have called the oob helper nevermind
 304     AssertCommand(["gnt-node", "power", "status", node_name],
 305                   fail=True)
 306     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 307
 308     AssertCommand(["gnt-node", "health", node_name],
 309                   fail=True)
 310     _AssertOobCall(verify_path, "health %s" % full_node_name)
 311
 312     AssertCommand(["gnt-node", "health"], fail=True)
 313
 314     # Correct Data, exit 0
 315     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 316
 317     AssertCommand(["gnt-node", "power", "status", node_name])
 318     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 319
 320     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 321                                                    ["disk0", "CRITICAL"]]))
 322
 323     AssertCommand(["gnt-node", "health", node_name])
 324     _AssertOobCall(verify_path, "health %s" % full_node_name)
 325
 326     AssertCommand(["gnt-node", "health"])
 327
 328     # Those commands should fail as they expect no data regardless of exit 0
 329     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 330     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 331
 332     try:
 333       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 334       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 335     finally:
 336       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 337
 338     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 339     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 340
 341     # Data, exit 1 (all should fail)
 342     _UpdateOobFile(exit_code_path, "1")
 343
 344     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 345     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 346
 347     try:
 348       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 349       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 350     finally:
 351       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 352
 353     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 354     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 355
 356     AssertCommand(["gnt-node", "power", "status", node_name],
 357                   fail=True)
 358     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 359
 360     AssertCommand(["gnt-node", "health", node_name],
 361                   fail=True)
 362     _AssertOobCall(verify_path, "health %s" % full_node_name)
 363
 364     AssertCommand(["gnt-node", "health"], fail=True)
 365
 366     # No data, exit 1 (all should fail)
 367     _UpdateOobFile(data_path, "")
 368     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 369     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 370
 371     try:
 372       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 373       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 374     finally:
 375       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 376
 377     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 378     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 379
 380     AssertCommand(["gnt-node", "power", "status", node_name],
 381                   fail=True)
 382     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 383
 384     AssertCommand(["gnt-node", "health", node_name],
 385                   fail=True)
 386     _AssertOobCall(verify_path, "health %s" % full_node_name)
 387
 388     AssertCommand(["gnt-node", "health"], fail=True)
 389
 390     # Different OOB script for node
 391     verify_path2 = qa_utils.UploadData(master["primary"], "")
 392     oob_script = ("#!/bin/sh\n"
 393                   "echo \"$@\" > %s\n") % verify_path2
 394     oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 395
 396     try:
 397       AssertCommand(["gnt-node", "modify", "--node-parameters",
 398                      "oob_program=%s" % oob_path2, node_name])
 399       AssertCommand(["gnt-node", "power", "on", node_name])
 400       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 401     finally:
 402       AssertCommand(["gnt-node", "modify", "--node-parameters",
 403                      "oob_program=default", node_name])
 404       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 405   finally:
 406     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 407                    "oob_program="])
 408     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 409                    exit_code_path])
 410
 411
 412 def TestNodeList():
 413   """gnt-node list"""
 414   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 415
 416
 417 def TestNodeListFields():
 418   """gnt-node list-fields"""
 419   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
 420
 421
 422 def TestNodeListDrbd(node):
 423   """gnt-node list-drbd"""
 424   AssertCommand(["gnt-node", "list-drbd", node["primary"]])