code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011, 2012 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.added:
  40     raise qa_error.Error("Node %s already in cluster" % node["primary"])
  41   elif readd and not node.added:
  42     raise qa_error.Error("Node %s not yet in cluster" % node["primary"])
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.get("secondary", None):
  46     cmd.append("--secondary-ip=%s" % node["secondary"])
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node["primary"])
  50
  51   AssertCommand(cmd)
  52
  53   if readd:
  54     assert node.added
  55   else:
  56     node.MarkAdded()
  57
  58
  59 def _NodeRemove(node):
  60   AssertCommand(["gnt-node", "remove", node["primary"]])
  61   node.MarkRemoved()
  62
  63
  64 def MakeNodeOffline(node, value):
  65   """gnt-node modify --offline=value"""
  66   # value in ["yes", "no"]
  67   AssertCommand(["gnt-node", "modify", "--offline", value, node["primary"]])
  68
  69
  70 def TestNodeAddAll():
  71   """Adding all nodes to cluster."""
  72   master = qa_config.GetMasterNode()
  73   for node in qa_config.get("nodes"):
  74     if node != master:
  75       _NodeAdd(node, readd=False)
  76
  77
  78 def MarkNodeAddedAll():
  79   """Mark all nodes as added.
  80
  81   This is useful if we don't create the cluster ourselves (in qa).
  82
  83   """
  84   master = qa_config.GetMasterNode()
  85   for node in qa_config.get("nodes"):
  86     if node != master:
  87       node.MarkAdded()
  88
  89
  90 def TestNodeRemoveAll():
  91   """Removing all nodes from cluster."""
  92   master = qa_config.GetMasterNode()
  93   for node in qa_config.get("nodes"):
  94     if node != master:
  95       _NodeRemove(node)
  96
  97
  98 def TestNodeReadd(node):
  99   """gnt-node add --readd"""
 100   _NodeAdd(node, readd=True)
 101
 102
 103 def TestNodeInfo():
 104   """gnt-node info"""
 105   AssertCommand(["gnt-node", "info"])
 106
 107
 108 def TestNodeVolumes():
 109   """gnt-node volumes"""
 110   AssertCommand(["gnt-node", "volumes"])
 111
 112
 113 def TestNodeStorage():
 114   """gnt-node storage"""
 115   master = qa_config.GetMasterNode()
 116
 117   for storage_type in constants.VALID_STORAGE_TYPES:
 118     # Test simple list
 119     AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
 120
 121     # Test all storage fields
 122     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 123            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 124                                     [constants.SF_NODE, constants.SF_TYPE])]
 125     AssertCommand(cmd)
 126
 127     # Get list of valid storage devices
 128     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 129            "--output=node,name,allocatable", "--separator=|",
 130            "--no-headers"]
 131     output = qa_utils.GetCommandOutput(master["primary"],
 132                                        utils.ShellQuoteArgs(cmd))
 133
 134     # Test with up to two devices
 135     testdevcount = 2
 136
 137     for line in output.splitlines()[:testdevcount]:
 138       (node_name, st_name, st_allocatable) = line.split("|")
 139
 140       # Dummy modification without any changes
 141       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 142       AssertCommand(cmd)
 143
 144       # Make sure we end up with the same value as before
 145       if st_allocatable.lower() == "y":
 146         test_allocatable = ["no", "yes"]
 147       else:
 148         test_allocatable = ["yes", "no"]
 149
 150       fail = (constants.SF_ALLOCATABLE not in
 151               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 152
 153       for i in test_allocatable:
 154         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 155                        node_name, storage_type, st_name], fail=fail)
 156
 157         # Verify list output
 158         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 159                "--output=name,allocatable", "--separator=|",
 160                "--no-headers", node_name]
 161         listout = qa_utils.GetCommandOutput(master["primary"],
 162                                             utils.ShellQuoteArgs(cmd))
 163         for line in listout.splitlines():
 164           (vfy_name, vfy_allocatable) = line.split("|")
 165           if vfy_name == st_name and not fail:
 166             AssertEqual(vfy_allocatable, i[0].upper())
 167           else:
 168             AssertEqual(vfy_allocatable, st_allocatable)
 169
 170       # Test repair functionality
 171       fail = (constants.SO_FIX_CONSISTENCY not in
 172               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 173       AssertCommand(["gnt-node", "repair-storage", node_name,
 174                      storage_type, st_name], fail=fail)
 175
 176
 177 def TestNodeFailover(node, node2):
 178   """gnt-node failover"""
 179   if qa_utils.GetNodeInstances(node2, secondaries=False):
 180     raise qa_error.UnusableNodeError("Secondary node has at least one"
 181                                      " primary instance. This test requires"
 182                                      " it to have no primary instances.")
 183
 184   # Fail over to secondary node
 185   AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
 186
 187   # ... and back again.
 188   AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
 189
 190
 191 def TestNodeEvacuate(node, node2):
 192   """gnt-node evacuate"""
 193   node3 = qa_config.AcquireNode(exclude=[node, node2])
 194   try:
 195     if qa_utils.GetNodeInstances(node3, secondaries=True):
 196       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 197                                        " secondary instance. This test requires"
 198                                        " it to have no secondary instances.")
 199
 200     # Evacuate all secondary instances
 201     AssertCommand(["gnt-node", "evacuate", "-f",
 202                    "--new-secondary=%s" % node3["primary"], node2["primary"]])
 203
 204     # ... and back again.
 205     AssertCommand(["gnt-node", "evacuate", "-f",
 206                    "--new-secondary=%s" % node2["primary"], node3["primary"]])
 207   finally:
 208     node3.Release()
 209
 210
 211 def TestNodeModify(node):
 212   """gnt-node modify"""
 213   for flag in ["master-candidate", "drained", "offline"]:
 214     for value in ["yes", "no"]:
 215       AssertCommand(["gnt-node", "modify", "--force",
 216                      "--%s=%s" % (flag, value), node["primary"]])
 217
 218   AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
 219                  "--auto-promote", node["primary"]])
 220
 221   # Test setting secondary IP address
 222   AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node["secondary"],
 223                  node["primary"]])
 224
 225
 226 def _CreateOobScriptStructure():
 227   """Create a simple OOB handling script and its structure."""
 228   master = qa_config.GetMasterNode()
 229
 230   data_path = qa_utils.UploadData(master["primary"], "")
 231   verify_path = qa_utils.UploadData(master["primary"], "")
 232   exit_code_path = qa_utils.UploadData(master["primary"], "")
 233
 234   oob_script = (("#!/bin/bash\n"
 235                  "echo \"$@\" > %s\n"
 236                  "cat %s\n"
 237                  "exit $(< %s)\n") %
 238                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 239                  utils.ShellQuote(exit_code_path)))
 240   oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 241
 242   return [oob_path, verify_path, data_path, exit_code_path]
 243
 244
 245 def _UpdateOobFile(path, data):
 246   """Updates the data file with data."""
 247   master = qa_config.GetMasterNode()
 248   qa_utils.UploadData(master["primary"], data, filename=path)
 249
 250
 251 def _AssertOobCall(verify_path, expected_args):
 252   """Assert the OOB call was performed with expetected args."""
 253   master = qa_config.GetMasterNode()
 254
 255   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 256   output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd,
 257                                      tty=False)
 258
 259   AssertEqual(expected_args, output.strip())
 260
 261
 262 def TestOutOfBand():
 263   """gnt-node power"""
 264   master = qa_config.GetMasterNode()
 265
 266   node = qa_config.AcquireNode(exclude=master)
 267
 268   master_name = master["primary"]
 269   node_name = node["primary"]
 270   full_node_name = qa_utils.ResolveNodeName(node)
 271
 272   (oob_path, verify_path,
 273    data_path, exit_code_path) = _CreateOobScriptStructure()
 274
 275   try:
 276     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 277                    "oob_program=%s" % oob_path])
 278
 279     # No data, exit 0
 280     _UpdateOobFile(exit_code_path, "0")
 281
 282     AssertCommand(["gnt-node", "power", "on", node_name])
 283     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 284
 285     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 286     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 287
 288     # Power off on master without options should fail
 289     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 290     # With force master it should still fail
 291     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 292                    master_name],
 293                   fail=True)
 294
 295     # Verify we can't transform back to online when not yet powered on
 296     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 297                   fail=True)
 298     # Now reset state
 299     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 300                    node_name])
 301
 302     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 303     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 304
 305     # Those commands should fail as they expect output which isn't provided yet
 306     # But they should have called the oob helper nevermind
 307     AssertCommand(["gnt-node", "power", "status", node_name],
 308                   fail=True)
 309     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 310
 311     AssertCommand(["gnt-node", "health", node_name],
 312                   fail=True)
 313     _AssertOobCall(verify_path, "health %s" % full_node_name)
 314
 315     AssertCommand(["gnt-node", "health"], fail=True)
 316
 317     # Correct Data, exit 0
 318     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 319
 320     AssertCommand(["gnt-node", "power", "status", node_name])
 321     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 322
 323     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 324                                                    ["disk0", "CRITICAL"]]))
 325
 326     AssertCommand(["gnt-node", "health", node_name])
 327     _AssertOobCall(verify_path, "health %s" % full_node_name)
 328
 329     AssertCommand(["gnt-node", "health"])
 330
 331     # Those commands should fail as they expect no data regardless of exit 0
 332     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 333     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 334
 335     try:
 336       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 337       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 338     finally:
 339       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 340
 341     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 342     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 343
 344     # Data, exit 1 (all should fail)
 345     _UpdateOobFile(exit_code_path, "1")
 346
 347     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 348     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 349
 350     try:
 351       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 352       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 353     finally:
 354       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 355
 356     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 357     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 358
 359     AssertCommand(["gnt-node", "power", "status", node_name],
 360                   fail=True)
 361     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 362
 363     AssertCommand(["gnt-node", "health", node_name],
 364                   fail=True)
 365     _AssertOobCall(verify_path, "health %s" % full_node_name)
 366
 367     AssertCommand(["gnt-node", "health"], fail=True)
 368
 369     # No data, exit 1 (all should fail)
 370     _UpdateOobFile(data_path, "")
 371     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 372     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 373
 374     try:
 375       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 376       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 377     finally:
 378       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 379
 380     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 381     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 382
 383     AssertCommand(["gnt-node", "power", "status", node_name],
 384                   fail=True)
 385     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 386
 387     AssertCommand(["gnt-node", "health", node_name],
 388                   fail=True)
 389     _AssertOobCall(verify_path, "health %s" % full_node_name)
 390
 391     AssertCommand(["gnt-node", "health"], fail=True)
 392
 393     # Different OOB script for node
 394     verify_path2 = qa_utils.UploadData(master["primary"], "")
 395     oob_script = ("#!/bin/sh\n"
 396                   "echo \"$@\" > %s\n") % verify_path2
 397     oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 398
 399     try:
 400       AssertCommand(["gnt-node", "modify", "--node-parameters",
 401                      "oob_program=%s" % oob_path2, node_name])
 402       AssertCommand(["gnt-node", "power", "on", node_name])
 403       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 404     finally:
 405       AssertCommand(["gnt-node", "modify", "--node-parameters",
 406                      "oob_program=default", node_name])
 407       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 408   finally:
 409     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 410                    "oob_program="])
 411     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 412                    exit_code_path])
 413
 414
 415 def TestNodeList():
 416   """gnt-node list"""
 417   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 418
 419
 420 def TestNodeListFields():
 421   """gnt-node list-fields"""
 422   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
 423
 424
 425 def TestNodeListDrbd(node):
 426   """gnt-node list-drbd"""
 427   AssertCommand(["gnt-node", "list-drbd", node["primary"]])