code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.get("_added", False):
  40     raise qa_error.Error("Node %s already in cluster" % node["primary"])
  41   elif readd and not node.get("_added", False):
  42     raise qa_error.Error("Node %s not yet in cluster" % node["primary"])
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.get("secondary", None):
  46     cmd.append("--secondary-ip=%s" % node["secondary"])
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node["primary"])
  50
  51   AssertCommand(cmd)
  52
  53   node["_added"] = True
  54
  55
  56 def _NodeRemove(node):
  57   AssertCommand(["gnt-node", "remove", node["primary"]])
  58   node["_added"] = False
  59
  60
  61 def TestNodeAddAll():
  62   """Adding all nodes to cluster."""
  63   master = qa_config.GetMasterNode()
  64   for node in qa_config.get("nodes"):
  65     if node != master:
  66       _NodeAdd(node, readd=False)
  67
  68
  69 def MarkNodeAddedAll():
  70   """Mark all nodes as added.
  71
  72   This is useful if we don't create the cluster ourselves (in qa).
  73
  74   """
  75   master = qa_config.GetMasterNode()
  76   for node in qa_config.get("nodes"):
  77     if node != master:
  78       node["_added"] = True
  79
  80
  81 def TestNodeRemoveAll():
  82   """Removing all nodes from cluster."""
  83   master = qa_config.GetMasterNode()
  84   for node in qa_config.get("nodes"):
  85     if node != master:
  86       _NodeRemove(node)
  87
  88
  89 def TestNodeReadd(node):
  90   """gnt-node add --readd"""
  91   _NodeAdd(node, readd=True)
  92
  93
  94 def TestNodeInfo():
  95   """gnt-node info"""
  96   AssertCommand(["gnt-node", "info"])
  97
  98
  99 def TestNodeVolumes():
 100   """gnt-node volumes"""
 101   AssertCommand(["gnt-node", "volumes"])
 102
 103
 104 def TestNodeStorage():
 105   """gnt-node storage"""
 106   master = qa_config.GetMasterNode()
 107
 108   for storage_type in constants.VALID_STORAGE_TYPES:
 109     # Test simple list
 110     AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
 111
 112     # Test all storage fields
 113     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 114            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 115                                     [constants.SF_NODE, constants.SF_TYPE])]
 116     AssertCommand(cmd)
 117
 118     # Get list of valid storage devices
 119     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 120            "--output=node,name,allocatable", "--separator=|",
 121            "--no-headers"]
 122     output = qa_utils.GetCommandOutput(master["primary"],
 123                                        utils.ShellQuoteArgs(cmd))
 124
 125     # Test with up to two devices
 126     testdevcount = 2
 127
 128     for line in output.splitlines()[:testdevcount]:
 129       (node_name, st_name, st_allocatable) = line.split("|")
 130
 131       # Dummy modification without any changes
 132       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 133       AssertCommand(cmd)
 134
 135       # Make sure we end up with the same value as before
 136       if st_allocatable.lower() == "y":
 137         test_allocatable = ["no", "yes"]
 138       else:
 139         test_allocatable = ["yes", "no"]
 140
 141       fail = (constants.SF_ALLOCATABLE not in
 142               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 143
 144       for i in test_allocatable:
 145         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 146                        node_name, storage_type, st_name], fail=fail)
 147
 148         # Verify list output
 149         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 150                "--output=name,allocatable", "--separator=|",
 151                "--no-headers", node_name]
 152         listout = qa_utils.GetCommandOutput(master["primary"],
 153                                             utils.ShellQuoteArgs(cmd))
 154         for line in listout.splitlines():
 155           (vfy_name, vfy_allocatable) = line.split("|")
 156           if vfy_name == st_name and not fail:
 157             AssertEqual(vfy_allocatable, i[0].upper())
 158           else:
 159             AssertEqual(vfy_allocatable, st_allocatable)
 160
 161       # Test repair functionality
 162       fail = (constants.SO_FIX_CONSISTENCY not in
 163               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 164       AssertCommand(["gnt-node", "repair-storage", node_name,
 165                      storage_type, st_name], fail=fail)
 166
 167
 168 def TestNodeFailover(node, node2):
 169   """gnt-node failover"""
 170   if qa_utils.GetNodeInstances(node2, secondaries=False):
 171     raise qa_error.UnusableNodeError("Secondary node has at least one"
 172                                      " primary instance. This test requires"
 173                                      " it to have no primary instances.")
 174
 175   # Fail over to secondary node
 176   AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
 177
 178   # ... and back again.
 179   AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
 180
 181
 182 def TestNodeEvacuate(node, node2):
 183   """gnt-node evacuate"""
 184   node3 = qa_config.AcquireNode(exclude=[node, node2])
 185   try:
 186     if qa_utils.GetNodeInstances(node3, secondaries=True):
 187       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 188                                        " secondary instance. This test requires"
 189                                        " it to have no secondary instances.")
 190
 191     # Evacuate all secondary instances
 192     AssertCommand(["gnt-node", "evacuate", "-f",
 193                    "--new-secondary=%s" % node3["primary"], node2["primary"]])
 194
 195     # ... and back again.
 196     AssertCommand(["gnt-node", "evacuate", "-f",
 197                    "--new-secondary=%s" % node2["primary"], node3["primary"]])
 198   finally:
 199     qa_config.ReleaseNode(node3)
 200
 201
 202 def TestNodeModify(node):
 203   """gnt-node modify"""
 204   for flag in ["master-candidate", "drained", "offline"]:
 205     for value in ["yes", "no"]:
 206       AssertCommand(["gnt-node", "modify", "--force",
 207                      "--%s=%s" % (flag, value), node["primary"]])
 208
 209   AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
 210                  "--auto-promote", node["primary"]])
 211
 212   # Test setting secondary IP address
 213   AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node["secondary"],
 214                  node["primary"]])
 215
 216
 217 def _CreateOobScriptStructure():
 218   """Create a simple OOB handling script and its structure."""
 219   master = qa_config.GetMasterNode()
 220
 221   data_path = qa_utils.UploadData(master["primary"], "")
 222   verify_path = qa_utils.UploadData(master["primary"], "")
 223   exit_code_path = qa_utils.UploadData(master["primary"], "")
 224
 225   oob_script = (("#!/bin/bash\n"
 226                  "echo \"$@\" > %s\n"
 227                  "cat %s\n"
 228                  "exit $(< %s)\n") %
 229                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 230                  utils.ShellQuote(exit_code_path)))
 231   oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 232
 233   return [oob_path, verify_path, data_path, exit_code_path]
 234
 235
 236 def _UpdateOobFile(path, data):
 237   """Updates the data file with data."""
 238   master = qa_config.GetMasterNode()
 239   qa_utils.UploadData(master["primary"], data, filename=path)
 240
 241
 242 def _AssertOobCall(verify_path, expected_args):
 243   """Assert the OOB call was performed with expetected args."""
 244   master = qa_config.GetMasterNode()
 245
 246   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 247   output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd,
 248                                      tty=False)
 249
 250   AssertEqual(expected_args, output.strip())
 251
 252
 253 def TestOutOfBand():
 254   """gnt-node power"""
 255   master = qa_config.GetMasterNode()
 256
 257   node = qa_config.AcquireNode(exclude=master)
 258
 259   master_name = master["primary"]
 260   node_name = node["primary"]
 261   full_node_name = qa_utils.ResolveNodeName(node)
 262
 263   (oob_path, verify_path,
 264    data_path, exit_code_path) = _CreateOobScriptStructure()
 265
 266   try:
 267     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 268                    "oob_program=%s" % oob_path])
 269
 270     # No data, exit 0
 271     _UpdateOobFile(exit_code_path, "0")
 272
 273     AssertCommand(["gnt-node", "power", "on", node_name])
 274     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 275
 276     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 277     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 278
 279     # Power off on master without options should fail
 280     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 281     # With force master it should still fail
 282     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 283                    master_name],
 284                   fail=True)
 285
 286     # Verify we can't transform back to online when not yet powered on
 287     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 288                   fail=True)
 289     # Now reset state
 290     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 291                    node_name])
 292
 293     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 294     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 295
 296     # Those commands should fail as they expect output which isn't provided yet
 297     # But they should have called the oob helper nevermind
 298     AssertCommand(["gnt-node", "power", "status", node_name],
 299                   fail=True)
 300     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 301
 302     AssertCommand(["gnt-node", "health", node_name],
 303                   fail=True)
 304     _AssertOobCall(verify_path, "health %s" % full_node_name)
 305
 306     AssertCommand(["gnt-node", "health"], fail=True)
 307
 308     # Correct Data, exit 0
 309     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 310
 311     AssertCommand(["gnt-node", "power", "status", node_name])
 312     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 313
 314     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 315                                                    ["disk0", "CRITICAL"]]))
 316
 317     AssertCommand(["gnt-node", "health", node_name])
 318     _AssertOobCall(verify_path, "health %s" % full_node_name)
 319
 320     AssertCommand(["gnt-node", "health"])
 321
 322     # Those commands should fail as they expect no data regardless of exit 0
 323     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 324     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 325
 326     try:
 327       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 328       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 329     finally:
 330       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 331
 332     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 333     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 334
 335     # Data, exit 1 (all should fail)
 336     _UpdateOobFile(exit_code_path, "1")
 337
 338     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 339     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 340
 341     try:
 342       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 343       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 344     finally:
 345       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 346
 347     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 348     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 349
 350     AssertCommand(["gnt-node", "power", "status", node_name],
 351                   fail=True)
 352     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 353
 354     AssertCommand(["gnt-node", "health", node_name],
 355                   fail=True)
 356     _AssertOobCall(verify_path, "health %s" % full_node_name)
 357
 358     AssertCommand(["gnt-node", "health"], fail=True)
 359
 360     # No data, exit 1 (all should fail)
 361     _UpdateOobFile(data_path, "")
 362     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 363     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 364
 365     try:
 366       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 367       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 368     finally:
 369       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 370
 371     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 372     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 373
 374     AssertCommand(["gnt-node", "power", "status", node_name],
 375                   fail=True)
 376     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 377
 378     AssertCommand(["gnt-node", "health", node_name],
 379                   fail=True)
 380     _AssertOobCall(verify_path, "health %s" % full_node_name)
 381
 382     AssertCommand(["gnt-node", "health"], fail=True)
 383
 384     # Different OOB script for node
 385     verify_path2 = qa_utils.UploadData(master["primary"], "")
 386     oob_script = ("#!/bin/sh\n"
 387                   "echo \"$@\" > %s\n") % verify_path2
 388     oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 389
 390     try:
 391       AssertCommand(["gnt-node", "modify", "--node-parameters",
 392                      "oob_program=%s" % oob_path2, node_name])
 393       AssertCommand(["gnt-node", "power", "on", node_name])
 394       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 395     finally:
 396       AssertCommand(["gnt-node", "modify", "--node-parameters",
 397                      "oob_program=default", node_name])
 398       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 399   finally:
 400     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 401                    "oob_program="])
 402     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 403                    exit_code_path])
 404
 405
 406 def TestNodeList():
 407   """gnt-node list"""
 408   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 409
 410
 411 def TestNodeListFields():
 412   """gnt-node list-fields"""
 413   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())