code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.get("_added", False):
  40     raise qa_error.Error("Node %s already in cluster" % node["primary"])
  41   elif readd and not node.get("_added", False):
  42     raise qa_error.Error("Node %s not yet in cluster" % node["primary"])
  43
  44   cmd = ["gnt-node", "add", "--no-ssh-key-check"]
  45   if node.get("secondary", None):
  46     cmd.append("--secondary-ip=%s" % node["secondary"])
  47   if readd:
  48     cmd.append("--readd")
  49   cmd.append(node["primary"])
  50
  51   AssertCommand(cmd)
  52
  53   node["_added"] = True
  54
  55
  56 def _NodeRemove(node):
  57   AssertCommand(["gnt-node", "remove", node["primary"]])
  58   node["_added"] = False
  59
  60
  61 def TestNodeAddAll():
  62   """Adding all nodes to cluster."""
  63   master = qa_config.GetMasterNode()
  64   for node in qa_config.get("nodes"):
  65     if node != master:
  66       _NodeAdd(node, readd=False)
  67
  68
  69 def MarkNodeAddedAll():
  70   """Mark all nodes as added.
  71
  72   This is useful if we don't create the cluster ourselves (in qa).
  73
  74   """
  75   master = qa_config.GetMasterNode()
  76   for node in qa_config.get("nodes"):
  77     if node != master:
  78       node["_added"] = True
  79
  80
  81 def TestNodeRemoveAll():
  82   """Removing all nodes from cluster."""
  83   master = qa_config.GetMasterNode()
  84   for node in qa_config.get("nodes"):
  85     if node != master:
  86       _NodeRemove(node)
  87
  88
  89 def TestNodeReadd(node):
  90   """gnt-node add --readd"""
  91   _NodeAdd(node, readd=True)
  92
  93
  94 def TestNodeInfo():
  95   """gnt-node info"""
  96   AssertCommand(["gnt-node", "info"])
  97
  98
  99 def TestNodeVolumes():
 100   """gnt-node volumes"""
 101   AssertCommand(["gnt-node", "volumes"])
 102
 103
 104 def TestNodeStorage():
 105   """gnt-node storage"""
 106   master = qa_config.GetMasterNode()
 107
 108   for storage_type in constants.VALID_STORAGE_TYPES:
 109     # Test simple list
 110     AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
 111
 112     # Test all storage fields
 113     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 114            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 115                                     [constants.SF_NODE, constants.SF_TYPE])]
 116     AssertCommand(cmd)
 117
 118     # Get list of valid storage devices
 119     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 120            "--output=node,name,allocatable", "--separator=|",
 121            "--no-headers"]
 122     output = qa_utils.GetCommandOutput(master["primary"],
 123                                        utils.ShellQuoteArgs(cmd))
 124
 125     # Test with up to two devices
 126     testdevcount = 2
 127
 128     for line in output.splitlines()[:testdevcount]:
 129       (node_name, st_name, st_allocatable) = line.split("|")
 130
 131       # Dummy modification without any changes
 132       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 133       AssertCommand(cmd)
 134
 135       # Make sure we end up with the same value as before
 136       if st_allocatable.lower() == "y":
 137         test_allocatable = ["no", "yes"]
 138       else:
 139         test_allocatable = ["yes", "no"]
 140
 141       fail = (constants.SF_ALLOCATABLE not in
 142               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 143
 144       for i in test_allocatable:
 145         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 146                        node_name, storage_type, st_name], fail=fail)
 147
 148         # Verify list output
 149         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 150                "--output=name,allocatable", "--separator=|",
 151                "--no-headers", node_name]
 152         listout = qa_utils.GetCommandOutput(master["primary"],
 153                                             utils.ShellQuoteArgs(cmd))
 154         for line in listout.splitlines():
 155           (vfy_name, vfy_allocatable) = line.split("|")
 156           if vfy_name == st_name and not fail:
 157             AssertEqual(vfy_allocatable, i[0].upper())
 158           else:
 159             AssertEqual(vfy_allocatable, st_allocatable)
 160
 161       # Test repair functionality
 162       fail = (constants.SO_FIX_CONSISTENCY not in
 163               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 164       AssertCommand(["gnt-node", "repair-storage", node_name,
 165                      storage_type, st_name], fail=fail)
 166
 167
 168 def TestNodeFailover(node, node2):
 169   """gnt-node failover"""
 170   if qa_utils.GetNodeInstances(node2, secondaries=False):
 171     raise qa_error.UnusableNodeError("Secondary node has at least one"
 172                                      " primary instance. This test requires"
 173                                      " it to have no primary instances.")
 174
 175   # Fail over to secondary node
 176   AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
 177
 178   # ... and back again.
 179   AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
 180
 181
 182 def TestNodeEvacuate(node, node2):
 183   """gnt-node evacuate"""
 184   node3 = qa_config.AcquireNode(exclude=[node, node2])
 185   try:
 186     if qa_utils.GetNodeInstances(node3, secondaries=True):
 187       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 188                                        " secondary instance. This test requires"
 189                                        " it to have no secondary instances.")
 190
 191     # Evacuate all secondary instances
 192     AssertCommand(["gnt-node", "evacuate", "-f",
 193                    "--new-secondary=%s" % node3["primary"], node2["primary"]])
 194
 195     # ... and back again.
 196     AssertCommand(["gnt-node", "evacuate", "-f",
 197                    "--new-secondary=%s" % node2["primary"], node3["primary"]])
 198   finally:
 199     qa_config.ReleaseNode(node3)
 200
 201
 202 def TestNodeModify(node):
 203   """gnt-node modify"""
 204   for flag in ["master-candidate", "drained", "offline"]:
 205     for value in ["yes", "no"]:
 206       AssertCommand(["gnt-node", "modify", "--force",
 207                      "--%s=%s" % (flag, value), node["primary"]])
 208
 209   AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
 210                  "--auto-promote", node["primary"]])
 211
 212
 213 def _CreateOobScriptStructure():
 214   """Create a simple OOB handling script and its structure."""
 215   master = qa_config.GetMasterNode()
 216
 217   data_path = qa_utils.UploadData(master["primary"], "")
 218   verify_path = qa_utils.UploadData(master["primary"], "")
 219   exit_code_path = qa_utils.UploadData(master["primary"], "")
 220
 221   oob_script = (("#!/bin/bash\n"
 222                  "echo \"$@\" > %s\n"
 223                  "cat %s\n"
 224                  "exit $(< %s)\n") %
 225                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 226                  utils.ShellQuote(exit_code_path)))
 227   oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 228
 229   return [oob_path, verify_path, data_path, exit_code_path]
 230
 231
 232 def _UpdateOobFile(path, data):
 233   """Updates the data file with data."""
 234   master = qa_config.GetMasterNode()
 235   qa_utils.UploadData(master["primary"], data, filename=path)
 236
 237
 238 def _AssertOobCall(verify_path, expected_args):
 239   """Assert the OOB call was performed with expetected args."""
 240   master = qa_config.GetMasterNode()
 241
 242   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 243   output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd,
 244                                      tty=False)
 245
 246   AssertEqual(expected_args, output.strip())
 247
 248
 249 def TestOutOfBand():
 250   """gnt-node power"""
 251   master = qa_config.GetMasterNode()
 252
 253   node = qa_config.AcquireNode(exclude=master)
 254
 255   master_name = master["primary"]
 256   node_name = node["primary"]
 257   full_node_name = qa_utils.ResolveNodeName(node)
 258
 259   (oob_path, verify_path,
 260    data_path, exit_code_path) = _CreateOobScriptStructure()
 261
 262   try:
 263     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 264                    "oob_program=%s" % oob_path])
 265
 266     # No data, exit 0
 267     _UpdateOobFile(exit_code_path, "0")
 268
 269     AssertCommand(["gnt-node", "power", "on", node_name])
 270     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 271
 272     AssertCommand(["gnt-node", "power", "-f", "off", node_name])
 273     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 274
 275     # Power off on master without options should fail
 276     AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
 277     # With force master it should still fail
 278     AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
 279                    master_name],
 280                   fail=True)
 281
 282     # Verify we can't transform back to online when not yet powered on
 283     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 284                   fail=True)
 285     # Now reset state
 286     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 287                    node_name])
 288
 289     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
 290     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 291
 292     # Those commands should fail as they expect output which isn't provided yet
 293     # But they should have called the oob helper nevermind
 294     AssertCommand(["gnt-node", "power", "status", node_name],
 295                   fail=True)
 296     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 297
 298     AssertCommand(["gnt-node", "health", node_name],
 299                   fail=True)
 300     _AssertOobCall(verify_path, "health %s" % full_node_name)
 301
 302     AssertCommand(["gnt-node", "health"], fail=True)
 303
 304     # Correct Data, exit 0
 305     _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
 306
 307     AssertCommand(["gnt-node", "power", "status", node_name])
 308     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 309
 310     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 311                                                    ["disk0", "CRITICAL"]]))
 312
 313     AssertCommand(["gnt-node", "health", node_name])
 314     _AssertOobCall(verify_path, "health %s" % full_node_name)
 315
 316     AssertCommand(["gnt-node", "health"])
 317
 318     # Those commands should fail as they expect no data regardless of exit 0
 319     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 320     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 321
 322     try:
 323       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 324       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 325     finally:
 326       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 327
 328     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 329     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 330
 331     # Data, exit 1 (all should fail)
 332     _UpdateOobFile(exit_code_path, "1")
 333
 334     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 335     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 336
 337     try:
 338       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 339       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 340     finally:
 341       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 342
 343     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 344     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 345
 346     AssertCommand(["gnt-node", "power", "status", node_name],
 347                   fail=True)
 348     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 349
 350     AssertCommand(["gnt-node", "health", node_name],
 351                   fail=True)
 352     _AssertOobCall(verify_path, "health %s" % full_node_name)
 353
 354     AssertCommand(["gnt-node", "health"], fail=True)
 355
 356     # No data, exit 1 (all should fail)
 357     _UpdateOobFile(data_path, "")
 358     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 359     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 360
 361     try:
 362       AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
 363       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 364     finally:
 365       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 366
 367     AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
 368     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 369
 370     AssertCommand(["gnt-node", "power", "status", node_name],
 371                   fail=True)
 372     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 373
 374     AssertCommand(["gnt-node", "health", node_name],
 375                   fail=True)
 376     _AssertOobCall(verify_path, "health %s" % full_node_name)
 377
 378     AssertCommand(["gnt-node", "health"], fail=True)
 379
 380     # Different OOB script for node
 381     verify_path2 = qa_utils.UploadData(master["primary"], "")
 382     oob_script = ("#!/bin/sh\n"
 383                   "echo \"$@\" > %s\n") % verify_path2
 384     oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 385
 386     try:
 387       AssertCommand(["gnt-node", "modify", "--node-parameters",
 388                      "oob_program=%s" % oob_path2, node_name])
 389       AssertCommand(["gnt-node", "power", "on", node_name])
 390       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 391     finally:
 392       AssertCommand(["gnt-node", "modify", "--node-parameters",
 393                      "oob_program=default", node_name])
 394       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 395   finally:
 396     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 397                    "oob_program="])
 398     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 399                    exit_code_path])
 400
 401
 402 def TestNodeList():
 403   """gnt-node list"""
 404   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 405
 406
 407 def TestNodeListFields():
 408   """gnt-node list-fields"""
 409   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())