code.grnet.gr Git - ganeti-local/blob - qa/qa_node.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007, 2011 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Node-related QA tests.
  23
  24 """
  25
  26 from ganeti import utils
  27 from ganeti import constants
  28 from ganeti import query
  29 from ganeti import serializer
  30
  31 import qa_config
  32 import qa_error
  33 import qa_utils
  34
  35 from qa_utils import AssertCommand, AssertEqual
  36
  37
  38 def _NodeAdd(node, readd=False):
  39   if not readd and node.get('_added', False):
  40     raise qa_error.Error("Node %s already in cluster" % node['primary'])
  41   elif readd and not node.get('_added', False):
  42     raise qa_error.Error("Node %s not yet in cluster" % node['primary'])
  43
  44   cmd = ['gnt-node', 'add', "--no-ssh-key-check"]
  45   if node.get('secondary', None):
  46     cmd.append('--secondary-ip=%s' % node['secondary'])
  47   if readd:
  48     cmd.append('--readd')
  49   cmd.append(node['primary'])
  50
  51   AssertCommand(cmd)
  52
  53   node['_added'] = True
  54
  55
  56 def _NodeRemove(node):
  57   AssertCommand(["gnt-node", "remove", node["primary"]])
  58   node['_added'] = False
  59
  60
  61 def TestNodeAddAll():
  62   """Adding all nodes to cluster."""
  63   master = qa_config.GetMasterNode()
  64   for node in qa_config.get('nodes'):
  65     if node != master:
  66       _NodeAdd(node, readd=False)
  67
  68
  69 def MarkNodeAddedAll():
  70   """Mark all nodes as added.
  71
  72   This is useful if we don't create the cluster ourselves (in qa).
  73
  74   """
  75   master = qa_config.GetMasterNode()
  76   for node in qa_config.get('nodes'):
  77     if node != master:
  78       node['_added'] = True
  79
  80
  81 def TestNodeRemoveAll():
  82   """Removing all nodes from cluster."""
  83   master = qa_config.GetMasterNode()
  84   for node in qa_config.get('nodes'):
  85     if node != master:
  86       _NodeRemove(node)
  87
  88
  89 def TestNodeReadd(node):
  90   """gnt-node add --readd"""
  91   _NodeAdd(node, readd=True)
  92
  93
  94 def TestNodeInfo():
  95   """gnt-node info"""
  96   AssertCommand(["gnt-node", "info"])
  97
  98
  99 def TestNodeVolumes():
 100   """gnt-node volumes"""
 101   AssertCommand(["gnt-node", "volumes"])
 102
 103
 104 def TestNodeStorage():
 105   """gnt-node storage"""
 106   master = qa_config.GetMasterNode()
 107
 108   for storage_type in constants.VALID_STORAGE_TYPES:
 109     # Test simple list
 110     AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
 111
 112     # Test all storage fields
 113     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 114            "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
 115                                     [constants.SF_NODE, constants.SF_TYPE])]
 116     AssertCommand(cmd)
 117
 118     # Get list of valid storage devices
 119     cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 120            "--output=node,name,allocatable", "--separator=|",
 121            "--no-headers"]
 122     output = qa_utils.GetCommandOutput(master["primary"],
 123                                        utils.ShellQuoteArgs(cmd))
 124
 125     # Test with up to two devices
 126     testdevcount = 2
 127
 128     for line in output.splitlines()[:testdevcount]:
 129       (node_name, st_name, st_allocatable) = line.split("|")
 130
 131       # Dummy modification without any changes
 132       cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
 133       AssertCommand(cmd)
 134
 135       # Make sure we end up with the same value as before
 136       if st_allocatable.lower() == "y":
 137         test_allocatable = ["no", "yes"]
 138       else:
 139         test_allocatable = ["yes", "no"]
 140
 141       fail = (constants.SF_ALLOCATABLE not in
 142               constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
 143
 144       for i in test_allocatable:
 145         AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
 146                        node_name, storage_type, st_name], fail=fail)
 147
 148         # Verify list output
 149         cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
 150                "--output=name,allocatable", "--separator=|",
 151                "--no-headers", node_name]
 152         listout = qa_utils.GetCommandOutput(master["primary"],
 153                                             utils.ShellQuoteArgs(cmd))
 154         for line in listout.splitlines():
 155           (vfy_name, vfy_allocatable) = line.split("|")
 156           if vfy_name == st_name and not fail:
 157             AssertEqual(vfy_allocatable, i[0].upper())
 158           else:
 159             AssertEqual(vfy_allocatable, st_allocatable)
 160
 161       # Test repair functionality
 162       fail = (constants.SO_FIX_CONSISTENCY not in
 163               constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
 164       AssertCommand(["gnt-node", "repair-storage", node_name,
 165                      storage_type, st_name], fail=fail)
 166
 167
 168 def TestNodeFailover(node, node2):
 169   """gnt-node failover"""
 170   if qa_utils.GetNodeInstances(node2, secondaries=False):
 171     raise qa_error.UnusableNodeError("Secondary node has at least one"
 172                                      " primary instance. This test requires"
 173                                      " it to have no primary instances.")
 174
 175   # Fail over to secondary node
 176   AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
 177
 178   # ... and back again.
 179   AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
 180
 181
 182 def TestNodeEvacuate(node, node2):
 183   """gnt-node evacuate"""
 184   node3 = qa_config.AcquireNode(exclude=[node, node2])
 185   try:
 186     if qa_utils.GetNodeInstances(node3, secondaries=True):
 187       raise qa_error.UnusableNodeError("Evacuation node has at least one"
 188                                        " secondary instance. This test requires"
 189                                        " it to have no secondary instances.")
 190
 191     # Evacuate all secondary instances
 192     AssertCommand(["gnt-node", "evacuate", "-f",
 193                    "--new-secondary=%s" % node3["primary"], node2["primary"]])
 194
 195     # ... and back again.
 196     AssertCommand(["gnt-node", "evacuate", "-f",
 197                    "--new-secondary=%s" % node2["primary"], node3["primary"]])
 198   finally:
 199     qa_config.ReleaseNode(node3)
 200
 201
 202 def TestNodeModify(node):
 203   """gnt-node modify"""
 204   for flag in ["master-candidate", "drained", "offline"]:
 205     for value in ["yes", "no"]:
 206       AssertCommand(["gnt-node", "modify", "--force",
 207                      "--%s=%s" % (flag, value), node["primary"]])
 208
 209   AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
 210                  "--auto-promote", node["primary"]])
 211
 212
 213 def _CreateOobScriptStructure():
 214   """Create a simple OOB handling script and its structure."""
 215   master = qa_config.GetMasterNode()
 216
 217   data_path = qa_utils.UploadData(master["primary"], "")
 218   verify_path = qa_utils.UploadData(master["primary"], "")
 219   exit_code_path = qa_utils.UploadData(master["primary"], "")
 220
 221   oob_script = (("#!/bin/bash\n"
 222                  "echo \"$@\" > %s\n"
 223                  "cat %s\n"
 224                  "exit $(< %s)\n") %
 225                 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
 226                  utils.ShellQuote(exit_code_path)))
 227   oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 228
 229   return [oob_path, verify_path, data_path, exit_code_path]
 230
 231
 232 def _UpdateOobFile(path, data):
 233   """Updates the data file with data."""
 234   master = qa_config.GetMasterNode()
 235   qa_utils.UploadData(master["primary"], data, filename=path)
 236
 237
 238 def _AssertOobCall(verify_path, expected_args):
 239   """Assert the OOB call was performed with expetected args."""
 240   master = qa_config.GetMasterNode()
 241
 242   verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
 243   output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd)
 244
 245   AssertEqual(expected_args, output.strip())
 246
 247
 248 def TestOutOfBand():
 249   """gnt-node power"""
 250   master = qa_config.GetMasterNode()
 251
 252   node = qa_config.AcquireNode(exclude=master)
 253
 254   master_name = master["primary"]
 255   full_master_name = qa_utils.ResolveNodeName(master)
 256   node_name = node["primary"]
 257   full_node_name = qa_utils.ResolveNodeName(node)
 258
 259   (oob_path, verify_path,
 260    data_path, exit_code_path) = _CreateOobScriptStructure()
 261
 262   try:
 263     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 264                    "oob_program=%s" % oob_path])
 265
 266     # No data, exit 0
 267     _UpdateOobFile(exit_code_path, "0")
 268
 269     AssertCommand(["gnt-node", "power", "on", node_name])
 270     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 271
 272     AssertCommand(["gnt-node", "power", "off", node_name])
 273     _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 274
 275     # Power off on master without options should fail
 276     AssertCommand(["gnt-node", "power", "off", master_name], fail=True)
 277     # With force master it should still fail
 278     AssertCommand(["gnt-node", "power", "--force-master", "off", master_name],
 279                   fail=True)
 280     AssertCommand(["gnt-node", "power", "--ignore-status", "off", master_name],
 281                   fail=True)
 282     # This should work again
 283     AssertCommand(["gnt-node", "power", "--ignore-status", "--force-master",
 284                    "off", master_name])
 285     _AssertOobCall(verify_path, "power-off %s" % full_master_name)
 286
 287     # Verify we can't transform back to online when not yet powered on
 288     AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
 289                   fail=True)
 290     # Now reset state
 291     AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
 292                    node_name])
 293
 294     AssertCommand(["gnt-node", "power", "cycle", node_name])
 295     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 296
 297     # Those commands should fail as they expect output which isn't provided yet
 298     # But they should have called the oob helper nevermind
 299     AssertCommand(["gnt-node", "power", "status", node_name],
 300                   fail=True)
 301     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 302
 303     AssertCommand(["gnt-node", "health", node_name],
 304                   fail=True)
 305     _AssertOobCall(verify_path, "health %s" % full_node_name)
 306
 307     AssertCommand(["gnt-node", "health"], fail=True)
 308
 309     # Correct Data, exit 0
 310     _UpdateOobFile(data_path, serializer.DumpJson({ "powered": True }))
 311
 312     AssertCommand(["gnt-node", "power", "status", node_name])
 313     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 314
 315     _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
 316                                                    ["disk0", "CRITICAL"]]))
 317
 318     AssertCommand(["gnt-node", "health", node_name])
 319     _AssertOobCall(verify_path, "health %s" % full_node_name)
 320
 321     AssertCommand(["gnt-node", "health"])
 322
 323
 324     # Those commands should fail as they expect no data regardless of exit 0
 325     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 326     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 327
 328     try:
 329       AssertCommand(["gnt-node", "power", "off", node_name], fail=True)
 330       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 331     finally:
 332       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 333
 334     AssertCommand(["gnt-node", "power", "cycle", node_name], fail=True)
 335     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 336
 337     # Data, exit 1 (all should fail)
 338     _UpdateOobFile(exit_code_path, "1")
 339
 340     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 341     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 342
 343     try:
 344       AssertCommand(["gnt-node", "power", "off", node_name], fail=True)
 345       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 346     finally:
 347       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 348
 349     AssertCommand(["gnt-node", "power", "cycle", node_name], fail=True)
 350     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 351
 352     AssertCommand(["gnt-node", "power", "status", node_name],
 353                   fail=True)
 354     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 355
 356     AssertCommand(["gnt-node", "health", node_name],
 357                   fail=True)
 358     _AssertOobCall(verify_path, "health %s" % full_node_name)
 359
 360     AssertCommand(["gnt-node", "health"], fail=True)
 361
 362     # No data, exit 1 (all should fail)
 363     _UpdateOobFile(data_path, "")
 364     AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
 365     _AssertOobCall(verify_path, "power-on %s" % full_node_name)
 366
 367     try:
 368       AssertCommand(["gnt-node", "power", "off", node_name], fail=True)
 369       _AssertOobCall(verify_path, "power-off %s" % full_node_name)
 370     finally:
 371       AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
 372
 373     AssertCommand(["gnt-node", "power", "cycle", node_name], fail=True)
 374     _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
 375
 376     AssertCommand(["gnt-node", "power", "status", node_name],
 377                   fail=True)
 378     _AssertOobCall(verify_path, "power-status %s" % full_node_name)
 379
 380     AssertCommand(["gnt-node", "health", node_name],
 381                   fail=True)
 382     _AssertOobCall(verify_path, "health %s" % full_node_name)
 383
 384     AssertCommand(["gnt-node", "health"], fail=True)
 385
 386     # Different OOB script for node
 387     verify_path2 = qa_utils.UploadData(master["primary"], "")
 388     oob_script = ("#!/bin/sh\n"
 389                   "echo \"$@\" > %s\n") % verify_path2
 390     oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
 391
 392     try:
 393       AssertCommand(["gnt-node", "modify", "--node-parameters",
 394                      "oob_program=%s" % oob_path2, node_name])
 395       AssertCommand(["gnt-node", "power", "on", node_name])
 396       _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
 397     finally:
 398       AssertCommand(["gnt-node", "modify", "--node-parameters",
 399                      "oob_program=default", node_name])
 400       AssertCommand(["rm", "-f", oob_path2, verify_path2])
 401   finally:
 402     AssertCommand(["gnt-cluster", "modify", "--node-parameters",
 403                    "oob_program="])
 404     AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
 405                    exit_code_path])
 406
 407
 408 def TestNodeList():
 409   """gnt-node list"""
 410   qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
 411
 412
 413 def TestNodeListFields():
 414   """gnt-node list-fields"""
 415   qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())