code.grnet.gr Git - ganeti-local/blob - qa/qa_instance.py

   1 #
   2 #
   3
   4 # Copyright (C) 2007 Google Inc.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful, but
  12 # WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  19 # 02110-1301, USA.
  20
  21
  22 """Instance related QA tests.
  23
  24 """
  25
  26 import re
  27 import time
  28
  29 from ganeti import utils
  30 from ganeti import constants
  31
  32 import qa_config
  33 import qa_utils
  34 import qa_error
  35
  36 from qa_utils import AssertEqual, AssertNotEqual, StartSSH
  37
  38
  39 def _GetDiskStatePath(disk):
  40   return "/sys/block/%s/device/state" % disk
  41
  42
  43 def _GetGenericAddParameters():
  44   params = ['-B', '%s=%s' % (constants.BE_MEMORY, qa_config.get('mem'))]
  45   for idx, size in enumerate(qa_config.get('disk')):
  46     params.extend(["--disk", "%s:size=%s" % (idx, size)])
  47   return params
  48
  49
  50 def _DiskTest(node, disk_template):
  51   master = qa_config.GetMasterNode()
  52
  53   instance = qa_config.AcquireInstance()
  54   try:
  55     cmd = (['gnt-instance', 'add',
  56             '--os-type=%s' % qa_config.get('os'),
  57             '--disk-template=%s' % disk_template,
  58             '--node=%s' % node] +
  59            _GetGenericAddParameters())
  60     cmd.append(instance['name'])
  61
  62     AssertEqual(StartSSH(master['primary'],
  63                          utils.ShellQuoteArgs(cmd)).wait(), 0)
  64     return instance
  65   except:
  66     qa_config.ReleaseInstance(instance)
  67     raise
  68
  69
  70 def TestInstanceAddWithPlainDisk(node):
  71   """gnt-instance add -t plain"""
  72   return _DiskTest(node['primary'], 'plain')
  73
  74
  75 def TestInstanceAddWithDrbdDisk(node, node2):
  76   """gnt-instance add -t drbd"""
  77   return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
  78                    'drbd')
  79
  80
  81 def TestInstanceRemove(instance):
  82   """gnt-instance remove"""
  83   master = qa_config.GetMasterNode()
  84
  85   cmd = ['gnt-instance', 'remove', '-f', instance['name']]
  86   AssertEqual(StartSSH(master['primary'],
  87                        utils.ShellQuoteArgs(cmd)).wait(), 0)
  88
  89   qa_config.ReleaseInstance(instance)
  90
  91
  92 def TestInstanceStartup(instance):
  93   """gnt-instance startup"""
  94   master = qa_config.GetMasterNode()
  95
  96   cmd = ['gnt-instance', 'startup', instance['name']]
  97   AssertEqual(StartSSH(master['primary'],
  98                        utils.ShellQuoteArgs(cmd)).wait(), 0)
  99
 100
 101 def TestInstanceShutdown(instance):
 102   """gnt-instance shutdown"""
 103   master = qa_config.GetMasterNode()
 104
 105   cmd = ['gnt-instance', 'shutdown', instance['name']]
 106   AssertEqual(StartSSH(master['primary'],
 107                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 108
 109
 110 def TestInstanceReboot(instance):
 111   """gnt-instance reboot"""
 112   master = qa_config.GetMasterNode()
 113
 114   options = qa_config.get('options', {})
 115   reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
 116
 117   for rtype in reboot_types:
 118     cmd = ['gnt-instance', 'reboot', '--type=%s' % rtype, instance['name']]
 119     AssertEqual(StartSSH(master['primary'],
 120                          utils.ShellQuoteArgs(cmd)).wait(), 0)
 121
 122
 123 def TestInstanceReinstall(instance):
 124   """gnt-instance reinstall"""
 125   master = qa_config.GetMasterNode()
 126
 127   cmd = ['gnt-instance', 'reinstall', '-f', instance['name']]
 128   AssertEqual(StartSSH(master['primary'],
 129                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 130
 131
 132 def TestInstanceRename(instance, rename_target):
 133   """gnt-instance rename"""
 134   master = qa_config.GetMasterNode()
 135
 136   rename_source = instance['name']
 137
 138   for name1, name2 in [(rename_source, rename_target),
 139                        (rename_target, rename_source)]:
 140     cmd = ['gnt-instance', 'rename', name1, name2]
 141     AssertEqual(StartSSH(master['primary'],
 142                          utils.ShellQuoteArgs(cmd)).wait(), 0)
 143
 144
 145 def TestInstanceFailover(instance):
 146   """gnt-instance failover"""
 147   master = qa_config.GetMasterNode()
 148
 149   cmd = ['gnt-instance', 'failover', '--force', instance['name']]
 150   AssertEqual(StartSSH(master['primary'],
 151                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 152
 153   # ... and back
 154   cmd = ['gnt-instance', 'failover', '--force', instance['name']]
 155   AssertEqual(StartSSH(master['primary'],
 156                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 157
 158
 159 def TestInstanceMigrate(instance):
 160   """gnt-instance migrate"""
 161   master = qa_config.GetMasterNode()
 162
 163   cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
 164   AssertEqual(StartSSH(master["primary"],
 165                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 166
 167   # ... and back
 168   cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
 169   AssertEqual(StartSSH(master["primary"],
 170                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 171
 172
 173 def TestInstanceInfo(instance):
 174   """gnt-instance info"""
 175   master = qa_config.GetMasterNode()
 176
 177   cmd = ['gnt-instance', 'info', instance['name']]
 178   AssertEqual(StartSSH(master['primary'],
 179                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 180
 181
 182 def TestInstanceModify(instance):
 183   """gnt-instance modify"""
 184   master = qa_config.GetMasterNode()
 185
 186   # Assume /sbin/init exists on all systems
 187   test_kernel = "/sbin/init"
 188   test_initrd = test_kernel
 189
 190   orig_memory = qa_config.get('mem')
 191   orig_bridge = qa_config.get('bridge', 'xen-br0')
 192   args = [
 193     ["-B", "%s=128" % constants.BE_MEMORY],
 194     ["-B", "%s=%s" % (constants.BE_MEMORY, orig_memory)],
 195     ["-B", "%s=2" % constants.BE_VCPUS],
 196     ["-B", "%s=1" % constants.BE_VCPUS],
 197     ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
 198
 199     ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
 200     ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
 201     ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
 202     ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
 203     ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
 204
 205     # TODO: bridge tests
 206     #["--bridge", "xen-br1"],
 207     #["--bridge", orig_bridge],
 208
 209     # TODO: Do these tests only with xen-hvm
 210     #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
 211     #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
 212     ]
 213   for alist in args:
 214     cmd = ['gnt-instance', 'modify'] + alist + [instance['name']]
 215     AssertEqual(StartSSH(master['primary'],
 216                          utils.ShellQuoteArgs(cmd)).wait(), 0)
 217
 218   # check no-modify
 219   cmd = ['gnt-instance', 'modify', instance['name']]
 220   AssertNotEqual(StartSSH(master['primary'],
 221                           utils.ShellQuoteArgs(cmd)).wait(), 0)
 222
 223
 224 def TestInstanceConvertDisk(instance, snode):
 225   """gnt-instance modify -t"""
 226   master = qa_config.GetMasterNode()
 227   cmd = ['gnt-instance', 'modify', '-t', 'plain', instance['name']]
 228   AssertEqual(StartSSH(master['primary'],
 229                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 230   cmd = ['gnt-instance', 'modify', '-t', 'drbd', '-n', snode['primary'],
 231          instance['name']]
 232   AssertEqual(StartSSH(master['primary'],
 233                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 234
 235
 236 def TestInstanceList():
 237   """gnt-instance list"""
 238   master = qa_config.GetMasterNode()
 239
 240   cmd = ['gnt-instance', 'list']
 241   AssertEqual(StartSSH(master['primary'],
 242                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 243
 244
 245 def TestInstanceConsole(instance):
 246   """gnt-instance console"""
 247   master = qa_config.GetMasterNode()
 248
 249   cmd = ['gnt-instance', 'console', '--show-cmd', instance['name']]
 250   AssertEqual(StartSSH(master['primary'],
 251                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 252
 253
 254 def TestReplaceDisks(instance, pnode, snode, othernode):
 255   """gnt-instance replace-disks"""
 256   master = qa_config.GetMasterNode()
 257
 258   def buildcmd(args):
 259     cmd = ['gnt-instance', 'replace-disks']
 260     cmd.extend(args)
 261     cmd.append(instance["name"])
 262     return cmd
 263
 264   cmd = buildcmd(["-p"])
 265   AssertEqual(StartSSH(master['primary'],
 266                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 267
 268   cmd = buildcmd(["-s"])
 269   AssertEqual(StartSSH(master['primary'],
 270                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 271
 272   cmd = buildcmd(["--new-secondary=%s" % othernode["primary"]])
 273   AssertEqual(StartSSH(master['primary'],
 274                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 275
 276   # Restore
 277   cmd = buildcmd(["--new-secondary=%s" % snode["primary"]])
 278   AssertEqual(StartSSH(master['primary'],
 279                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 280
 281
 282 def TestInstanceExport(instance, node):
 283   """gnt-backup export"""
 284   master = qa_config.GetMasterNode()
 285
 286   cmd = ['gnt-backup', 'export', '-n', node['primary'], instance['name']]
 287   AssertEqual(StartSSH(master['primary'],
 288                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 289
 290   return qa_utils.ResolveInstanceName(instance)
 291
 292
 293 def TestInstanceExportWithRemove(instance, node):
 294   """gnt-backup export --remove-instance"""
 295   master = qa_config.GetMasterNode()
 296
 297   cmd = ['gnt-backup', 'export', '-n', node['primary'], "--remove-instance",
 298          instance['name']]
 299   AssertEqual(StartSSH(master['primary'],
 300                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 301
 302
 303 def TestInstanceImport(node, newinst, expnode, name):
 304   """gnt-backup import"""
 305   master = qa_config.GetMasterNode()
 306
 307   cmd = (['gnt-backup', 'import',
 308           '--disk-template=plain',
 309           '--no-ip-check',
 310           '--net', '0:mac=generate',
 311           '--src-node=%s' % expnode['primary'],
 312           '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
 313           '--node=%s' % node['primary']] +
 314          _GetGenericAddParameters())
 315   cmd.append(newinst['name'])
 316   AssertEqual(StartSSH(master['primary'],
 317                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 318
 319
 320 def TestBackupList(expnode):
 321   """gnt-backup list"""
 322   master = qa_config.GetMasterNode()
 323
 324   cmd = ['gnt-backup', 'list', '--node=%s' % expnode['primary']]
 325   AssertEqual(StartSSH(master['primary'],
 326                        utils.ShellQuoteArgs(cmd)).wait(), 0)
 327
 328
 329 def _TestInstanceDiskFailure(instance, node, node2, onmaster):
 330   """Testing disk failure."""
 331   master = qa_config.GetMasterNode()
 332   sq = utils.ShellQuoteArgs
 333
 334   instance_full = qa_utils.ResolveInstanceName(instance)
 335   node_full = qa_utils.ResolveNodeName(node)
 336   node2_full = qa_utils.ResolveNodeName(node2)
 337
 338   print qa_utils.FormatInfo("Getting physical disk names")
 339   cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
 340          '--output=node,phys,instance',
 341          node['primary'], node2['primary']]
 342   output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
 343
 344   # Get physical disk names
 345   re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
 346   node2disk = {}
 347   for line in output.splitlines():
 348     (node_name, phys, inst) = line.split('|')
 349     if inst == instance_full:
 350       if node_name not in node2disk:
 351         node2disk[node_name] = []
 352
 353       m = re_disk.match(phys)
 354       if not m:
 355         raise qa_error.Error("Unknown disk name format: %s" % disk)
 356
 357       name = m.group(1)
 358       if name not in node2disk[node_name]:
 359         node2disk[node_name].append(name)
 360
 361   if [node2_full, node_full][int(onmaster)] not in node2disk:
 362     raise qa_error.Error("Couldn't find physical disks used on"
 363                          " %s node" % ["secondary", "master"][int(onmaster)])
 364
 365   print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
 366                             " disks")
 367   for node_name, disks in node2disk.iteritems():
 368     cmds = []
 369     for disk in disks:
 370       cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
 371     AssertEqual(StartSSH(node_name, ' && '.join(cmds)).wait(), 0)
 372
 373   print qa_utils.FormatInfo("Getting device paths")
 374   cmd = ['gnt-instance', 'activate-disks', instance['name']]
 375   output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
 376   devpath = []
 377   for line in output.splitlines():
 378     (_, _, tmpdevpath) = line.split(':')
 379     devpath.append(tmpdevpath)
 380   print devpath
 381
 382   print qa_utils.FormatInfo("Getting drbd device paths")
 383   cmd = ['gnt-instance', 'info', instance['name']]
 384   output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
 385   pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
 386              r'\s+primary:\s+(/dev/drbd\d+)\s+')
 387   drbddevs = re.findall(pattern, output, re.M)
 388   print drbddevs
 389
 390   halted_disks = []
 391   try:
 392     print qa_utils.FormatInfo("Deactivating disks")
 393     cmds = []
 394     for name in node2disk[[node2_full, node_full][int(onmaster)]]:
 395       halted_disks.append(name)
 396       cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
 397     AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
 398                          ' && '.join(cmds)).wait(), 0)
 399
 400     print qa_utils.FormatInfo("Write to disks and give some time to notice"
 401                               " to notice the problem")
 402     cmds = []
 403     for disk in devpath:
 404       cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
 405                       "if=%s" % disk, "of=%s" % disk]))
 406     for _ in (0, 1, 2):
 407       AssertEqual(StartSSH(node['primary'], ' && '.join(cmds)).wait(), 0)
 408       time.sleep(3)
 409
 410     print qa_utils.FormatInfo("Debugging info")
 411     for name in drbddevs:
 412       cmd = ['drbdsetup', name, 'show']
 413       AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
 414
 415     cmd = ['gnt-instance', 'info', instance['name']]
 416     AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
 417
 418   finally:
 419     print qa_utils.FormatInfo("Activating disks again")
 420     cmds = []
 421     for name in halted_disks:
 422       cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
 423     AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
 424                          '; '.join(cmds)).wait(), 0)
 425
 426   if onmaster:
 427     for name in drbddevs:
 428       cmd = ['drbdsetup', name, 'detach']
 429       AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
 430   else:
 431     for name in drbddevs:
 432       cmd = ['drbdsetup', name, 'disconnect']
 433       AssertEqual(StartSSH(node2['primary'], sq(cmd)).wait(), 0)
 434
 435   # TODO
 436   #cmd = ['vgs']
 437   #AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
 438   #                     sq(cmd)).wait(), 0)
 439
 440   print qa_utils.FormatInfo("Making sure disks are up again")
 441   cmd = ['gnt-instance', 'replace-disks', instance['name']]
 442   AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
 443
 444   print qa_utils.FormatInfo("Restarting instance")
 445   cmd = ['gnt-instance', 'shutdown', instance['name']]
 446   AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
 447
 448   cmd = ['gnt-instance', 'startup', instance['name']]
 449   AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
 450
 451   cmd = ['gnt-cluster', 'verify']
 452   AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
 453
 454
 455 def TestInstanceMasterDiskFailure(instance, node, node2):
 456   """Testing disk failure on master node."""
 457   print qa_utils.FormatError("Disk failure on primary node cannot be"
 458                              " tested due to potential crashes.")
 459   # The following can cause crashes, thus it's disabled until fixed
 460   #return _TestInstanceDiskFailure(instance, node, node2, True)
 461
 462
 463 def TestInstanceSecondaryDiskFailure(instance, node, node2):
 464   """Testing disk failure on secondary node."""
 465   return _TestInstanceDiskFailure(instance, node, node2, False)