4 # Copyright (C) 2007, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Instance related QA tests.
29 from ganeti import utils
30 from ganeti import constants
31 from ganeti import query
32 from ganeti import pathutils
38 from qa_utils import AssertIn, AssertCommand, AssertEqual
39 from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
42 def _GetDiskStatePath(disk):
43 return "/sys/block/%s/device/state" % disk
46 def _GetGenericAddParameters(inst, force_mac=None):
48 params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
49 qa_config.get(constants.BE_MINMEM),
51 qa_config.get(constants.BE_MAXMEM)))
52 for idx, size in enumerate(qa_config.get("disk")):
53 params.extend(["--disk", "%s:size=%s" % (idx, size)])
55 # Set static MAC address if configured
59 nic0_mac = qa_config.GetInstanceNicMac(inst)
61 params.extend(["--net", "0:mac=%s" % nic0_mac])
66 def _DiskTest(node, disk_template):
67 instance = qa_config.AcquireInstance()
69 cmd = (["gnt-instance", "add",
70 "--os-type=%s" % qa_config.get("os"),
71 "--disk-template=%s" % disk_template,
73 _GetGenericAddParameters(instance))
74 cmd.append(instance["name"])
78 _CheckSsconfInstanceList(instance["name"])
82 qa_config.ReleaseInstance(instance)
86 def _GetInstanceInfo(instance):
87 """Return information about the actual state of an instance.
89 @type instance: string
90 @param instance: the instance name
91 @return: a dictionary with two keys:
92 - "nodes": instance nodes, a list of strings
93 - "volumes": instance volume IDs, a list of strings
96 master = qa_config.GetMasterNode()
97 infocmd = utils.ShellQuoteArgs(["gnt-instance", "info", instance])
98 info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
99 re_node = re.compile(r"^\s+-\s+(?:primary|secondaries):\s+(\S.+)$")
100 node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
101 # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
103 # node2.fqdn,node3.fqdn
104 # node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
105 # FIXME This works with no more than 2 secondaries
106 re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
107 re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
110 for line in info_out.splitlines():
111 m = re_node.match(line)
114 m2 = re_nodelist.match(nodestr)
116 nodes.extend(filter(None, m2.groups()))
118 nodes.append(nodestr)
119 m = re_vol.match(line)
121 vols.append(m.group(1))
124 return {"nodes": nodes, "volumes": vols}
127 def _DestroyInstanceVolumes(instance):
128 """Remove all the LVM volumes of an instance.
130 This is used to simulate HW errors (dead nodes, broken disks...); the
131 configuration of the instance is not affected.
132 @type instance: dictionary
133 @param instance: the instance
136 info = _GetInstanceInfo(instance["name"])
137 vols = info["volumes"]
138 for node in info["nodes"]:
139 AssertCommand(["lvremove", "-f"] + vols, node=node)
142 def _GetBoolInstanceField(instance, field):
143 """Get the Boolean value of a field of an instance.
145 @type instance: string
146 @param instance: Instance name
148 @param field: Name of the field
151 master = qa_config.GetMasterNode()
152 infocmd = utils.ShellQuoteArgs(["gnt-instance", "list", "--no-headers",
153 "-o", field, instance])
154 info_out = qa_utils.GetCommandOutput(master["primary"], infocmd).strip()
157 elif info_out == "N":
160 raise qa_error.Error("Field %s of instance %s has a non-Boolean value:"
161 " %s" % (field, instance, info_out))
164 @InstanceCheck(None, INST_UP, RETURN_VALUE)
165 def TestInstanceAddWithPlainDisk(node):
166 """gnt-instance add -t plain"""
167 return _DiskTest(node["primary"], "plain")
170 @InstanceCheck(None, INST_UP, RETURN_VALUE)
171 def TestInstanceAddWithDrbdDisk(node, node2):
172 """gnt-instance add -t drbd"""
173 return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
177 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
178 def TestInstanceRemove(instance):
179 """gnt-instance remove"""
180 AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
182 qa_config.ReleaseInstance(instance)
185 @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
186 def TestInstanceStartup(instance):
187 """gnt-instance startup"""
188 AssertCommand(["gnt-instance", "startup", instance["name"]])
191 @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
192 def TestInstanceShutdown(instance):
193 """gnt-instance shutdown"""
194 AssertCommand(["gnt-instance", "shutdown", instance["name"]])
197 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
198 def TestInstanceReboot(instance):
199 """gnt-instance reboot"""
200 options = qa_config.get("options", {})
201 reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
202 name = instance["name"]
203 for rtype in reboot_types:
204 AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
206 AssertCommand(["gnt-instance", "shutdown", name])
207 qa_utils.RunInstanceCheck(instance, False)
208 AssertCommand(["gnt-instance", "reboot", name])
210 master = qa_config.GetMasterNode()
211 cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
212 result_output = qa_utils.GetCommandOutput(master["primary"],
213 utils.ShellQuoteArgs(cmd))
214 AssertEqual(result_output.strip(), constants.INSTST_RUNNING)
217 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
218 def TestInstanceReinstall(instance):
219 """gnt-instance reinstall"""
220 AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
222 # Test with non-existant OS definition
223 AssertCommand(["gnt-instance", "reinstall", "-f",
224 "--os-type=NonExistantOsForQa",
229 def _ReadSsconfInstanceList():
230 """Reads ssconf_instance_list from the master node.
233 master = qa_config.GetMasterNode()
235 cmd = ["cat", utils.PathJoin(pathutils.DATA_DIR,
236 "ssconf_%s" % constants.SS_INSTANCE_LIST)]
238 return qa_utils.GetCommandOutput(master["primary"],
239 utils.ShellQuoteArgs(cmd)).splitlines()
242 def _CheckSsconfInstanceList(instance):
243 """Checks if a certain instance is in the ssconf instance list.
245 @type instance: string
246 @param instance: Instance name
249 AssertIn(qa_utils.ResolveInstanceName(instance),
250 _ReadSsconfInstanceList())
253 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
254 def TestInstanceRenameAndBack(rename_source, rename_target):
255 """gnt-instance rename
257 This must leave the instance with the original name, not the target
261 _CheckSsconfInstanceList(rename_source)
263 # first do a rename to a different actual name, expecting it to fail
264 qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
266 AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
268 _CheckSsconfInstanceList(rename_source)
270 qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
272 # Check instance volume tags correctly updated
273 # FIXME: this is LVM specific!
274 info = _GetInstanceInfo(rename_source)
275 tags_cmd = ("lvs -o tags --noheadings %s | grep " %
276 (" ".join(info["volumes"]), ))
278 # and now rename instance to rename_target...
279 AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
280 _CheckSsconfInstanceList(rename_target)
281 qa_utils.RunInstanceCheck(rename_source, False)
282 qa_utils.RunInstanceCheck(rename_target, False)
284 # NOTE: tags might not be the exactly as the instance name, due to
285 # charset restrictions; hence the test might be flaky
286 if rename_source != rename_target:
287 for node in info["nodes"]:
288 AssertCommand(tags_cmd + rename_source, node=node, fail=True)
289 AssertCommand(tags_cmd + rename_target, node=node, fail=False)
292 AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
293 _CheckSsconfInstanceList(rename_source)
294 qa_utils.RunInstanceCheck(rename_target, False)
296 if rename_source != rename_target:
297 for node in info["nodes"]:
298 AssertCommand(tags_cmd + rename_source, node=node, fail=False)
299 AssertCommand(tags_cmd + rename_target, node=node, fail=True)
302 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
303 def TestInstanceFailover(instance):
304 """gnt-instance failover"""
305 cmd = ["gnt-instance", "failover", "--force", instance["name"]]
309 qa_utils.RunInstanceCheck(instance, True)
315 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
316 def TestInstanceMigrate(instance, toggle_always_failover=True):
317 """gnt-instance migrate"""
318 cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
319 af_par = constants.BE_ALWAYS_FAILOVER
320 af_field = "be/" + constants.BE_ALWAYS_FAILOVER
321 af_init_val = _GetBoolInstanceField(instance["name"], af_field)
325 # TODO: Verify the choice between failover and migration
326 qa_utils.RunInstanceCheck(instance, True)
328 # ... and back (possibly with always_failover toggled)
329 if toggle_always_failover:
330 AssertCommand(["gnt-instance", "modify", "-B",
331 ("%s=%s" % (af_par, not af_init_val)),
334 # TODO: Verify the choice between failover and migration
335 qa_utils.RunInstanceCheck(instance, True)
336 if toggle_always_failover:
337 AssertCommand(["gnt-instance", "modify", "-B",
338 ("%s=%s" % (af_par, af_init_val)), instance["name"]])
340 # TODO: Split into multiple tests
341 AssertCommand(["gnt-instance", "shutdown", instance["name"]])
342 qa_utils.RunInstanceCheck(instance, False)
343 AssertCommand(cmd, fail=True)
344 AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
346 AssertCommand(["gnt-instance", "start", instance["name"]])
348 # @InstanceCheck enforces the check that the instance is running
349 qa_utils.RunInstanceCheck(instance, True)
351 AssertCommand(["gnt-instance", "modify", "-B",
353 (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
357 qa_utils.RunInstanceCheck(instance, True)
358 # TODO: Verify that a failover has been done instead of a migration
360 # TODO: Verify whether the default value is restored here (not hardcoded)
361 AssertCommand(["gnt-instance", "modify", "-B",
363 (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
367 qa_utils.RunInstanceCheck(instance, True)
370 def TestInstanceInfo(instance):
371 """gnt-instance info"""
372 AssertCommand(["gnt-instance", "info", instance["name"]])
375 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
376 def TestInstanceModify(instance):
377 """gnt-instance modify"""
378 default_hv = qa_config.GetDefaultHypervisor()
380 # Assume /sbin/init exists on all systems
381 test_kernel = "/sbin/init"
382 test_initrd = test_kernel
384 orig_maxmem = qa_config.get(constants.BE_MAXMEM)
385 orig_minmem = qa_config.get(constants.BE_MINMEM)
386 #orig_bridge = qa_config.get("bridge", "xen-br0")
389 ["-B", "%s=128" % constants.BE_MINMEM],
390 ["-B", "%s=128" % constants.BE_MAXMEM],
391 ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
392 constants.BE_MAXMEM, orig_maxmem)],
393 ["-B", "%s=2" % constants.BE_VCPUS],
394 ["-B", "%s=1" % constants.BE_VCPUS],
395 ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
396 ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
397 ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
399 ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
400 ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
403 #["--bridge", "xen-br1"],
404 #["--bridge", orig_bridge],
407 if default_hv == constants.HT_XEN_PVM:
409 ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
410 ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
411 ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
413 elif default_hv == constants.HT_XEN_HVM:
415 ["-H", "%s=acn" % constants.HV_BOOT_ORDER],
416 ["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
420 AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
423 AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
425 # Marking offline while instance is running must fail...
426 AssertCommand(["gnt-instance", "modify", "--offline", instance["name"]],
429 # ...while making it online is ok, and should work
430 AssertCommand(["gnt-instance", "modify", "--online", instance["name"]])
433 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
434 def TestInstanceStoppedModify(instance):
435 """gnt-instance modify (stopped instance)"""
436 name = instance["name"]
438 # Instance was not marked offline; try marking it online once more
439 AssertCommand(["gnt-instance", "modify", "--online", name])
441 # Mark instance as offline
442 AssertCommand(["gnt-instance", "modify", "--offline", name])
444 # When the instance is offline shutdown should only work with --force,
445 # while start should never work
446 AssertCommand(["gnt-instance", "shutdown", name], fail=True)
447 AssertCommand(["gnt-instance", "shutdown", "--force", name])
448 AssertCommand(["gnt-instance", "start", name], fail=True)
449 AssertCommand(["gnt-instance", "start", "--force", name], fail=True)
451 # Also do offline to offline
452 AssertCommand(["gnt-instance", "modify", "--offline", name])
455 AssertCommand(["gnt-instance", "modify", "--online", name])
458 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
459 def TestInstanceConvertDisk(instance, snode):
460 """gnt-instance modify -t"""
461 name = instance["name"]
462 AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
463 AssertCommand(["gnt-instance", "modify", "-t", "drbd",
464 "-n", snode["primary"], name])
467 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
468 def TestInstanceGrowDisk(instance):
469 """gnt-instance grow-disk"""
470 name = instance["name"]
471 all_size = qa_config.get("disk")
472 all_grow = qa_config.get("disk-growth")
474 # missing disk sizes but instance grow disk has been enabled,
475 # let's set fixed/nomimal growth
476 all_grow = ["128M" for _ in all_size]
477 for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
478 # succeed in grow by amount
479 AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
480 # fail in grow to the old size
481 AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
483 # succeed to grow to old size + 2 * growth
484 int_size = utils.ParseUnit(size)
485 int_grow = utils.ParseUnit(grow)
486 AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
487 str(int_size + 2 * int_grow)])
490 def TestInstanceList():
491 """gnt-instance list"""
492 qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
495 def TestInstanceListFields():
496 """gnt-instance list-fields"""
497 qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())
500 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
501 def TestInstanceConsole(instance):
502 """gnt-instance console"""
503 AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
506 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
507 def TestReplaceDisks(instance, pnode, snode, othernode):
508 """gnt-instance replace-disks"""
509 # pylint: disable=W0613
510 # due to unused pnode arg
511 # FIXME: should be removed from the function completely
513 cmd = ["gnt-instance", "replace-disks"]
515 cmd.append(instance["name"])
518 options = qa_config.get("options", {})
519 use_ialloc = options.get("use-iallocators", True)
523 # A placeholder; the actual command choice depends on use_ialloc
525 # Restore the original secondary
526 ["--new-secondary=%s" % snode["primary"]],
530 data = ["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT]
532 data = ["--new-secondary=%s" % othernode["primary"]]
533 AssertCommand(buildcmd(data))
535 AssertCommand(buildcmd(["-a"]))
536 AssertCommand(["gnt-instance", "stop", instance["name"]])
537 AssertCommand(buildcmd(["-a"]), fail=True)
538 AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
539 AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
541 AssertCommand(buildcmd(["-a"]))
542 AssertCommand(["gnt-instance", "start", instance["name"]])
545 def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
547 """Execute gnt-instance recreate-disks and check the result
549 @param cmdargs: Arguments (instance name excluded)
550 @param instance: Instance to operate on
551 @param fail: True if the command is expected to fail
552 @param check: If True and fail is False, check that the disks work
553 @prama destroy: If True, destroy the old disks first
557 _DestroyInstanceVolumes(instance)
558 AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs +
559 [instance["name"]]), fail)
560 if not fail and check:
561 # Quick check that the disks are there
562 AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
563 AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
565 AssertCommand(["gnt-instance", "deactivate-disks", instance["name"]])
568 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
569 def TestRecreateDisks(instance, pnode, snode, othernodes):
570 """gnt-instance recreate-disks
572 @param instance: Instance to work on
573 @param pnode: Primary node
574 @param snode: Secondary node, or None for sigle-homed instances
575 @param othernodes: list/tuple of nodes where to temporarily recreate disks
578 options = qa_config.get("options", {})
579 use_ialloc = options.get("use-iallocators", True)
580 other_seq = ":".join([n["primary"] for n in othernodes])
581 orig_seq = pnode["primary"]
583 orig_seq = orig_seq + ":" + snode["primary"]
584 # These fail because the instance is running
585 _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
587 _AssertRecreateDisks(["-I", "hail"], instance, fail=True, destroy=False)
589 _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
590 AssertCommand(["gnt-instance", "stop", instance["name"]])
591 # Disks exist: this should fail
592 _AssertRecreateDisks([], instance, fail=True, destroy=False)
593 # Recreate disks in place
594 _AssertRecreateDisks([], instance)
597 _AssertRecreateDisks(["-I", "hail"], instance)
598 # Move disks somewhere else
599 _AssertRecreateDisks(["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT],
602 _AssertRecreateDisks(["-n", other_seq], instance)
604 _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
605 # This and InstanceCheck decoration check that the disks are working
606 AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
607 AssertCommand(["gnt-instance", "start", instance["name"]])
610 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
611 def TestInstanceExport(instance, node):
612 """gnt-backup export -n ..."""
613 name = instance["name"]
614 AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
615 return qa_utils.ResolveInstanceName(name)
618 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
619 def TestInstanceExportWithRemove(instance, node):
620 """gnt-backup export --remove-instance"""
621 AssertCommand(["gnt-backup", "export", "-n", node["primary"],
622 "--remove-instance", instance["name"]])
625 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
626 def TestInstanceExportNoTarget(instance):
627 """gnt-backup export (without target node, should fail)"""
628 AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
631 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
632 def TestInstanceImport(newinst, node, expnode, name):
633 """gnt-backup import"""
634 cmd = (["gnt-backup", "import",
635 "--disk-template=plain",
637 "--src-node=%s" % expnode["primary"],
638 "--src-dir=%s/%s" % (pathutils.EXPORT_DIR, name),
639 "--node=%s" % node["primary"]] +
640 _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
641 cmd.append(newinst["name"])
645 def TestBackupList(expnode):
646 """gnt-backup list"""
647 AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
649 qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
650 namefield=None, test_unknown=False)
653 def TestBackupListFields():
654 """gnt-backup list-fields"""
655 qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())
658 def _TestInstanceDiskFailure(instance, node, node2, onmaster):
659 """Testing disk failure."""
660 master = qa_config.GetMasterNode()
661 sq = utils.ShellQuoteArgs
663 instance_full = qa_utils.ResolveInstanceName(instance["name"])
664 node_full = qa_utils.ResolveNodeName(node)
665 node2_full = qa_utils.ResolveNodeName(node2)
667 print qa_utils.FormatInfo("Getting physical disk names")
668 cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
669 "--output=node,phys,instance",
670 node["primary"], node2["primary"]]
671 output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
673 # Get physical disk names
674 re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
676 for line in output.splitlines():
677 (node_name, phys, inst) = line.split("|")
678 if inst == instance_full:
679 if node_name not in node2disk:
680 node2disk[node_name] = []
682 m = re_disk.match(phys)
684 raise qa_error.Error("Unknown disk name format: %s" % phys)
687 if name not in node2disk[node_name]:
688 node2disk[node_name].append(name)
690 if [node2_full, node_full][int(onmaster)] not in node2disk:
691 raise qa_error.Error("Couldn't find physical disks used on"
692 " %s node" % ["secondary", "master"][int(onmaster)])
694 print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
696 for node_name, disks in node2disk.iteritems():
699 cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
700 AssertCommand(" && ".join(cmds), node=node_name)
702 print qa_utils.FormatInfo("Getting device paths")
703 cmd = ["gnt-instance", "activate-disks", instance["name"]]
704 output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
706 for line in output.splitlines():
707 (_, _, tmpdevpath) = line.split(":")
708 devpath.append(tmpdevpath)
711 print qa_utils.FormatInfo("Getting drbd device paths")
712 cmd = ["gnt-instance", "info", instance["name"]]
713 output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
714 pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
715 r"\s+primary:\s+(/dev/drbd\d+)\s+")
716 drbddevs = re.findall(pattern, output, re.M)
721 print qa_utils.FormatInfo("Deactivating disks")
723 for name in node2disk[[node2_full, node_full][int(onmaster)]]:
724 halted_disks.append(name)
725 cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
726 AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
728 print qa_utils.FormatInfo("Write to disks and give some time to notice"
732 cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
733 "if=%s" % disk, "of=%s" % disk]))
735 AssertCommand(" && ".join(cmds), node=node)
738 print qa_utils.FormatInfo("Debugging info")
739 for name in drbddevs:
740 AssertCommand(["drbdsetup", name, "show"], node=node)
742 AssertCommand(["gnt-instance", "info", instance["name"]])
745 print qa_utils.FormatInfo("Activating disks again")
747 for name in halted_disks:
748 cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
749 AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
752 for name in drbddevs:
753 AssertCommand(["drbdsetup", name, "detach"], node=node)
755 for name in drbddevs:
756 AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
759 #AssertCommand(["vgs"], [node2, node][int(onmaster)])
761 print qa_utils.FormatInfo("Making sure disks are up again")
762 AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
764 print qa_utils.FormatInfo("Restarting instance")
765 AssertCommand(["gnt-instance", "shutdown", instance["name"]])
766 AssertCommand(["gnt-instance", "startup", instance["name"]])
768 AssertCommand(["gnt-cluster", "verify"])
771 def TestInstanceMasterDiskFailure(instance, node, node2):
772 """Testing disk failure on master node."""
773 # pylint: disable=W0613
775 print qa_utils.FormatError("Disk failure on primary node cannot be"
776 " tested due to potential crashes.")
777 # The following can cause crashes, thus it's disabled until fixed
778 #return _TestInstanceDiskFailure(instance, node, node2, True)
781 def TestInstanceSecondaryDiskFailure(instance, node, node2):
782 """Testing disk failure on secondary node."""
783 return _TestInstanceDiskFailure(instance, node, node2, False)