4 # Copyright (C) 2007, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Instance related QA tests.
29 from ganeti import utils
30 from ganeti import constants
31 from ganeti import query
32 from ganeti import pathutils
38 from qa_utils import AssertIn, AssertCommand, AssertEqual
39 from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
42 def _GetDiskStatePath(disk):
43 return "/sys/block/%s/device/state" % disk
46 def _GetGenericAddParameters(inst, force_mac=None):
48 params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
49 qa_config.get(constants.BE_MINMEM),
51 qa_config.get(constants.BE_MAXMEM)))
52 for idx, size in enumerate(qa_config.get("disk")):
53 params.extend(["--disk", "%s:size=%s" % (idx, size)])
55 # Set static MAC address if configured
59 nic0_mac = qa_config.GetInstanceNicMac(inst)
61 params.extend(["--net", "0:mac=%s" % nic0_mac])
66 def _DiskTest(node, disk_template):
67 instance = qa_config.AcquireInstance()
69 cmd = (["gnt-instance", "add",
70 "--os-type=%s" % qa_config.get("os"),
71 "--disk-template=%s" % disk_template,
73 _GetGenericAddParameters(instance))
74 cmd.append(instance["name"])
78 _CheckSsconfInstanceList(instance["name"])
82 qa_config.ReleaseInstance(instance)
86 def _GetInstanceInfo(instance):
87 """Return information about the actual state of an instance.
89 The pieces of information returned are:
90 - "nodes": instance nodes, a list of strings
91 - "volumes": instance volume IDs, a list of strings
92 @type instance: dictionary
93 @param instance: the instance
97 master = qa_config.GetMasterNode()
98 infocmd = utils.ShellQuoteArgs(["gnt-instance", "info", instance["name"]])
99 info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
100 re_node = re.compile(r"^\s+-\s+(?:primary|secondaries):\s+(\S.+)$")
101 node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
102 # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
104 # node2.fqdn,node3.fqdn
105 # node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
106 # FIXME This works with no more than 2 secondaries
107 re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
108 re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
111 for line in info_out.splitlines():
112 m = re_node.match(line)
115 m2 = re_nodelist.match(nodestr)
117 nodes.extend(filter(None, m2.groups()))
119 nodes.append(nodestr)
120 m = re_vol.match(line)
122 vols.append(m.group(1))
125 return {"nodes": nodes, "volumes": vols}
128 def _DestroyInstanceVolumes(instance):
129 """Remove all the LVM volumes of an instance.
131 This is used to simulate HW errors (dead nodes, broken disks...); the
132 configuration of the instance is not affected.
133 @type instance: dictionary
134 @param instance: the instance
137 info = _GetInstanceInfo(instance)
138 vols = info["volumes"]
139 for node in info["nodes"]:
140 AssertCommand(["lvremove", "-f"] + vols, node=node)
143 def _GetBoolInstanceField(instance, field):
144 """Get the Boolean value of a field of an instance.
146 @type instance: string
147 @param instance: Instance name
149 @param field: Name of the field
152 master = qa_config.GetMasterNode()
153 infocmd = utils.ShellQuoteArgs(["gnt-instance", "list", "--no-headers",
154 "-o", field, instance])
155 info_out = qa_utils.GetCommandOutput(master["primary"], infocmd).strip()
158 elif info_out == "N":
161 raise qa_error.Error("Field %s of instance %s has a non-Boolean value:"
162 " %s" % (field, instance, info_out))
165 @InstanceCheck(None, INST_UP, RETURN_VALUE)
166 def TestInstanceAddWithPlainDisk(node):
167 """gnt-instance add -t plain"""
168 return _DiskTest(node["primary"], "plain")
171 @InstanceCheck(None, INST_UP, RETURN_VALUE)
172 def TestInstanceAddWithDrbdDisk(node, node2):
173 """gnt-instance add -t drbd"""
174 return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
178 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
179 def TestInstanceRemove(instance):
180 """gnt-instance remove"""
181 AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
183 qa_config.ReleaseInstance(instance)
186 @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
187 def TestInstanceStartup(instance):
188 """gnt-instance startup"""
189 AssertCommand(["gnt-instance", "startup", instance["name"]])
192 @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
193 def TestInstanceShutdown(instance):
194 """gnt-instance shutdown"""
195 AssertCommand(["gnt-instance", "shutdown", instance["name"]])
198 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
199 def TestInstanceReboot(instance):
200 """gnt-instance reboot"""
201 options = qa_config.get("options", {})
202 reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
203 name = instance["name"]
204 for rtype in reboot_types:
205 AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
207 AssertCommand(["gnt-instance", "shutdown", name])
208 qa_utils.RunInstanceCheck(instance, False)
209 AssertCommand(["gnt-instance", "reboot", name])
211 master = qa_config.GetMasterNode()
212 cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
213 result_output = qa_utils.GetCommandOutput(master["primary"],
214 utils.ShellQuoteArgs(cmd))
215 AssertEqual(result_output.strip(), constants.INSTST_RUNNING)
218 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
219 def TestInstanceReinstall(instance):
220 """gnt-instance reinstall"""
221 AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
224 def _ReadSsconfInstanceList():
225 """Reads ssconf_instance_list from the master node.
228 master = qa_config.GetMasterNode()
230 cmd = ["cat", utils.PathJoin(pathutils.DATA_DIR,
231 "ssconf_%s" % constants.SS_INSTANCE_LIST)]
233 return qa_utils.GetCommandOutput(master["primary"],
234 utils.ShellQuoteArgs(cmd)).splitlines()
237 def _CheckSsconfInstanceList(instance):
238 """Checks if a certain instance is in the ssconf instance list.
240 @type instance: string
241 @param instance: Instance name
244 AssertIn(qa_utils.ResolveInstanceName(instance),
245 _ReadSsconfInstanceList())
248 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
249 def TestInstanceRenameAndBack(rename_source, rename_target):
250 """gnt-instance rename
252 This must leave the instance with the original name, not the target
256 _CheckSsconfInstanceList(rename_source)
258 # first do a rename to a different actual name, expecting it to fail
259 qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
261 AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
263 _CheckSsconfInstanceList(rename_source)
265 qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
267 # and now rename instance to rename_target...
268 AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
269 _CheckSsconfInstanceList(rename_target)
270 qa_utils.RunInstanceCheck(rename_source, False)
271 qa_utils.RunInstanceCheck(rename_target, False)
274 AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
275 _CheckSsconfInstanceList(rename_source)
276 qa_utils.RunInstanceCheck(rename_target, False)
279 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
280 def TestInstanceFailover(instance):
281 """gnt-instance failover"""
282 cmd = ["gnt-instance", "failover", "--force", instance["name"]]
286 qa_utils.RunInstanceCheck(instance, True)
292 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
293 def TestInstanceMigrate(instance, toggle_always_failover=True):
294 """gnt-instance migrate"""
295 cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
296 af_par = constants.BE_ALWAYS_FAILOVER
297 af_field = "be/" + constants.BE_ALWAYS_FAILOVER
298 af_init_val = _GetBoolInstanceField(instance["name"], af_field)
302 # TODO: Verify the choice between failover and migration
303 qa_utils.RunInstanceCheck(instance, True)
305 # ... and back (possibly with always_failover toggled)
306 if toggle_always_failover:
307 AssertCommand(["gnt-instance", "modify", "-B",
308 ("%s=%s" % (af_par, not af_init_val)),
311 # TODO: Verify the choice between failover and migration
312 qa_utils.RunInstanceCheck(instance, True)
313 if toggle_always_failover:
314 AssertCommand(["gnt-instance", "modify", "-B",
315 ("%s=%s" % (af_par, af_init_val)), instance["name"]])
317 # TODO: Split into multiple tests
318 AssertCommand(["gnt-instance", "shutdown", instance["name"]])
319 qa_utils.RunInstanceCheck(instance, False)
320 AssertCommand(cmd, fail=True)
321 AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
323 AssertCommand(["gnt-instance", "start", instance["name"]])
325 # @InstanceCheck enforces the check that the instance is running
328 def TestInstanceInfo(instance):
329 """gnt-instance info"""
330 AssertCommand(["gnt-instance", "info", instance["name"]])
333 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
334 def TestInstanceModify(instance):
335 """gnt-instance modify"""
336 default_hv = qa_config.GetDefaultHypervisor()
338 # Assume /sbin/init exists on all systems
339 test_kernel = "/sbin/init"
340 test_initrd = test_kernel
342 orig_maxmem = qa_config.get(constants.BE_MAXMEM)
343 orig_minmem = qa_config.get(constants.BE_MINMEM)
344 #orig_bridge = qa_config.get("bridge", "xen-br0")
347 ["-B", "%s=128" % constants.BE_MINMEM],
348 ["-B", "%s=128" % constants.BE_MAXMEM],
349 ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
350 constants.BE_MAXMEM, orig_maxmem)],
351 ["-B", "%s=2" % constants.BE_VCPUS],
352 ["-B", "%s=1" % constants.BE_VCPUS],
353 ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
354 ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
355 ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
357 ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
358 ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
361 #["--bridge", "xen-br1"],
362 #["--bridge", orig_bridge],
365 if default_hv == constants.HT_XEN_PVM:
367 ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
368 ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
369 ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
371 elif default_hv == constants.HT_XEN_HVM:
373 ["-H", "%s=acn" % constants.HV_BOOT_ORDER],
374 ["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
378 AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
381 AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
383 # Marking offline/online while instance is running must fail
384 for arg in ["--online", "--offline"]:
385 AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)
388 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
389 def TestInstanceStoppedModify(instance):
390 """gnt-instance modify (stopped instance)"""
391 name = instance["name"]
393 # Instance was not marked offline; try marking it online once more
394 AssertCommand(["gnt-instance", "modify", "--online", name])
396 # Mark instance as offline
397 AssertCommand(["gnt-instance", "modify", "--offline", name])
400 AssertCommand(["gnt-instance", "modify", "--online", name])
403 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
404 def TestInstanceConvertDisk(instance, snode):
405 """gnt-instance modify -t"""
406 name = instance["name"]
407 AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
408 AssertCommand(["gnt-instance", "modify", "-t", "drbd",
409 "-n", snode["primary"], name])
412 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
413 def TestInstanceGrowDisk(instance):
414 """gnt-instance grow-disk"""
415 name = instance["name"]
416 all_size = qa_config.get("disk")
417 all_grow = qa_config.get("disk-growth")
419 # missing disk sizes but instance grow disk has been enabled,
420 # let's set fixed/nomimal growth
421 all_grow = ["128M" for _ in all_size]
422 for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
423 # succeed in grow by amount
424 AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
425 # fail in grow to the old size
426 AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
428 # succeed to grow to old size + 2 * growth
429 int_size = utils.ParseUnit(size)
430 int_grow = utils.ParseUnit(grow)
431 AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
432 str(int_size + 2 * int_grow)])
435 def TestInstanceList():
436 """gnt-instance list"""
437 qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
440 def TestInstanceListFields():
441 """gnt-instance list-fields"""
442 qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())
445 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
446 def TestInstanceConsole(instance):
447 """gnt-instance console"""
448 AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
451 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
452 def TestReplaceDisks(instance, pnode, snode, othernode):
453 """gnt-instance replace-disks"""
454 # pylint: disable=W0613
455 # due to unused pnode arg
456 # FIXME: should be removed from the function completely
458 cmd = ["gnt-instance", "replace-disks"]
460 cmd.append(instance["name"])
466 ["--new-secondary=%s" % othernode["primary"]],
467 ["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT],
469 AssertCommand(buildcmd(data))
471 # Restore the original secondary, if needed
472 currsec = _GetInstanceInfo(instance)["nodes"][1]
473 if currsec != snode["primary"]:
474 AssertCommand(buildcmd(["--new-secondary=%s" % snode["primary"]]))
476 AssertCommand(buildcmd(["-a"]))
477 AssertCommand(["gnt-instance", "stop", instance["name"]])
478 AssertCommand(buildcmd(["-a"]), fail=True)
479 AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
480 AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
482 AssertCommand(buildcmd(["-a"]))
483 AssertCommand(["gnt-instance", "start", instance["name"]])
486 def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
488 """Execute gnt-instance recreate-disks and check the result
490 @param cmdargs: Arguments (instance name excluded)
491 @param instance: Instance to operate on
492 @param fail: True if the command is expected to fail
493 @param check: If True and fail is False, check that the disks work
494 @prama destroy: If True, destroy the old disks first
498 _DestroyInstanceVolumes(instance)
499 AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs +
500 [instance["name"]]), fail)
501 if not fail and check:
502 # Quick check that the disks are there
503 AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
504 AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
506 AssertCommand(["gnt-instance", "deactivate-disks", instance["name"]])
509 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
510 def TestRecreateDisks(instance, pnode, snode, othernodes):
511 """gnt-instance recreate-disks
513 @param instance: Instance to work on
514 @param pnode: Primary node
515 @param snode: Secondary node, or None for sigle-homed instances
516 @param othernodes: list/tuple of nodes where to temporarily recreate disks
519 other_seq = ":".join([n["primary"] for n in othernodes])
520 orig_seq = pnode["primary"]
522 orig_seq = orig_seq + ":" + snode["primary"]
523 # These fail because the instance is running
524 _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
525 _AssertRecreateDisks(["-I", "hail"], instance, fail=True, destroy=False)
526 AssertCommand(["gnt-instance", "stop", instance["name"]])
527 # Disks exist: this should fail
528 _AssertRecreateDisks([], instance, fail=True, destroy=False)
529 # Recreate disks in place
530 _AssertRecreateDisks([], instance)
532 _AssertRecreateDisks(["-I", "hail"], instance)
533 # Move disks somewhere else
534 _AssertRecreateDisks(["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT], instance)
536 _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
537 # This and InstanceCheck decoration check that the disks are working
538 AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
539 AssertCommand(["gnt-instance", "start", instance["name"]])
542 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
543 def TestInstanceExport(instance, node):
544 """gnt-backup export -n ..."""
545 name = instance["name"]
546 AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
547 return qa_utils.ResolveInstanceName(name)
550 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
551 def TestInstanceExportWithRemove(instance, node):
552 """gnt-backup export --remove-instance"""
553 AssertCommand(["gnt-backup", "export", "-n", node["primary"],
554 "--remove-instance", instance["name"]])
557 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
558 def TestInstanceExportNoTarget(instance):
559 """gnt-backup export (without target node, should fail)"""
560 AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
563 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
564 def TestInstanceImport(newinst, node, expnode, name):
565 """gnt-backup import"""
566 cmd = (["gnt-backup", "import",
567 "--disk-template=plain",
569 "--src-node=%s" % expnode["primary"],
570 "--src-dir=%s/%s" % (pathutils.EXPORT_DIR, name),
571 "--node=%s" % node["primary"]] +
572 _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
573 cmd.append(newinst["name"])
577 def TestBackupList(expnode):
578 """gnt-backup list"""
579 AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
581 qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
582 namefield=None, test_unknown=False)
585 def TestBackupListFields():
586 """gnt-backup list-fields"""
587 qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())
590 def _TestInstanceDiskFailure(instance, node, node2, onmaster):
591 """Testing disk failure."""
592 master = qa_config.GetMasterNode()
593 sq = utils.ShellQuoteArgs
595 instance_full = qa_utils.ResolveInstanceName(instance["name"])
596 node_full = qa_utils.ResolveNodeName(node)
597 node2_full = qa_utils.ResolveNodeName(node2)
599 print qa_utils.FormatInfo("Getting physical disk names")
600 cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
601 "--output=node,phys,instance",
602 node["primary"], node2["primary"]]
603 output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
605 # Get physical disk names
606 re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
608 for line in output.splitlines():
609 (node_name, phys, inst) = line.split("|")
610 if inst == instance_full:
611 if node_name not in node2disk:
612 node2disk[node_name] = []
614 m = re_disk.match(phys)
616 raise qa_error.Error("Unknown disk name format: %s" % phys)
619 if name not in node2disk[node_name]:
620 node2disk[node_name].append(name)
622 if [node2_full, node_full][int(onmaster)] not in node2disk:
623 raise qa_error.Error("Couldn't find physical disks used on"
624 " %s node" % ["secondary", "master"][int(onmaster)])
626 print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
628 for node_name, disks in node2disk.iteritems():
631 cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
632 AssertCommand(" && ".join(cmds), node=node_name)
634 print qa_utils.FormatInfo("Getting device paths")
635 cmd = ["gnt-instance", "activate-disks", instance["name"]]
636 output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
638 for line in output.splitlines():
639 (_, _, tmpdevpath) = line.split(":")
640 devpath.append(tmpdevpath)
643 print qa_utils.FormatInfo("Getting drbd device paths")
644 cmd = ["gnt-instance", "info", instance["name"]]
645 output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
646 pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
647 r"\s+primary:\s+(/dev/drbd\d+)\s+")
648 drbddevs = re.findall(pattern, output, re.M)
653 print qa_utils.FormatInfo("Deactivating disks")
655 for name in node2disk[[node2_full, node_full][int(onmaster)]]:
656 halted_disks.append(name)
657 cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
658 AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
660 print qa_utils.FormatInfo("Write to disks and give some time to notice"
664 cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
665 "if=%s" % disk, "of=%s" % disk]))
667 AssertCommand(" && ".join(cmds), node=node)
670 print qa_utils.FormatInfo("Debugging info")
671 for name in drbddevs:
672 AssertCommand(["drbdsetup", name, "show"], node=node)
674 AssertCommand(["gnt-instance", "info", instance["name"]])
677 print qa_utils.FormatInfo("Activating disks again")
679 for name in halted_disks:
680 cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
681 AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
684 for name in drbddevs:
685 AssertCommand(["drbdsetup", name, "detach"], node=node)
687 for name in drbddevs:
688 AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
691 #AssertCommand(["vgs"], [node2, node][int(onmaster)])
693 print qa_utils.FormatInfo("Making sure disks are up again")
694 AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
696 print qa_utils.FormatInfo("Restarting instance")
697 AssertCommand(["gnt-instance", "shutdown", instance["name"]])
698 AssertCommand(["gnt-instance", "startup", instance["name"]])
700 AssertCommand(["gnt-cluster", "verify"])
703 def TestInstanceMasterDiskFailure(instance, node, node2):
704 """Testing disk failure on master node."""
705 # pylint: disable=W0613
707 print qa_utils.FormatError("Disk failure on primary node cannot be"
708 " tested due to potential crashes.")
709 # The following can cause crashes, thus it's disabled until fixed
710 #return _TestInstanceDiskFailure(instance, node, node2, True)
713 def TestInstanceSecondaryDiskFailure(instance, node, node2):
714 """Testing disk failure on secondary node."""
715 return _TestInstanceDiskFailure(instance, node, node2, False)