24 |
24 |
"""
|
25 |
25 |
|
26 |
26 |
import re
|
27 |
|
import time
|
28 |
27 |
|
29 |
28 |
from ganeti import utils
|
30 |
29 |
from ganeti import constants
|
... | ... | |
654 |
653 |
def TestBackupListFields():
|
655 |
654 |
"""gnt-backup list-fields"""
|
656 |
655 |
qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())
|
657 |
|
|
658 |
|
|
659 |
|
def _TestInstanceDiskFailure(instance, node, node2, onmaster):
|
660 |
|
"""Testing disk failure."""
|
661 |
|
master = qa_config.GetMasterNode()
|
662 |
|
sq = utils.ShellQuoteArgs
|
663 |
|
|
664 |
|
instance_full = qa_utils.ResolveInstanceName(instance["name"])
|
665 |
|
node_full = qa_utils.ResolveNodeName(node)
|
666 |
|
node2_full = qa_utils.ResolveNodeName(node2)
|
667 |
|
|
668 |
|
print qa_utils.FormatInfo("Getting physical disk names")
|
669 |
|
cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
|
670 |
|
"--output=node,phys,instance",
|
671 |
|
node["primary"], node2["primary"]]
|
672 |
|
output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
|
673 |
|
|
674 |
|
# Get physical disk names
|
675 |
|
re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
|
676 |
|
node2disk = {}
|
677 |
|
for line in output.splitlines():
|
678 |
|
(node_name, phys, inst) = line.split("|")
|
679 |
|
if inst == instance_full:
|
680 |
|
if node_name not in node2disk:
|
681 |
|
node2disk[node_name] = []
|
682 |
|
|
683 |
|
m = re_disk.match(phys)
|
684 |
|
if not m:
|
685 |
|
raise qa_error.Error("Unknown disk name format: %s" % phys)
|
686 |
|
|
687 |
|
name = m.group(1)
|
688 |
|
if name not in node2disk[node_name]:
|
689 |
|
node2disk[node_name].append(name)
|
690 |
|
|
691 |
|
if [node2_full, node_full][int(onmaster)] not in node2disk:
|
692 |
|
raise qa_error.Error("Couldn't find physical disks used on"
|
693 |
|
" %s node" % ["secondary", "master"][int(onmaster)])
|
694 |
|
|
695 |
|
print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
|
696 |
|
" disks")
|
697 |
|
for node_name, disks in node2disk.iteritems():
|
698 |
|
cmds = []
|
699 |
|
for disk in disks:
|
700 |
|
cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
|
701 |
|
AssertCommand(" && ".join(cmds), node=node_name)
|
702 |
|
|
703 |
|
print qa_utils.FormatInfo("Getting device paths")
|
704 |
|
cmd = ["gnt-instance", "activate-disks", instance["name"]]
|
705 |
|
output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
|
706 |
|
devpath = []
|
707 |
|
for line in output.splitlines():
|
708 |
|
(_, _, tmpdevpath) = line.split(":")
|
709 |
|
devpath.append(tmpdevpath)
|
710 |
|
print devpath
|
711 |
|
|
712 |
|
print qa_utils.FormatInfo("Getting drbd device paths")
|
713 |
|
cmd = ["gnt-instance", "info", instance["name"]]
|
714 |
|
output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
|
715 |
|
pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
|
716 |
|
r"\s+primary:\s+(/dev/drbd\d+)\s+")
|
717 |
|
drbddevs = re.findall(pattern, output, re.M)
|
718 |
|
print drbddevs
|
719 |
|
|
720 |
|
halted_disks = []
|
721 |
|
try:
|
722 |
|
print qa_utils.FormatInfo("Deactivating disks")
|
723 |
|
cmds = []
|
724 |
|
for name in node2disk[[node2_full, node_full][int(onmaster)]]:
|
725 |
|
halted_disks.append(name)
|
726 |
|
cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
|
727 |
|
AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
|
728 |
|
|
729 |
|
print qa_utils.FormatInfo("Write to disks and give some time to notice"
|
730 |
|
" the problem")
|
731 |
|
cmds = []
|
732 |
|
for disk in devpath:
|
733 |
|
cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
|
734 |
|
"if=%s" % disk, "of=%s" % disk]))
|
735 |
|
for _ in (0, 1, 2):
|
736 |
|
AssertCommand(" && ".join(cmds), node=node)
|
737 |
|
time.sleep(3)
|
738 |
|
|
739 |
|
print qa_utils.FormatInfo("Debugging info")
|
740 |
|
for name in drbddevs:
|
741 |
|
AssertCommand(["drbdsetup", name, "show"], node=node)
|
742 |
|
|
743 |
|
AssertCommand(["gnt-instance", "info", instance["name"]])
|
744 |
|
|
745 |
|
finally:
|
746 |
|
print qa_utils.FormatInfo("Activating disks again")
|
747 |
|
cmds = []
|
748 |
|
for name in halted_disks:
|
749 |
|
cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
|
750 |
|
AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
|
751 |
|
|
752 |
|
if onmaster:
|
753 |
|
for name in drbddevs:
|
754 |
|
AssertCommand(["drbdsetup", name, "detach"], node=node)
|
755 |
|
else:
|
756 |
|
for name in drbddevs:
|
757 |
|
AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
|
758 |
|
|
759 |
|
# TODO
|
760 |
|
#AssertCommand(["vgs"], [node2, node][int(onmaster)])
|
761 |
|
|
762 |
|
print qa_utils.FormatInfo("Making sure disks are up again")
|
763 |
|
AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
|
764 |
|
|
765 |
|
print qa_utils.FormatInfo("Restarting instance")
|
766 |
|
AssertCommand(["gnt-instance", "shutdown", instance["name"]])
|
767 |
|
AssertCommand(["gnt-instance", "startup", instance["name"]])
|
768 |
|
|
769 |
|
AssertCommand(["gnt-cluster", "verify"])
|
770 |
|
|
771 |
|
|
772 |
|
def TestInstanceMasterDiskFailure(instance, node, node2):
|
773 |
|
"""Testing disk failure on master node."""
|
774 |
|
# pylint: disable=W0613
|
775 |
|
# due to unused args
|
776 |
|
print qa_utils.FormatError("Disk failure on primary node cannot be"
|
777 |
|
" tested due to potential crashes.")
|
778 |
|
# The following can cause crashes, thus it's disabled until fixed
|
779 |
|
#return _TestInstanceDiskFailure(instance, node, node2, True)
|
780 |
|
|
781 |
|
|
782 |
|
def TestInstanceSecondaryDiskFailure(instance, node, node2):
|
783 |
|
"""Testing disk failure on secondary node."""
|
784 |
|
return _TestInstanceDiskFailure(instance, node, node2, False)
|