Add QA for recreating single instance disks
[ganeti-local] / qa / qa_instance.py
index 9dab9da..55d5fb7 100644 (file)
@@ -23,6 +23,7 @@
 
 """
 
+import operator
 import re
 
 from ganeti import utils
@@ -62,7 +63,7 @@ def _GetGenericAddParameters(inst, force_mac=None):
   return params
 
 
-def _DiskTest(node, disk_template):
+def _DiskTest(node, disk_template, fail=False):
   instance = qa_config.AcquireInstance()
   try:
     cmd = (["gnt-instance", "add",
@@ -72,25 +73,34 @@ def _DiskTest(node, disk_template):
            _GetGenericAddParameters(instance))
     cmd.append(instance["name"])
 
-    AssertCommand(cmd)
+    AssertCommand(cmd, fail=fail)
 
-    _CheckSsconfInstanceList(instance["name"])
-    qa_config.SetInstanceTemplate(instance, disk_template)
+    if not fail:
+      _CheckSsconfInstanceList(instance["name"])
+      qa_config.SetInstanceTemplate(instance, disk_template)
 
-    return instance
+      return instance
   except:
     qa_config.ReleaseInstance(instance)
     raise
 
+  # Handle the case where creation is expected to fail
+  assert fail
+  qa_config.ReleaseInstance(instance)
+  return None
+
 
 def _GetInstanceInfo(instance):
   """Return information about the actual state of an instance.
 
   @type instance: string
   @param instance: the instance name
-  @return: a dictionary with two keys:
+  @return: a dictionary with the following keys:
       - "nodes": instance nodes, a list of strings
       - "volumes": instance volume IDs, a list of strings
+      - "drbd-minors": DRBD minors used by the instance, a dictionary where
+        keys are nodes, and values are lists of integers (or an empty
+        dictionary for non-DRBD instances)
 
   """
   master = qa_config.GetMasterNode()
@@ -105,8 +115,10 @@ def _GetInstanceInfo(instance):
   # FIXME This works with no more than 2 secondaries
   re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
   re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
+  re_drbdnode = re.compile(r"^\s+node[AB]:\s+([^\s,]+),\s+minor=([0-9]+)$")
   nodes = []
   vols = []
+  drbd_min = {}
   for line in info_out.splitlines():
     m = re_node.match(line)
     if m:
@@ -119,9 +131,17 @@ def _GetInstanceInfo(instance):
     m = re_vol.match(line)
     if m:
       vols.append(m.group(1))
+    m = re_drbdnode.match(line)
+    if m:
+      node = m.group(1)
+      minor = int(m.group(2))
+      if drbd_min.get(node) is not None:
+        drbd_min[node].append(minor)
+      else:
+        drbd_min[node] = [minor]
   assert vols
   assert nodes
-  return {"nodes": nodes, "volumes": vols}
+  return {"nodes": nodes, "volumes": vols, "drbd-minors": drbd_min}
 
 
 def _DestroyInstanceVolumes(instance):
@@ -139,19 +159,33 @@ def _DestroyInstanceVolumes(instance):
     AssertCommand(["lvremove", "-f"] + vols, node=node)
 
 
-def _GetBoolInstanceField(instance, field):
-  """Get the Boolean value of a field of an instance.
+def _GetInstanceField(instance, field):
+  """Get the value of a field of an instance.
 
   @type instance: string
   @param instance: Instance name
   @type field: string
   @param field: Name of the field
+  @rtype: string
 
   """
   master = qa_config.GetMasterNode()
   infocmd = utils.ShellQuoteArgs(["gnt-instance", "list", "--no-headers",
-                                  "-o", field, instance])
-  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd).strip()
+                                  "--units", "m", "-o", field, instance])
+  return qa_utils.GetCommandOutput(master["primary"], infocmd).strip()
+
+
+def _GetBoolInstanceField(instance, field):
+  """Get the Boolean value of a field of an instance.
+
+  @type instance: string
+  @param instance: Instance name
+  @type field: string
+  @param field: Name of the field
+  @rtype: bool
+
+  """
+  info_out = _GetInstanceField(instance, field)
   if info_out == "Y":
     return True
   elif info_out == "N":
@@ -161,16 +195,93 @@ def _GetBoolInstanceField(instance, field):
                          " %s" % (field, instance, info_out))
 
 
-@InstanceCheck(None, INST_UP, RETURN_VALUE)
-def TestInstanceAddWithPlainDisk(node):
+def _GetNumInstanceField(instance, field):
+  """Get a numeric value of a field of an instance.
+
+  @type instance: string
+  @param instance: Instance name
+  @type field: string
+  @param field: Name of the field
+  @rtype: int or float
+
+  """
+  info_out = _GetInstanceField(instance, field)
+  try:
+    ret = int(info_out)
+  except ValueError:
+    try:
+      ret = float(info_out)
+    except ValueError:
+      raise qa_error.Error("Field %s of instance %s has a non-numeric value:"
+                           " %s" % (field, instance, info_out))
+  return ret
+
+
+def GetInstanceSpec(instance, spec):
+  """Return the current spec for the given parameter.
+
+  @type instance: string
+  @param instance: Instance name
+  @type spec: string
+  @param spec: one of the supported parameters: "mem-size", "cpu-count",
+      "disk-count", "disk-size", "nic-count"
+  @rtype: tuple
+  @return: (minspec, maxspec); minspec and maxspec can be different only for
+      memory and disk size
+
+  """
+  specmap = {
+    "mem-size": ["be/minmem", "be/maxmem"],
+    "cpu-count": ["vcpus"],
+    "disk-count": ["disk.count"],
+    "disk-size": ["disk.size/ "],
+    "nic-count": ["nic.count"],
+    }
+  # For disks, first we need the number of disks
+  if spec == "disk-size":
+    (numdisk, _) = GetInstanceSpec(instance, "disk-count")
+    fields = ["disk.size/%s" % k for k in range(0, numdisk)]
+  else:
+    assert spec in specmap, "%s not in %s" % (spec, specmap)
+    fields = specmap[spec]
+  values = [_GetNumInstanceField(instance, f) for f in fields]
+  return (min(values), max(values))
+
+
+def IsFailoverSupported(instance):
+  templ = qa_config.GetInstanceTemplate(instance)
+  return templ in constants.DTS_MIRRORED
+
+
+def IsMigrationSupported(instance):
+  templ = qa_config.GetInstanceTemplate(instance)
+  return templ in constants.DTS_MIRRORED
+
+
+def IsDiskReplacingSupported(instance):
+  templ = qa_config.GetInstanceTemplate(instance)
+  return templ == constants.DT_DRBD8
+
+
+def IsDiskSupported(instance):
+  templ = qa_config.GetInstanceTemplate(instance)
+  return templ != constants.DT_DISKLESS
+
+
+def TestInstanceAddWithPlainDisk(nodes, fail=False):
   """gnt-instance add -t plain"""
-  return _DiskTest(node["primary"], "plain")
+  assert len(nodes) == 1
+  instance = _DiskTest(nodes[0]["primary"], constants.DT_PLAIN, fail=fail)
+  if not fail:
+    qa_utils.RunInstanceCheck(instance, True)
+  return instance
 
 
 @InstanceCheck(None, INST_UP, RETURN_VALUE)
-def TestInstanceAddWithDrbdDisk(node, node2):
+def TestInstanceAddWithDrbdDisk(nodes):
   """gnt-instance add -t drbd"""
-  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
+  assert len(nodes) == 2
+  return _DiskTest(":".join(map(operator.itemgetter("primary"), nodes)),
                    "drbd")
 
 
@@ -302,6 +413,11 @@ def TestInstanceRenameAndBack(rename_source, rename_target):
 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceFailover(instance):
   """gnt-instance failover"""
+  if not IsFailoverSupported(instance):
+    print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
+                              " test")
+    return
+
   cmd = ["gnt-instance", "failover", "--force", instance["name"]]
 
   # failover ...
@@ -315,6 +431,11 @@ def TestInstanceFailover(instance):
 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceMigrate(instance, toggle_always_failover=True):
   """gnt-instance migrate"""
+  if not IsMigrationSupported(instance):
+    print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
+                              " test")
+    return
+
   cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
   af_par = constants.BE_ALWAYS_FAILOVER
   af_field = "be/" + constants.BE_ALWAYS_FAILOVER
@@ -456,17 +577,40 @@ def TestInstanceStoppedModify(instance):
 
 
 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
-def TestInstanceConvertDisk(instance, snode):
+def TestInstanceConvertDiskToPlain(instance, inodes):
   """gnt-instance modify -t"""
   name = instance["name"]
+  template = qa_config.GetInstanceTemplate(instance)
+  if template != "drbd":
+    print qa_utils.FormatInfo("Unsupported template %s, skipping conversion"
+                              " test" % template)
+    return
+  assert len(inodes) == 2
   AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
   AssertCommand(["gnt-instance", "modify", "-t", "drbd",
-                 "-n", snode["primary"], name])
+                 "-n", inodes[1]["primary"], name])
+
+
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
+def TestInstanceModifyDisks(instance):
+  """gnt-instance modify --disk"""
+  if not IsDiskSupported(instance):
+    print qa_utils.FormatInfo("Instance doesn't support disks, skipping test")
+    return
+
+  size = qa_config.get("disk")[-1]
+  name = instance["name"]
+  build_cmd = lambda arg: ["gnt-instance", "modify", "--disk", arg, name]
+  AssertCommand(build_cmd("add:size=%s" % size))
+  AssertCommand(build_cmd("remove"))
 
 
 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
 def TestInstanceGrowDisk(instance):
   """gnt-instance grow-disk"""
+  if qa_config.GetExclusiveStorage():
+    print qa_utils.FormatInfo("Test not supported with exclusive_storage")
+    return
   name = instance["name"]
   all_size = qa_config.get("disk")
   all_grow = qa_config.get("disk-growth")
@@ -504,17 +648,25 @@ def TestInstanceConsole(instance):
 
 
 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
-def TestReplaceDisks(instance, pnode, snode, othernode):
+def TestReplaceDisks(instance, curr_nodes, other_nodes):
   """gnt-instance replace-disks"""
-  # pylint: disable=W0613
-  # due to unused pnode arg
-  # FIXME: should be removed from the function completely
   def buildcmd(args):
     cmd = ["gnt-instance", "replace-disks"]
     cmd.extend(args)
     cmd.append(instance["name"])
     return cmd
 
+  if not IsDiskReplacingSupported(instance):
+    print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
+                              " skipping test")
+    return
+
+  # Currently all supported templates have one primary and one secondary node
+  assert len(curr_nodes) == 2
+  snode = curr_nodes[1]
+  assert len(other_nodes) == 1
+  othernode = other_nodes[0]
+
   options = qa_config.get("options", {})
   use_ialloc = options.get("use-iallocators", True)
   for data in [
@@ -566,21 +718,18 @@ def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
 
 
 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
-def TestRecreateDisks(instance, pnode, snode, othernodes):
+def TestRecreateDisks(instance, inodes, othernodes):
   """gnt-instance recreate-disks
 
   @param instance: Instance to work on
-  @param pnode: Primary node
-  @param snode: Secondary node, or None for sigle-homed instances
+  @param inodes: List of the current nodes of the instance
   @param othernodes: list/tuple of nodes where to temporarily recreate disks
 
   """
   options = qa_config.get("options", {})
   use_ialloc = options.get("use-iallocators", True)
   other_seq = ":".join([n["primary"] for n in othernodes])
-  orig_seq = pnode["primary"]
-  if snode:
-    orig_seq = orig_seq + ":" + snode["primary"]
+  orig_seq = ":".join([n["primary"] for n in inodes])
   # These fail because the instance is running
   _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
   if use_ialloc:
@@ -601,7 +750,13 @@ def TestRecreateDisks(instance, pnode, snode, othernodes):
   else:
     _AssertRecreateDisks(["-n", other_seq], instance)
   # Move disks back
-  _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
+  _AssertRecreateDisks(["-n", orig_seq], instance)
+  # Recreate the disks one by one
+  for idx in range(0, len(qa_config.get("disk"))):
+    # Only the first call should destroy all the disk
+    destroy = (idx == 0)
+    _AssertRecreateDisks(["--disk=%s" % idx], instance, destroy=destroy,
+                         check=False)
   # This and InstanceCheck decoration check that the disks are working
   AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
   AssertCommand(["gnt-instance", "start", instance["name"]])
@@ -656,3 +811,24 @@ def TestBackupList(expnode):
 def TestBackupListFields():
   """gnt-backup list-fields"""
   qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())
+
+
+def TestRemoveInstanceOfflineNode(instance, snode, set_offline, set_online):
+  """gtn-instance remove with an off-line node
+
+  @param instance: instance
+  @param snode: secondary node, to be set offline
+  @param set_offline: function to call to set the node off-line
+  @param set_online: function to call to set the node on-line
+
+  """
+  info = _GetInstanceInfo(instance["name"])
+  set_offline(snode)
+  try:
+    TestInstanceRemove(instance)
+  finally:
+    set_online(snode)
+  # Clean up the disks on the offline node
+  for minor in info["drbd-minors"][snode["primary"]]:
+    AssertCommand(["drbdsetup", str(minor), "down"], node=snode)
+  AssertCommand(["lvremove", "-f"] + info["volumes"], node=snode)