X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/26a5056d5c41625c1f2e7ffc7fb16a5a9f379b02..21e2734ff4966735948b309d9664819f47810e64:/qa/qa_instance.py

diff --git a/qa/qa_instance.py b/qa/qa_instance.py
index 2d0843a..b604d97 100644
--- a/qa/qa_instance.py
+++ b/qa/qa_instance.py
@@ -29,19 +29,21 @@ import time
 from ganeti import utils
 from ganeti import constants
 from ganeti import query
+from ganeti import pathutils
 
 import qa_config
 import qa_utils
 import qa_error
 
 from qa_utils import AssertIn, AssertCommand, AssertEqual
+from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
 
 
 def _GetDiskStatePath(disk):
   return "/sys/block/%s/device/state" % disk
 
 
-def _GetGenericAddParameters():
+def _GetGenericAddParameters(inst, force_mac=None):
   params = ["-B"]
   params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
                                  qa_config.get(constants.BE_MINMEM),
@@ -49,6 +51,15 @@ def _GetGenericAddParameters():
                                  qa_config.get(constants.BE_MAXMEM)))
   for idx, size in enumerate(qa_config.get("disk")):
     params.extend(["--disk", "%s:size=%s" % (idx, size)])
+
+  # Set static MAC address if configured
+  if force_mac:
+    nic0_mac = force_mac
+  else:
+    nic0_mac = qa_config.GetInstanceNicMac(inst)
+  if nic0_mac:
+    params.extend(["--net", "0:mac=%s" % nic0_mac])
+
   return params
 
 
@@ -59,7 +70,7 @@ def _DiskTest(node, disk_template):
             "--os-type=%s" % qa_config.get("os"),
             "--disk-template=%s" % disk_template,
             "--node=%s" % node] +
-           _GetGenericAddParameters())
+           _GetGenericAddParameters(instance))
     cmd.append(instance["name"])
 
     AssertCommand(cmd)
@@ -72,17 +83,98 @@ def _DiskTest(node, disk_template):
     raise
 
 
+def _GetInstanceInfo(instance):
+  """Return information about the actual state of an instance.
+
+  @type instance: string
+  @param instance: the instance name
+  @return: a dictionary with two keys:
+      - "nodes": instance nodes, a list of strings
+      - "volumes": instance volume IDs, a list of strings
+
+  """
+  master = qa_config.GetMasterNode()
+  infocmd = utils.ShellQuoteArgs(["gnt-instance", "info", instance])
+  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
+  re_node = re.compile(r"^\s+-\s+(?:primary|secondaries):\s+(\S.+)$")
+  node_elem = r"([^,()]+)(?:\s+\([^)]+\))?"
+  # re_nodelist matches a list of nodes returned by gnt-instance info, e.g.:
+  #  node1.fqdn
+  #  node2.fqdn,node3.fqdn
+  #  node4.fqdn (group mygroup, group UUID 01234567-abcd-0123-4567-0123456789ab)
+  # FIXME This works with no more than 2 secondaries
+  re_nodelist = re.compile(node_elem + "(?:," + node_elem + ")?$")
+  re_vol = re.compile(r"^\s+logical_id:\s+(\S+)$")
+  nodes = []
+  vols = []
+  for line in info_out.splitlines():
+    m = re_node.match(line)
+    if m:
+      nodestr = m.group(1)
+      m2 = re_nodelist.match(nodestr)
+      if m2:
+        nodes.extend(filter(None, m2.groups()))
+      else:
+        nodes.append(nodestr)
+    m = re_vol.match(line)
+    if m:
+      vols.append(m.group(1))
+  assert vols
+  assert nodes
+  return {"nodes": nodes, "volumes": vols}
+
+
+def _DestroyInstanceVolumes(instance):
+  """Remove all the LVM volumes of an instance.
+
+  This is used to simulate HW errors (dead nodes, broken disks...); the
+  configuration of the instance is not affected.
+  @type instance: dictionary
+  @param instance: the instance
+
+  """
+  info = _GetInstanceInfo(instance["name"])
+  vols = info["volumes"]
+  for node in info["nodes"]:
+    AssertCommand(["lvremove", "-f"] + vols, node=node)
+
+
+def _GetBoolInstanceField(instance, field):
+  """Get the Boolean value of a field of an instance.
+
+  @type instance: string
+  @param instance: Instance name
+  @type field: string
+  @param field: Name of the field
+
+  """
+  master = qa_config.GetMasterNode()
+  infocmd = utils.ShellQuoteArgs(["gnt-instance", "list", "--no-headers",
+                                  "-o", field, instance])
+  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd).strip()
+  if info_out == "Y":
+    return True
+  elif info_out == "N":
+    return False
+  else:
+    raise qa_error.Error("Field %s of instance %s has a non-Boolean value:"
+                         " %s" % (field, instance, info_out))
+
+
+@InstanceCheck(None, INST_UP, RETURN_VALUE)
 def TestInstanceAddWithPlainDisk(node):
   """gnt-instance add -t plain"""
   return _DiskTest(node["primary"], "plain")
 
 
+@InstanceCheck(None, INST_UP, RETURN_VALUE)
 def TestInstanceAddWithDrbdDisk(node, node2):
   """gnt-instance add -t drbd"""
   return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
                    "drbd")
 
 
+@InstanceCheck(None, INST_DOWN, FIRST_ARG)
 def TestInstanceRemove(instance):
   """gnt-instance remove"""
   AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
@@ -90,16 +182,19 @@ def TestInstanceRemove(instance):
   qa_config.ReleaseInstance(instance)
 
 
+@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
 def TestInstanceStartup(instance):
   """gnt-instance startup"""
   AssertCommand(["gnt-instance", "startup", instance["name"]])
 
 
+@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
 def TestInstanceShutdown(instance):
   """gnt-instance shutdown"""
   AssertCommand(["gnt-instance", "shutdown", instance["name"]])
 
 
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceReboot(instance):
   """gnt-instance reboot"""
   options = qa_config.get("options", {})
@@ -109,6 +204,7 @@ def TestInstanceReboot(instance):
     AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
 
   AssertCommand(["gnt-instance", "shutdown", name])
+  qa_utils.RunInstanceCheck(instance, False)
   AssertCommand(["gnt-instance", "reboot", name])
 
   master = qa_config.GetMasterNode()
@@ -118,10 +214,17 @@ def TestInstanceReboot(instance):
   AssertEqual(result_output.strip(), constants.INSTST_RUNNING)
 
 
+@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
 def TestInstanceReinstall(instance):
   """gnt-instance reinstall"""
   AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
 
+  # Test with non-existant OS definition
+  AssertCommand(["gnt-instance", "reinstall", "-f",
+                 "--os-type=NonExistantOsForQa",
+                 instance["name"]],
+                fail=True)
+
 
 def _ReadSsconfInstanceList():
   """Reads ssconf_instance_list from the master node.
@@ -129,7 +232,7 @@ def _ReadSsconfInstanceList():
   """
   master = qa_config.GetMasterNode()
 
-  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
+  cmd = ["cat", utils.PathJoin(pathutils.DATA_DIR,
                                "ssconf_%s" % constants.SS_INSTANCE_LIST)]
 
   return qa_utils.GetCommandOutput(master["primary"],
@@ -147,6 +250,7 @@ def _CheckSsconfInstanceList(instance):
            _ReadSsconfInstanceList())
 
 
+@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
 def TestInstanceRenameAndBack(rename_source, rename_target):
   """gnt-instance rename
 
@@ -155,6 +259,7 @@ def TestInstanceRenameAndBack(rename_source, rename_target):
 
   """
   _CheckSsconfInstanceList(rename_source)
+
   # first do a rename to a different actual name, expecting it to fail
   qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
   try:
@@ -163,48 +268,103 @@ def TestInstanceRenameAndBack(rename_source, rename_target):
     _CheckSsconfInstanceList(rename_source)
   finally:
     qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
+
+  # Check instance volume tags correctly updated
+  # FIXME: this is LVM specific!
+  info = _GetInstanceInfo(rename_source)
+  tags_cmd = ("lvs -o tags --noheadings %s | grep " %
+              (" ".join(info["volumes"]), ))
+
   # and now rename instance to rename_target...
   AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
   _CheckSsconfInstanceList(rename_target)
+  qa_utils.RunInstanceCheck(rename_source, False)
+  qa_utils.RunInstanceCheck(rename_target, False)
+
+  # NOTE: tags might not be the exactly as the instance name, due to
+  # charset restrictions; hence the test might be flaky
+  if rename_source != rename_target:
+    for node in info["nodes"]:
+      AssertCommand(tags_cmd + rename_source, node=node, fail=True)
+      AssertCommand(tags_cmd + rename_target, node=node, fail=False)
+
   # and back
   AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
   _CheckSsconfInstanceList(rename_source)
+  qa_utils.RunInstanceCheck(rename_target, False)
 
+  if rename_source != rename_target:
+    for node in info["nodes"]:
+      AssertCommand(tags_cmd + rename_source, node=node, fail=False)
+      AssertCommand(tags_cmd + rename_target, node=node, fail=True)
 
+
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceFailover(instance):
   """gnt-instance failover"""
   cmd = ["gnt-instance", "failover", "--force", instance["name"]]
+
   # failover ...
   AssertCommand(cmd)
+  qa_utils.RunInstanceCheck(instance, True)
+
   # ... and back
   AssertCommand(cmd)
 
 
-def TestInstanceMigrate(instance):
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
+def TestInstanceMigrate(instance, toggle_always_failover=True):
   """gnt-instance migrate"""
   cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
+  af_par = constants.BE_ALWAYS_FAILOVER
+  af_field = "be/" + constants.BE_ALWAYS_FAILOVER
+  af_init_val = _GetBoolInstanceField(instance["name"], af_field)
+
   # migrate ...
   AssertCommand(cmd)
-  # ... and back
+  # TODO: Verify the choice between failover and migration
+  qa_utils.RunInstanceCheck(instance, True)
+
+  # ... and back (possibly with always_failover toggled)
+  if toggle_always_failover:
+    AssertCommand(["gnt-instance", "modify", "-B",
+                   ("%s=%s" % (af_par, not af_init_val)),
+                   instance["name"]])
   AssertCommand(cmd)
+  # TODO: Verify the choice between failover and migration
+  qa_utils.RunInstanceCheck(instance, True)
+  if toggle_always_failover:
+    AssertCommand(["gnt-instance", "modify", "-B",
+                   ("%s=%s" % (af_par, af_init_val)), instance["name"]])
+
+  # TODO: Split into multiple tests
   AssertCommand(["gnt-instance", "shutdown", instance["name"]])
+  qa_utils.RunInstanceCheck(instance, False)
   AssertCommand(cmd, fail=True)
   AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
                  instance["name"]])
   AssertCommand(["gnt-instance", "start", instance["name"]])
   AssertCommand(cmd)
+  # @InstanceCheck enforces the check that the instance is running
+  qa_utils.RunInstanceCheck(instance, True)
+
   AssertCommand(["gnt-instance", "modify", "-B",
                  ("%s=%s" %
                   (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
                  instance["name"]])
-  AssertCommand(cmd, fail=True)
-  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
-                 instance["name"]])
+
+  AssertCommand(cmd)
+  qa_utils.RunInstanceCheck(instance, True)
+  # TODO: Verify that a failover has been done instead of a migration
+
+  # TODO: Verify whether the default value is restored here (not hardcoded)
   AssertCommand(["gnt-instance", "modify", "-B",
                  ("%s=%s" %
                   (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
                  instance["name"]])
+
   AssertCommand(cmd)
+  qa_utils.RunInstanceCheck(instance, True)
 
 
 def TestInstanceInfo(instance):
@@ -212,8 +372,11 @@ def TestInstanceInfo(instance):
   AssertCommand(["gnt-instance", "info", instance["name"]])
 
 
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceModify(instance):
   """gnt-instance modify"""
+  default_hv = qa_config.GetDefaultHypervisor()
+
   # Assume /sbin/init exists on all systems
   test_kernel = "/sbin/init"
   test_initrd = test_kernel
@@ -221,6 +384,7 @@ def TestInstanceModify(instance):
   orig_maxmem = qa_config.get(constants.BE_MAXMEM)
   orig_minmem = qa_config.get(constants.BE_MINMEM)
   #orig_bridge = qa_config.get("bridge", "xen-br0")
+
   args = [
     ["-B", "%s=128" % constants.BE_MINMEM],
     ["-B", "%s=128" % constants.BE_MAXMEM],
@@ -234,29 +398,39 @@ def TestInstanceModify(instance):
 
     ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
     ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
-    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
-    ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
-    ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
 
     # TODO: bridge tests
     #["--bridge", "xen-br1"],
     #["--bridge", orig_bridge],
-
-    # TODO: Do these tests only with xen-hvm
-    #["-H", "%s=acn" % constants.HV_BOOT_ORDER],
-    #["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
     ]
+
+  if default_hv == constants.HT_XEN_PVM:
+    args.extend([
+      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
+      ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
+      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
+      ])
+  elif default_hv == constants.HT_XEN_HVM:
+    args.extend([
+      ["-H", "%s=acn" % constants.HV_BOOT_ORDER],
+      ["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
+      ])
+
   for alist in args:
     AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
 
   # check no-modify
   AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
 
-  # Marking offline/online while instance is running must fail
-  for arg in ["--online", "--offline"]:
-    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)
+  # Marking offline while instance is running must fail...
+  AssertCommand(["gnt-instance", "modify", "--offline", instance["name"]],
+                 fail=True)
+
+  # ...while making it online is ok, and should work
+  AssertCommand(["gnt-instance", "modify", "--online", instance["name"]])
 
 
+@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
 def TestInstanceStoppedModify(instance):
   """gnt-instance modify (stopped instance)"""
   name = instance["name"]
@@ -267,10 +441,21 @@ def TestInstanceStoppedModify(instance):
   # Mark instance as offline
   AssertCommand(["gnt-instance", "modify", "--offline", name])
 
+  # When the instance is offline shutdown should only work with --force,
+  # while start should never work
+  AssertCommand(["gnt-instance", "shutdown", name], fail=True)
+  AssertCommand(["gnt-instance", "shutdown", "--force", name])
+  AssertCommand(["gnt-instance", "start", name], fail=True)
+  AssertCommand(["gnt-instance", "start", "--force", name], fail=True)
+
+  # Also do offline to offline
+  AssertCommand(["gnt-instance", "modify", "--offline", name])
+
   # And online again
   AssertCommand(["gnt-instance", "modify", "--online", name])
 
 
+@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
 def TestInstanceConvertDisk(instance, snode):
   """gnt-instance modify -t"""
   name = instance["name"]
@@ -279,6 +464,7 @@ def TestInstanceConvertDisk(instance, snode):
                  "-n", snode["primary"], name])
 
 
+@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
 def TestInstanceGrowDisk(instance):
   """gnt-instance grow-disk"""
   name = instance["name"]
@@ -311,11 +497,13 @@ def TestInstanceListFields():
   qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())
 
 
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceConsole(instance):
   """gnt-instance console"""
   AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
 
 
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestReplaceDisks(instance, pnode, snode, othernode):
   """gnt-instance replace-disks"""
   # pylint: disable=W0613
@@ -327,23 +515,99 @@ def TestReplaceDisks(instance, pnode, snode, othernode):
     cmd.append(instance["name"])
     return cmd
 
+  options = qa_config.get("options", {})
+  use_ialloc = options.get("use-iallocators", True)
   for data in [
     ["-p"],
     ["-s"],
-    ["--new-secondary=%s" % othernode["primary"]],
-    # and restore
+    # A placeholder; the actual command choice depends on use_ialloc
+    None,
+    # Restore the original secondary
     ["--new-secondary=%s" % snode["primary"]],
     ]:
+    if data is None:
+      if use_ialloc:
+        data = ["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT]
+      else:
+        data = ["--new-secondary=%s" % othernode["primary"]]
     AssertCommand(buildcmd(data))
 
   AssertCommand(buildcmd(["-a"]))
   AssertCommand(["gnt-instance", "stop", instance["name"]])
   AssertCommand(buildcmd(["-a"]), fail=True)
   AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
+  AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
+                 instance["name"]])
   AssertCommand(buildcmd(["-a"]))
   AssertCommand(["gnt-instance", "start", instance["name"]])
 
 
+def _AssertRecreateDisks(cmdargs, instance, fail=False, check=True,
+                         destroy=True):
+  """Execute gnt-instance recreate-disks and check the result
+
+  @param cmdargs: Arguments (instance name excluded)
+  @param instance: Instance to operate on
+  @param fail: True if the command is expected to fail
+  @param check: If True and fail is False, check that the disks work
+  @prama destroy: If True, destroy the old disks first
+
+  """
+  if destroy:
+    _DestroyInstanceVolumes(instance)
+  AssertCommand((["gnt-instance", "recreate-disks"] + cmdargs +
+                 [instance["name"]]), fail)
+  if not fail and check:
+    # Quick check that the disks are there
+    AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
+    AssertCommand(["gnt-instance", "activate-disks", "--wait-for-sync",
+                   instance["name"]])
+    AssertCommand(["gnt-instance", "deactivate-disks", instance["name"]])
+
+
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
+def TestRecreateDisks(instance, pnode, snode, othernodes):
+  """gnt-instance recreate-disks
+
+  @param instance: Instance to work on
+  @param pnode: Primary node
+  @param snode: Secondary node, or None for sigle-homed instances
+  @param othernodes: list/tuple of nodes where to temporarily recreate disks
+
+  """
+  options = qa_config.get("options", {})
+  use_ialloc = options.get("use-iallocators", True)
+  other_seq = ":".join([n["primary"] for n in othernodes])
+  orig_seq = pnode["primary"]
+  if snode:
+    orig_seq = orig_seq + ":" + snode["primary"]
+  # These fail because the instance is running
+  _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
+  if use_ialloc:
+    _AssertRecreateDisks(["-I", "hail"], instance, fail=True, destroy=False)
+  else:
+    _AssertRecreateDisks(["-n", other_seq], instance, fail=True, destroy=False)
+  AssertCommand(["gnt-instance", "stop", instance["name"]])
+  # Disks exist: this should fail
+  _AssertRecreateDisks([], instance, fail=True, destroy=False)
+  # Recreate disks in place
+  _AssertRecreateDisks([], instance)
+  # Move disks away
+  if use_ialloc:
+    _AssertRecreateDisks(["-I", "hail"], instance)
+    # Move disks somewhere else
+    _AssertRecreateDisks(["-I", constants.DEFAULT_IALLOCATOR_SHORTCUT],
+                         instance)
+  else:
+    _AssertRecreateDisks(["-n", other_seq], instance)
+  # Move disks back
+  _AssertRecreateDisks(["-n", orig_seq], instance, check=False)
+  # This and InstanceCheck decoration check that the disks are working
+  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
+  AssertCommand(["gnt-instance", "start", instance["name"]])
+
+
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceExport(instance, node):
   """gnt-backup export -n ..."""
   name = instance["name"]
@@ -351,27 +615,29 @@ def TestInstanceExport(instance, node):
   return qa_utils.ResolveInstanceName(name)
 
 
+@InstanceCheck(None, INST_DOWN, FIRST_ARG)
 def TestInstanceExportWithRemove(instance, node):
   """gnt-backup export --remove-instance"""
   AssertCommand(["gnt-backup", "export", "-n", node["primary"],
                  "--remove-instance", instance["name"]])
 
 
+@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
 def TestInstanceExportNoTarget(instance):
   """gnt-backup export (without target node, should fail)"""
   AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
 
 
-def TestInstanceImport(node, newinst, expnode, name):
+@InstanceCheck(None, INST_DOWN, FIRST_ARG)
+def TestInstanceImport(newinst, node, expnode, name):
   """gnt-backup import"""
   cmd = (["gnt-backup", "import",
           "--disk-template=plain",
           "--no-ip-check",
-          "--net", "0:mac=generate",
           "--src-node=%s" % expnode["primary"],
-          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
+          "--src-dir=%s/%s" % (pathutils.EXPORT_DIR, name),
           "--node=%s" % node["primary"]] +
-         _GetGenericAddParameters())
+         _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
   cmd.append(newinst["name"])
   AssertCommand(cmd)
 
@@ -460,7 +726,7 @@ def _TestInstanceDiskFailure(instance, node, node2, onmaster):
     AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
 
     print qa_utils.FormatInfo("Write to disks and give some time to notice"
-                              " to notice the problem")
+                              " the problem")
     cmds = []
     for disk in devpath:
       cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",