Synnefo » snf-ganeti

#

#

# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful, but

# WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

# 02110-1301, USA.

"""Cluster related QA tests.

"""

import re

import tempfile

import os.path

from ganeti import constants

from ganeti import compat

from ganeti import utils

from ganeti import pathutils

import qa_config

import qa_daemon

import qa_utils

import qa_error

import qa_instance

from qa_utils import AssertEqual, AssertCommand, GetCommandOutput

# Prefix for LVM volumes created by QA code during tests

_QA_LV_PREFIX = "qa-"

#: cluster verify command

_CLUSTER_VERIFY = ["gnt-cluster", "verify"]

def _RemoveFileFromAllNodes(filename):

  """Removes a file from all nodes.

  """

  for node in qa_config.get("nodes"):

    AssertCommand(["rm", "-f", filename], node=node)

def _CheckFileOnAllNodes(filename, content):

  """Verifies the content of the given file on all nodes.

  """

  cmd = utils.ShellQuoteArgs(["cat", filename])

  for node in qa_config.get("nodes"):

    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)

def _GetClusterField(field_path):

  """Get the value of a cluster field.

  @type field_path: list of strings

  @param field_path: Names of the groups/fields to navigate to get the desired

      value, e.g. C{["Default node parameters", "oob_program"]}

  @return: The effective value of the field (the actual type depends on the

      chosen field)

  """

  assert isinstance(field_path, list)

  assert field_path

  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])

  for key in field_path:

    ret = ret[key]

  return ret

# Cluster-verify errors (date, "ERROR", then error code)

_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")

def _GetCVErrorCodes(cvout):

  errs = set()

  warns = set()

  for l in cvout.splitlines():

    m = _CVERROR_RE.match(l)

    if m:

      etype = m.group(1)

      ecode = m.group(2)

      if etype == "ERROR":

        errs.add(ecode)

      elif etype == "WARNING":

        warns.add(ecode)

  return (errs, warns)

def _CheckVerifyErrors(actual, expected, etype):

  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)

  if not actual.issuperset(exp_codes):

    missing = exp_codes.difference(actual)

    raise qa_error.Error("Cluster-verify didn't return these expected"

                         " %ss: %s" % (etype, utils.CommaJoin(missing)))

def AssertClusterVerify(fail=False, errors=None, warnings=None):

  """Run cluster-verify and check the result

  @type fail: bool

  @param fail: if cluster-verify is expected to fail instead of succeeding

  @type errors: list of tuples

  @param errors: List of CV_XXX errors that are expected; if specified, all the

      errors listed must appear in cluster-verify output. A non-empty value

      implies C{fail=True}.

  @type warnings: list of tuples

  @param warnings: Same as C{errors} but for warnings.

  """

  cvcmd = "gnt-cluster verify"

  mnode = qa_config.GetMasterNode()

  if errors or warnings:

    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",

                             fail=(fail or errors))

    (act_errs, act_warns) = _GetCVErrorCodes(cvout)

    if errors:

      _CheckVerifyErrors(act_errs, errors, "error")

    if warnings:

      _CheckVerifyErrors(act_warns, warnings, "warning")

  else:

    AssertCommand(cvcmd, fail=fail, node=mnode)

# data for testing failures due to bad keys/values for disk parameters

_FAIL_PARAMS = ["nonexistent:resync-rate=1",

                "drbd:nonexistent=1",

                "drbd:resync-rate=invalid",

                ]

def TestClusterInitDisk():

  """gnt-cluster init -D"""

  name = qa_config.get("name")

  for param in _FAIL_PARAMS:

    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)

def TestClusterInit(rapi_user, rapi_secret):

  """gnt-cluster init"""

  master = qa_config.GetMasterNode()

  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)

  rapi_dir = os.path.dirname(rapi_users_path)

  # First create the RAPI credentials

  fh = tempfile.NamedTemporaryFile()

  try:

    fh.write("%s %s write\n" % (rapi_user, rapi_secret))

    fh.flush()

    tmpru = qa_utils.UploadFile(master.primary, fh.name)

    try:

      AssertCommand(["mkdir", "-p", rapi_dir])

      AssertCommand(["mv", tmpru, rapi_users_path])

    finally:

      AssertCommand(["rm", "-f", tmpru])

  finally:

    fh.close()

  # Initialize cluster

  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()

  cmd = [

    "gnt-cluster", "init",

    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),

    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),

    "--enabled-disk-templates=%s" %

      ",".join(enabled_disk_templates),

    ]

  if constants.DT_FILE in enabled_disk_templates:

    cmd.append(

        "--file-storage-dir=%s" %

        qa_config.get("file-storage-dir", pathutils.DEFAULT_FILE_STORAGE_DIR))

  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",

                    "nic-count"):

    for spec_val in ("min", "max", "std"):

      spec = qa_config.get("ispec_%s_%s" %

                           (spec_type.replace("-", "_"), spec_val), None)

      if spec is not None:

        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))

  if master.secondary:

    cmd.append("--secondary-ip=%s" % master.secondary)

  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):

    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)

    if vgname:

      cmd.append("--vg-name=%s" % vgname)

    else:

      raise qa_error.Error("Please specify a volume group if you enable"

                           " lvm-based disk templates in the QA.")

  master_netdev = qa_config.get("master-netdev", None)

  if master_netdev:

    cmd.append("--master-netdev=%s" % master_netdev)

  nicparams = qa_config.get("default-nicparams", None)

  if nicparams:

    cmd.append("--nic-parameters=%s" %

               ",".join(utils.FormatKeyValue(nicparams)))

  # Cluster value of the exclusive-storage node parameter

  e_s = qa_config.get("exclusive-storage")

  if e_s is not None:

    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])

  else:

    e_s = False

  qa_config.SetExclusiveStorage(e_s)

  extra_args = qa_config.get("cluster-init-args")

  if extra_args:

    cmd.extend(extra_args)

  cmd.append(qa_config.get("name"))

  AssertCommand(cmd)

  cmd = ["gnt-cluster", "modify"]

  # hypervisor parameter modifications

  hvp = qa_config.get("hypervisor-parameters", {})

  for k, v in hvp.items():

    cmd.extend(["-H", "%s:%s" % (k, v)])

  # backend parameter modifications

  bep = qa_config.get("backend-parameters", "")

  if bep:

    cmd.extend(["-B", bep])

  if len(cmd) > 2:

    AssertCommand(cmd)

  # OS parameters

  osp = qa_config.get("os-parameters", {})

  for k, v in osp.items():

    AssertCommand(["gnt-os", "modify", "-O", v, k])

  # OS hypervisor parameters

  os_hvp = qa_config.get("os-hvp", {})

  for os_name in os_hvp:

    for hv, hvp in os_hvp[os_name].items():

      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])

def TestClusterRename():

  """gnt-cluster rename"""

  cmd = ["gnt-cluster", "rename", "-f"]

  original_name = qa_config.get("name")

  rename_target = qa_config.get("rename", None)

  if rename_target is None:

    print qa_utils.FormatError('"rename" entry is missing')

    return

  for data in [

    cmd + [rename_target],

    _CLUSTER_VERIFY,

    cmd + [original_name],

    _CLUSTER_VERIFY,

    ]:

    AssertCommand(data)

def TestClusterOob():

  """out-of-band framework"""

  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()

  AssertCommand(_CLUSTER_VERIFY)

  AssertCommand(["gnt-cluster", "modify", "--node-parameters",

                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %

                 utils.NewUUID()])

  AssertCommand(_CLUSTER_VERIFY, fail=True)

  AssertCommand(["touch", oob_path_exists])

  AssertCommand(["chmod", "0400", oob_path_exists])

  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])

  try:

    AssertCommand(["gnt-cluster", "modify", "--node-parameters",

                   "oob_program=%s" % oob_path_exists])

    AssertCommand(_CLUSTER_VERIFY, fail=True)

    AssertCommand(["chmod", "0500", oob_path_exists])

    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])

    AssertCommand(_CLUSTER_VERIFY)

  finally:

    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])

  AssertCommand(["gnt-cluster", "modify", "--node-parameters",

                 "oob_program="])

def TestClusterEpo():

  """gnt-cluster epo"""

  master = qa_config.GetMasterNode()

  # Assert that OOB is unavailable for all nodes

  result_output = GetCommandOutput(master.primary,

                                   "gnt-node list --verbose --no-headers -o"

                                   " powered")

  AssertEqual(compat.all(powered == "(unavail)"

                         for powered in result_output.splitlines()), True)

  # Conflicting

  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)

  # --all doesn't expect arguments

  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)

  # Unless --all is given master is not allowed to be in the list

  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)

  # This shouldn't fail

  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])

  # All instances should have been stopped now

  result_output = GetCommandOutput(master.primary,

                                   "gnt-instance list --no-headers -o status")

  # ERROR_down because the instance is stopped but not recorded as such

  AssertEqual(compat.all(status == "ERROR_down"

                         for status in result_output.splitlines()), True)

  # Now start everything again

  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])

  # All instances should have been started now

  result_output = GetCommandOutput(master.primary,

                                   "gnt-instance list --no-headers -o status")

  AssertEqual(compat.all(status == "running"

                         for status in result_output.splitlines()), True)

def TestClusterVerify():

  """gnt-cluster verify"""

  AssertCommand(_CLUSTER_VERIFY)

  AssertCommand(["gnt-cluster", "verify-disks"])

def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes):

  """gnt-cluster verify-disks with broken DRBD"""

  qa_daemon.TestPauseWatcher()

  try:

    info = qa_instance.GetInstanceInfo(instance.name)

    snode = inst_nodes[1]

    for idx, minor in enumerate(info["drbd-minors"][snode.primary]):

      if idx % 2 == 0:

        break_drbd_cmd = \

          "(drbdsetup %d down >/dev/null 2>&1;" \

          " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \

          (minor, minor)

      else:

        break_drbd_cmd = \

          "(drbdsetup %d detach >/dev/null 2>&1;" \

          " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \

          (minor, minor)

      AssertCommand(break_drbd_cmd, node=snode)

    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,

                                     "gnt-cluster verify-disks")

    activation_msg = "Activating disks for instance '%s'" % instance.name

    if activation_msg not in verify_output:

      raise qa_error.Error("gnt-cluster verify-disks did not activate broken"

                           " DRBD disks:\n%s" % verify_output)

    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,

                                     "gnt-cluster verify-disks")

    if activation_msg in verify_output:

      raise qa_error.Error("gnt-cluster verify-disks wants to activate broken"

                           " DRBD disks on second attempt:\n%s" % verify_output)

    AssertCommand(_CLUSTER_VERIFY)

  finally:

    qa_daemon.TestResumeWatcher()

def TestJobqueue():

  """gnt-debug test-jobqueue"""

  AssertCommand(["gnt-debug", "test-jobqueue"])

def TestDelay(node):

  """gnt-debug delay"""

  AssertCommand(["gnt-debug", "delay", "1"])

  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])

  AssertCommand(["gnt-debug", "delay", "--no-master",

                 "-n", node.primary, "1"])

def TestClusterReservedLvs():

  """gnt-cluster reserved lvs"""

  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)

  lvname = _QA_LV_PREFIX + "test"

  lvfullname = "/".join([vgname, lvname])

  for fail, cmd in [

    (False, _CLUSTER_VERIFY),

    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),

    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),

    (True, _CLUSTER_VERIFY),

    (False, ["gnt-cluster", "modify", "--reserved-lvs",

             "%s,.*/other-test" % lvfullname]),

    (False, _CLUSTER_VERIFY),

    (False, ["gnt-cluster", "modify", "--reserved-lvs",

             ".*/%s.*" % _QA_LV_PREFIX]),

    (False, _CLUSTER_VERIFY),

    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),

    (True, _CLUSTER_VERIFY),

    (False, ["lvremove", "-f", lvfullname]),

    (False, _CLUSTER_VERIFY),

    ]:

    AssertCommand(cmd, fail=fail)

def TestClusterModifyEmpty():

  """gnt-cluster modify"""

  AssertCommand(["gnt-cluster", "modify"], fail=True)

def TestClusterModifyDisk():

  """gnt-cluster modify -D"""

  for param in _FAIL_PARAMS:

    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)

def TestClusterModifyDiskTemplates():

  """gnt-cluster modify --enabled-disk-templates=..."""

  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()

  default_disk_template = qa_config.GetDefaultDiskTemplate()

  _TestClusterModifyDiskTemplatesArguments(default_disk_template,

                                           enabled_disk_templates)

  _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates)

  _RestoreEnabledDiskTemplates()

  nodes = qa_config.AcquireManyNodes(2)

  instance_template = enabled_disk_templates[0]

  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)

  _TestClusterModifyUnusedDiskTemplate(instance_template)

  _TestClusterModifyUsedDiskTemplate(instance_template,

                                     enabled_disk_templates)

  qa_instance.TestInstanceRemove(instance)

  _RestoreEnabledDiskTemplates()

def _RestoreEnabledDiskTemplates():

  """Sets the list of enabled disk templates back to the list of enabled disk

     templates from the QA configuration. This can be used to make sure that

     the tests that modify the list of disk templates do not interfere with

     other tests.

  """

  cmd = ["gnt-cluster", "modify", "--enabled-disk-templates=%s" %

         ",".join(qa_config.GetEnabledDiskTemplates())]

  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):

    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)

    cmd.append("--vg-name=%s" % vgname)

  AssertCommand(cmd, fail=False)

def _TestClusterModifyDiskTemplatesArguments(default_disk_template,

                                             enabled_disk_templates):

  """Tests argument handling of 'gnt-cluster modify' with respect to

     the parameter '--enabled-disk-templates'. This test is independent

     of instances.

  """

  _RestoreEnabledDiskTemplates()

  # bogus templates

  AssertCommand(["gnt-cluster", "modify",

                 "--enabled-disk-templates=pinkbunny"],

                fail=True)

  # duplicate entries do no harm

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s,%s" %

      (default_disk_template, default_disk_template)],

    fail=False)

  if constants.DT_DRBD8 in enabled_disk_templates:

    # interaction with --drbd-usermode-helper option

    drbd_usermode_helper = qa_config.get("drbd-usermode-helper", None)

    if not drbd_usermode_helper:

      drbd_usermode_helper = "/bin/true"

    # specifying a helper when drbd gets disabled is ok. Note that drbd still

    # has to be installed on the nodes in this case

    AssertCommand(["gnt-cluster", "modify",

                   "--drbd-usermode-helper=%s" % drbd_usermode_helper,

                   "--enabled-disk-templates=%s" % constants.DT_DISKLESS],

                   fail=False)

    # specifying a helper when drbd is re-enabled

    AssertCommand(["gnt-cluster", "modify",

                   "--drbd-usermode-helper=%s" % drbd_usermode_helper,

                   "--enabled-disk-templates=%s" %

                     ",".join(enabled_disk_templates)],

                  fail=False)

def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates):

  """Tests argument handling of 'gnt-cluster modify' with respect to

     the parameter '--enabled-disk-templates' and '--vg-name'. This test is

     independent of instances.

  """

  if not utils.IsLvmEnabled(enabled_disk_templates):

    # These tests only make sense if lvm is enabled for QA

    return

  # determine an LVM and a non-LVM disk template for the tests

  non_lvm_templates = list(set(enabled_disk_templates)

                           - set(utils.GetLvmDiskTemplates()))

  lvm_template = list(set(enabled_disk_templates)

                      .intersection(set(utils.GetLvmDiskTemplates())))[0]

  non_lvm_template = None

  if non_lvm_templates:

    non_lvm_template = non_lvm_templates[0]

  else:

    # If no non-lvm disk template is available for QA, choose 'diskless' and

    # hope for the best.

    non_lvm_template = constants.ST_DISKLESS

  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)

  # Clean start: unset volume group name, disable lvm storage

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" % non_lvm_template,

     "--vg-name="],

    fail=False)

  # Try to enable lvm, when no volume group is given

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" % lvm_template],

    fail=True)

  # Set volume group, with lvm still disabled: just a warning

  AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False)

  # Try unsetting vg name and enabling lvm at the same time

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" % lvm_template,

     "--vg-name="],

    fail=True)

  # Enable lvm with vg name present

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" % lvm_template],

    fail=False)

  # Try unsetting vg name with lvm still enabled

  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True)

  # Disable lvm with vg name still set

  AssertCommand(

    ["gnt-cluster", "modify", "--enabled-disk-templates=%s" % non_lvm_template],

    fail=False)

  # Try unsetting vg name with lvm disabled

  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False)

  # Set vg name and enable lvm at the same time

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" % lvm_template,

     "--vg-name=%s" % vgname],

    fail=False)

  # Unset vg name and disable lvm at the same time

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" % non_lvm_template,

     "--vg-name="],

    fail=False)

  _RestoreEnabledDiskTemplates()

def _TestClusterModifyUsedDiskTemplate(instance_template,

                                       enabled_disk_templates):

  """Tests that disk templates that are currently in use by instances cannot

     be disabled on the cluster.

  """

  # If the list of enabled disk templates contains only one template

  # we need to add some other templates, because the list of enabled disk

  # templates can only be set to a non-empty list.

  new_disk_templates = list(set(enabled_disk_templates)

                              - set([instance_template]))

  if not new_disk_templates:

    new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK])

                                - set([instance_template]))

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" %

       ",".join(new_disk_templates)],

    fail=True)

def _TestClusterModifyUnusedDiskTemplate(instance_template):

  """Tests that unused disk templates can be disabled safely."""

  all_disk_templates = constants.DISK_TEMPLATES

  if not utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):

    all_disk_templates = list(set(all_disk_templates) -

                              set(utils.GetLvmDiskTemplates()))

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" %

       ",".join(all_disk_templates)],

    fail=False)

  new_disk_templates = [instance_template]

  AssertCommand(

    ["gnt-cluster", "modify",

     "--enabled-disk-templates=%s" %

       ",".join(new_disk_templates)],

    fail=False)

def TestClusterModifyBe():

  """gnt-cluster modify -B"""

  for fail, cmd in [

    # max/min mem

    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),

    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),

    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),

    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),

    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),

    # vcpus

    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),

    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),

    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),

    # auto_balance

    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),

    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),

    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),

    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),

    ]:

    AssertCommand(cmd, fail=fail)

  # redo the original-requested BE parameters, if any

  bep = qa_config.get("backend-parameters", "")

  if bep:

    AssertCommand(["gnt-cluster", "modify", "-B", bep])

def _GetClusterIPolicy():

  """Return the run-time values of the cluster-level instance policy.

  @rtype: tuple

  @return: (policy, specs), where:

      - policy is a dictionary of the policy values, instance specs excluded

      - specs is a dictionary containing only the specs, using the internal

        format (see L{constants.IPOLICY_DEFAULTS} for an example)

  """

  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])

  policy = info["Instance policy - limits for instances"]

  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)

  # Sanity checks

  assert "minmax" in ret_specs and "std" in ret_specs

  assert len(ret_specs["minmax"]) > 0

  assert len(ret_policy) > 0

  return (ret_policy, ret_specs)