X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/b5f33afadcf4f132d096420ece528aae9b486b99..9c265dd7fc7fd55a844d71bbc5483682b56a3b91:/qa/qa_cluster.py diff --git a/qa/qa_cluster.py b/qa/qa_cluster.py index 410196b..f532826 100644 --- a/qa/qa_cluster.py +++ b/qa/qa_cluster.py @@ -35,6 +35,7 @@ from ganeti import pathutils import qa_config import qa_utils import qa_error +import qa_instance from qa_utils import AssertEqual, AssertCommand, GetCommandOutput @@ -63,53 +64,52 @@ def _CheckFileOnAllNodes(filename, content): AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content) -# "gnt-cluster info" fields -_CIFIELD_RE = re.compile(r"^[-\s]*(?P[^\s:]+):\s*(?P\S.*)$") +def _GetClusterField(field_path): + """Get the value of a cluster field. - -def _GetBoolClusterField(field): - """Get the Boolean value of a cluster field. - - This function currently assumes that the field name is unique in the cluster - configuration. An assertion checks this assumption. - - @type field: string - @param field: Name of the field - @rtype: bool - @return: The effective value of the field + @type field_path: list of strings + @param field_path: Names of the groups/fields to navigate to get the desired + value, e.g. C{["Default node parameters", "oob_program"]} + @return: The effective value of the field (the actual type depends on the + chosen field) """ - master = qa_config.GetMasterNode() - infocmd = "gnt-cluster info" - info_out = qa_utils.GetCommandOutput(master.primary, infocmd) - ret = None - for l in info_out.splitlines(): - m = _CIFIELD_RE.match(l) - # FIXME: There should be a way to specify a field through a hierarchy - if m and m.group("field") == field: - # Make sure that ignoring the hierarchy doesn't cause a double match - assert ret is None - ret = (m.group("value").lower() == "true") - if ret is not None: - return ret - raise qa_error.Error("Field not found in cluster configuration: %s" % field) + assert isinstance(field_path, list) + assert field_path + ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"]) + for key in field_path: + ret = ret[key] + return ret # Cluster-verify errors (date, "ERROR", then error code) -_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- ERROR:([A-Z0-9_-]+):") +_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):") def _GetCVErrorCodes(cvout): - ret = set() + errs = set() + warns = set() for l in cvout.splitlines(): m = _CVERROR_RE.match(l) if m: - ecode = m.group(1) - ret.add(ecode) - return ret + etype = m.group(1) + ecode = m.group(2) + if etype == "ERROR": + errs.add(ecode) + elif etype == "WARNING": + warns.add(ecode) + return (errs, warns) -def AssertClusterVerify(fail=False, errors=None): +def _CheckVerifyErrors(actual, expected, etype): + exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected) + if not actual.issuperset(exp_codes): + missing = exp_codes.difference(actual) + raise qa_error.Error("Cluster-verify didn't return these expected" + " %ss: %s" % (etype, utils.CommaJoin(missing))) + + +def AssertClusterVerify(fail=False, errors=None, warnings=None): """Run cluster-verify and check the result @type fail: bool @@ -118,19 +118,20 @@ def AssertClusterVerify(fail=False, errors=None): @param errors: List of CV_XXX errors that are expected; if specified, all the errors listed must appear in cluster-verify output. A non-empty value implies C{fail=True}. + @type warnings: list of tuples + @param warnings: Same as C{errors} but for warnings. """ cvcmd = "gnt-cluster verify" mnode = qa_config.GetMasterNode() - if errors: + if errors or warnings: cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes", - fail=True) - actual = _GetCVErrorCodes(cvout) - expected = compat.UniqueFrozenset(e for (_, e, _) in errors) - if not actual.issuperset(expected): - missing = expected.difference(actual) - raise qa_error.Error("Cluster-verify didn't return these expected" - " errors: %s" % utils.CommaJoin(missing)) + fail=(fail or errors)) + (act_errs, act_warns) = _GetCVErrorCodes(cvout) + if errors: + _CheckVerifyErrors(act_errs, errors, "error") + if warnings: + _CheckVerifyErrors(act_warns, warnings, "warning") else: AssertCommand(cvcmd, fail=fail, node=mnode) @@ -153,7 +154,8 @@ def TestClusterInit(rapi_user, rapi_secret): """gnt-cluster init""" master = qa_config.GetMasterNode() - rapi_dir = os.path.dirname(pathutils.RAPI_USERS_FILE) + rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE) + rapi_dir = os.path.dirname(rapi_users_path) # First create the RAPI credentials fh = tempfile.NamedTemporaryFile() @@ -164,7 +166,7 @@ def TestClusterInit(rapi_user, rapi_secret): tmpru = qa_utils.UploadFile(master.primary, fh.name) try: AssertCommand(["mkdir", "-p", rapi_dir]) - AssertCommand(["mv", tmpru, pathutils.RAPI_USERS_FILE]) + AssertCommand(["mv", tmpru, rapi_users_path]) finally: AssertCommand(["rm", "-f", tmpru]) finally: @@ -175,6 +177,8 @@ def TestClusterInit(rapi_user, rapi_secret): "gnt-cluster", "init", "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4), "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()), + "--enabled-disk-templates=%s" % + ",".join(qa_config.GetEnabledDiskTemplates()) ] for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count", @@ -182,7 +186,7 @@ def TestClusterInit(rapi_user, rapi_secret): for spec_val in ("min", "max", "std"): spec = qa_config.get("ispec_%s_%s" % (spec_type.replace("-", "_"), spec_val), None) - if spec: + if spec is not None: cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec)) if master.secondary: @@ -209,7 +213,12 @@ def TestClusterInit(rapi_user, rapi_secret): e_s = False qa_config.SetExclusiveStorage(e_s) + extra_args = qa_config.get("cluster-init-args") + if extra_args: + cmd.extend(extra_args) + cmd.append(qa_config.get("name")) + AssertCommand(cmd) cmd = ["gnt-cluster", "modify"] @@ -382,6 +391,105 @@ def TestClusterModifyDisk(): AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True) +def TestClusterModifyDiskTemplates(): + """gnt-cluster modify --enabled-disk-templates=...""" + enabled_disk_templates = qa_config.GetEnabledDiskTemplates() + default_disk_template = qa_config.GetDefaultDiskTemplate() + + _TestClusterModifyDiskTemplatesArguments(default_disk_template, + enabled_disk_templates) + + _RestoreEnabledDiskTemplates() + nodes = qa_config.AcquireManyNodes(2) + + instance_template = enabled_disk_templates[0] + instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template) + + _TestClusterModifyUnusedDiskTemplate(instance_template) + _TestClusterModifyUsedDiskTemplate(instance_template, + enabled_disk_templates) + + qa_instance.TestInstanceRemove(instance) + _RestoreEnabledDiskTemplates() + + +def _RestoreEnabledDiskTemplates(): + """Sets the list of enabled disk templates back to the list of enabled disk + templates from the QA configuration. This can be used to make sure that + the tests that modify the list of disk templates do not interfere with + other tests. + + """ + AssertCommand( + ["gnt-cluster", "modify", + "--enabled-disk-template=%s" % + ",".join(qa_config.GetEnabledDiskTemplates())], + fail=False) + + +def _TestClusterModifyDiskTemplatesArguments(default_disk_template, + enabled_disk_templates): + """Tests argument handling of 'gnt-cluster modify' with respect to + the parameter '--enabled-disk-templates'. This test is independent + of instances. + + """ + AssertCommand( + ["gnt-cluster", "modify", + "--enabled-disk-template=%s" % + ",".join(enabled_disk_templates)], + fail=False) + + # bogus templates + AssertCommand(["gnt-cluster", "modify", + "--enabled-disk-templates=pinkbunny"], + fail=True) + + # duplicate entries do no harm + AssertCommand( + ["gnt-cluster", "modify", + "--enabled-disk-templates=%s,%s" % + (default_disk_template, default_disk_template)], + fail=False) + + +def _TestClusterModifyUsedDiskTemplate(instance_template, + enabled_disk_templates): + """Tests that disk templates that are currently in use by instances cannot + be disabled on the cluster. + + """ + # If the list of enabled disk templates contains only one template + # we need to add some other templates, because the list of enabled disk + # templates can only be set to a non-empty list. + new_disk_templates = list(set(enabled_disk_templates) + - set([instance_template])) + if not new_disk_templates: + new_disk_templates = list(set(constants.DISK_TEMPLATES) + - set([instance_template])) + AssertCommand( + ["gnt-cluster", "modify", + "--enabled-disk-templates=%s" % + ",".join(new_disk_templates)], + fail=True) + + +def _TestClusterModifyUnusedDiskTemplate(instance_template): + """Tests that unused disk templates can be disabled safely.""" + all_disk_templates = constants.DISK_TEMPLATES + AssertCommand( + ["gnt-cluster", "modify", + "--enabled-disk-templates=%s" % + ",".join(all_disk_templates)], + fail=False) + new_disk_templates = [instance_template] + AssertCommand( + ["gnt-cluster", "modify", + "--enabled-disk-templates=%s" % + ",".join(new_disk_templates)], + fail=False) + + def TestClusterModifyBe(): """gnt-cluster modify -B""" for fail, cmd in [ @@ -418,6 +526,151 @@ def TestClusterModifyBe(): AssertCommand(["gnt-cluster", "modify", "-B", bep]) +def _GetClusterIPolicy(): + """Return the run-time values of the cluster-level instance policy. + + @rtype: tuple + @return: (policy, specs), where: + - policy is a dictionary of the policy values, instance specs excluded + - specs is dict of dict, specs[par][key] is a spec value, where key is + "min", "max", or "std" + + """ + info = qa_utils.GetObjectInfo(["gnt-cluster", "info"]) + policy = info["Instance policy - limits for instances"] + (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy) + + # Sanity checks + assert len(ret_specs) > 0 + good = all("min" in d and "std" in d and "max" in d + for d in ret_specs.values()) + assert good, "Missing item in specs: %s" % ret_specs + assert len(ret_policy) > 0 + return (ret_policy, ret_specs) + + +def TestClusterModifyIPolicy(): + """gnt-cluster modify --ipolicy-*""" + basecmd = ["gnt-cluster", "modify"] + (old_policy, old_specs) = _GetClusterIPolicy() + for par in ["vcpu-ratio", "spindle-ratio"]: + curr_val = float(old_policy[par]) + test_values = [ + (True, 1.0), + (True, 1.5), + (True, 2), + (False, "a"), + # Restore the old value + (True, curr_val), + ] + for (good, val) in test_values: + cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)] + AssertCommand(cmd, fail=not good) + if good: + curr_val = val + # Check the affected parameter + (eff_policy, eff_specs) = _GetClusterIPolicy() + AssertEqual(float(eff_policy[par]), curr_val) + # Check everything else + AssertEqual(eff_specs, old_specs) + for p in eff_policy.keys(): + if p == par: + continue + AssertEqual(eff_policy[p], old_policy[p]) + + # Disk templates are treated slightly differently + par = "disk-templates" + disp_str = "enabled disk templates" + curr_val = old_policy[disp_str] + test_values = [ + (True, constants.DT_PLAIN), + (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)), + (False, "thisisnotadisktemplate"), + (False, ""), + # Restore the old value + (True, curr_val.replace(" ", "")), + ] + for (good, val) in test_values: + cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)] + AssertCommand(cmd, fail=not good) + if good: + curr_val = val + # Check the affected parameter + (eff_policy, eff_specs) = _GetClusterIPolicy() + AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val) + # Check everything else + AssertEqual(eff_specs, old_specs) + for p in eff_policy.keys(): + if p == disp_str: + continue + AssertEqual(eff_policy[p], old_policy[p]) + + +def TestClusterSetISpecs(new_specs, fail=False, old_values=None): + """Change instance specs. + + @type new_specs: dict of dict + @param new_specs: new_specs[par][key], where key is "min", "max", "std". It + can be an empty dictionary. + @type fail: bool + @param fail: if the change is expected to fail + @type old_values: tuple + @param old_values: (old_policy, old_specs), as returned by + L{_GetClusterIPolicy} + @return: same as L{_GetClusterIPolicy} + + """ + build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts + return qa_utils.TestSetISpecs(new_specs, get_policy_fn=_GetClusterIPolicy, + build_cmd_fn=build_cmd, fail=fail, + old_values=old_values) + + +def TestClusterModifyISpecs(): + """gnt-cluster modify --specs-*""" + params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"] + (cur_policy, cur_specs) = _GetClusterIPolicy() + for par in params: + test_values = [ + (True, 0, 4, 12), + (True, 4, 4, 12), + (True, 4, 12, 12), + (True, 4, 4, 4), + (False, 4, 0, 12), + (False, 4, 16, 12), + (False, 4, 4, 0), + (False, 12, 4, 4), + (False, 12, 4, 0), + (False, "a", 4, 12), + (False, 0, "a", 12), + (False, 0, 4, "a"), + # This is to restore the old values + (True, + cur_specs[par]["min"], cur_specs[par]["std"], cur_specs[par]["max"]) + ] + for (good, mn, st, mx) in test_values: + new_vals = {par: {"min": str(mn), "std": str(st), "max": str(mx)}} + cur_state = (cur_policy, cur_specs) + # We update cur_specs, as we've copied the values to restore already + (cur_policy, cur_specs) = TestClusterSetISpecs(new_vals, fail=not good, + old_values=cur_state) + + # Get the ipolicy command + mnode = qa_config.GetMasterNode() + initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd") + modcmd = ["gnt-cluster", "modify"] + opts = initcmd.split() + assert opts[0:2] == ["gnt-cluster", "init"] + for k in range(2, len(opts) - 1): + if opts[k].startswith("--ipolicy-"): + assert k + 2 <= len(opts) + modcmd.extend(opts[k:k + 2]) + # Re-apply the ipolicy (this should be a no-op) + AssertCommand(modcmd) + new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd") + AssertEqual(initcmd, new_initcmd) + + def TestClusterInfo(): """gnt-cluster info""" AssertCommand(["gnt-cluster", "info"]) @@ -505,7 +758,7 @@ def TestClusterBurnin(): master = qa_config.GetMasterNode() options = qa_config.get("options", {}) - disk_template = options.get("burnin-disk-template", "drbd") + disk_template = options.get("burnin-disk-template", constants.DT_DRBD8) parallel = options.get("burnin-in-parallel", False) check_inst = options.get("burnin-check-instances", False) do_rename = options.get("burnin-rename", "") @@ -527,13 +780,14 @@ def TestClusterBurnin(): script = qa_utils.UploadFile(master.primary, "../tools/burnin") try: + disks = qa_config.GetDiskOptions() # Run burnin cmd = [script, "--os=%s" % qa_config.get("os"), "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM), "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM), - "--disk-size=%s" % ",".join(qa_config.get("disk")), - "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")), + "--disk-size=%s" % ",".join([d.get("size") for d in disks]), + "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]), "--disk-template=%s" % disk_template] if parallel: cmd.append("--parallel") @@ -570,34 +824,48 @@ def TestClusterMasterFailover(): failovermaster.Release() +def _NodeQueueDrainFile(node): + """Returns path to queue drain file for a node. + + """ + return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE) + + +def _AssertDrainFile(node, **kwargs): + """Checks for the queue drain file. + + """ + AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs) + + def TestClusterMasterFailoverWithDrainedQueue(): """gnt-cluster master-failover with drained queue""" - drain_check = ["test", "-f", pathutils.JOB_QUEUE_DRAIN_FILE] - master = qa_config.GetMasterNode() failovermaster = qa_config.AcquireNode(exclude=master) # Ensure queue is not drained for node in [master, failovermaster]: - AssertCommand(drain_check, node=node, fail=True) + _AssertDrainFile(node, fail=True) # Drain queue on failover master - AssertCommand(["touch", pathutils.JOB_QUEUE_DRAIN_FILE], node=failovermaster) + AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)], + node=failovermaster) cmd = ["gnt-cluster", "master-failover"] try: - AssertCommand(drain_check, node=failovermaster) + _AssertDrainFile(failovermaster) AssertCommand(cmd, node=failovermaster) - AssertCommand(drain_check, fail=True) - AssertCommand(drain_check, node=failovermaster, fail=True) + _AssertDrainFile(master, fail=True) + _AssertDrainFile(failovermaster, fail=True) # Back to original master node AssertCommand(cmd, node=master) finally: failovermaster.Release() - AssertCommand(drain_check, fail=True) - AssertCommand(drain_check, node=failovermaster, fail=True) + # Ensure queue is not drained + for node in [master, failovermaster]: + _AssertDrainFile(node, fail=True) def TestClusterCopyfile(): @@ -656,10 +924,11 @@ def TestSetExclStorCluster(newvalue): @return: The old value of exclusive_storage """ - oldvalue = _GetBoolClusterField("exclusive_storage") + es_path = ["Default node parameters", "exclusive_storage"] + oldvalue = _GetClusterField(es_path) AssertCommand(["gnt-cluster", "modify", "--node-parameters", "exclusive_storage=%s" % newvalue]) - effvalue = _GetBoolClusterField("exclusive_storage") + effvalue = _GetClusterField(es_path) if effvalue != newvalue: raise qa_error.Error("exclusive_storage has the wrong value: %s instead" " of %s" % (effvalue, newvalue)) @@ -667,24 +936,6 @@ def TestSetExclStorCluster(newvalue): return oldvalue -def _BuildSetESCmd(value, node_name): - return ["gnt-node", "modify", "--node-parameters", - "exclusive_storage=%s" % value, node_name] - - -def TestExclStorSingleNode(node): - """cluster-verify reports exclusive_storage set only on one node. - - """ - node_name = node.primary - es_val = _GetBoolClusterField("exclusive_storage") - assert not es_val - AssertCommand(_BuildSetESCmd(True, node_name)) - AssertClusterVerify(fail=True, errors=[constants.CV_EGROUPMIXEDESFLAG]) - AssertCommand(_BuildSetESCmd("default", node_name)) - AssertClusterVerify() - - def TestExclStorSharedPv(node): """cluster-verify reports LVs that share the same PV with exclusive_storage.