4 # Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Cluster related QA tests.
30 from ganeti import constants
31 from ganeti import compat
32 from ganeti import utils
33 from ganeti import pathutils
40 from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
43 # Prefix for LVM volumes created by QA code during tests
46 #: cluster verify command
47 _CLUSTER_VERIFY = ["gnt-cluster", "verify"]
50 def _RemoveFileFromAllNodes(filename):
51 """Removes a file from all nodes.
54 for node in qa_config.get("nodes"):
55 AssertCommand(["rm", "-f", filename], node=node)
58 def _CheckFileOnAllNodes(filename, content):
59 """Verifies the content of the given file on all nodes.
62 cmd = utils.ShellQuoteArgs(["cat", filename])
63 for node in qa_config.get("nodes"):
64 AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
67 def _GetClusterField(field_path):
68 """Get the value of a cluster field.
70 @type field_path: list of strings
71 @param field_path: Names of the groups/fields to navigate to get the desired
72 value, e.g. C{["Default node parameters", "oob_program"]}
73 @return: The effective value of the field (the actual type depends on the
77 assert isinstance(field_path, list)
79 ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
80 for key in field_path:
85 # Cluster-verify errors (date, "ERROR", then error code)
86 _CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
89 def _GetCVErrorCodes(cvout):
92 for l in cvout.splitlines():
93 m = _CVERROR_RE.match(l)
99 elif etype == "WARNING":
104 def _CheckVerifyErrors(actual, expected, etype):
105 exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
106 if not actual.issuperset(exp_codes):
107 missing = exp_codes.difference(actual)
108 raise qa_error.Error("Cluster-verify didn't return these expected"
109 " %ss: %s" % (etype, utils.CommaJoin(missing)))
112 def AssertClusterVerify(fail=False, errors=None, warnings=None):
113 """Run cluster-verify and check the result
116 @param fail: if cluster-verify is expected to fail instead of succeeding
117 @type errors: list of tuples
118 @param errors: List of CV_XXX errors that are expected; if specified, all the
119 errors listed must appear in cluster-verify output. A non-empty value
120 implies C{fail=True}.
121 @type warnings: list of tuples
122 @param warnings: Same as C{errors} but for warnings.
125 cvcmd = "gnt-cluster verify"
126 mnode = qa_config.GetMasterNode()
127 if errors or warnings:
128 cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
129 fail=(fail or errors))
130 (act_errs, act_warns) = _GetCVErrorCodes(cvout)
132 _CheckVerifyErrors(act_errs, errors, "error")
134 _CheckVerifyErrors(act_warns, warnings, "warning")
136 AssertCommand(cvcmd, fail=fail, node=mnode)
139 # data for testing failures due to bad keys/values for disk parameters
140 _FAIL_PARAMS = ["nonexistent:resync-rate=1",
141 "drbd:nonexistent=1",
142 "drbd:resync-rate=invalid",
146 def TestClusterInitDisk():
147 """gnt-cluster init -D"""
148 name = qa_config.get("name")
149 for param in _FAIL_PARAMS:
150 AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
153 def TestClusterInit(rapi_user, rapi_secret):
154 """gnt-cluster init"""
155 master = qa_config.GetMasterNode()
157 rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
158 rapi_dir = os.path.dirname(rapi_users_path)
160 # First create the RAPI credentials
161 fh = tempfile.NamedTemporaryFile()
163 fh.write("%s %s write\n" % (rapi_user, rapi_secret))
166 tmpru = qa_utils.UploadFile(master.primary, fh.name)
168 AssertCommand(["mkdir", "-p", rapi_dir])
169 AssertCommand(["mv", tmpru, rapi_users_path])
171 AssertCommand(["rm", "-f", tmpru])
177 "gnt-cluster", "init",
178 "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
179 "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
180 "--enabled-disk-templates=%s" %
181 ",".join(qa_config.GetEnabledDiskTemplates())
184 for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
186 for spec_val in ("min", "max", "std"):
187 spec = qa_config.get("ispec_%s_%s" %
188 (spec_type.replace("-", "_"), spec_val), None)
190 cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
193 cmd.append("--secondary-ip=%s" % master.secondary)
195 vgname = qa_config.get("vg-name", None)
197 cmd.append("--vg-name=%s" % vgname)
199 master_netdev = qa_config.get("master-netdev", None)
201 cmd.append("--master-netdev=%s" % master_netdev)
203 nicparams = qa_config.get("default-nicparams", None)
205 cmd.append("--nic-parameters=%s" %
206 ",".join(utils.FormatKeyValue(nicparams)))
208 # Cluster value of the exclusive-storage node parameter
209 e_s = qa_config.get("exclusive-storage")
211 cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
214 qa_config.SetExclusiveStorage(e_s)
216 extra_args = qa_config.get("cluster-init-args")
218 cmd.extend(extra_args)
220 cmd.append(qa_config.get("name"))
224 cmd = ["gnt-cluster", "modify"]
226 # hypervisor parameter modifications
227 hvp = qa_config.get("hypervisor-parameters", {})
228 for k, v in hvp.items():
229 cmd.extend(["-H", "%s:%s" % (k, v)])
230 # backend parameter modifications
231 bep = qa_config.get("backend-parameters", "")
233 cmd.extend(["-B", bep])
239 osp = qa_config.get("os-parameters", {})
240 for k, v in osp.items():
241 AssertCommand(["gnt-os", "modify", "-O", v, k])
243 # OS hypervisor parameters
244 os_hvp = qa_config.get("os-hvp", {})
245 for os_name in os_hvp:
246 for hv, hvp in os_hvp[os_name].items():
247 AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
250 def TestClusterRename():
251 """gnt-cluster rename"""
252 cmd = ["gnt-cluster", "rename", "-f"]
254 original_name = qa_config.get("name")
255 rename_target = qa_config.get("rename", None)
256 if rename_target is None:
257 print qa_utils.FormatError('"rename" entry is missing')
261 cmd + [rename_target],
263 cmd + [original_name],
269 def TestClusterOob():
270 """out-of-band framework"""
271 oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
273 AssertCommand(_CLUSTER_VERIFY)
274 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
275 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
278 AssertCommand(_CLUSTER_VERIFY, fail=True)
280 AssertCommand(["touch", oob_path_exists])
281 AssertCommand(["chmod", "0400", oob_path_exists])
282 AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
285 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
286 "oob_program=%s" % oob_path_exists])
288 AssertCommand(_CLUSTER_VERIFY, fail=True)
290 AssertCommand(["chmod", "0500", oob_path_exists])
291 AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
293 AssertCommand(_CLUSTER_VERIFY)
295 AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
297 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
301 def TestClusterEpo():
302 """gnt-cluster epo"""
303 master = qa_config.GetMasterNode()
305 # Assert that OOB is unavailable for all nodes
306 result_output = GetCommandOutput(master.primary,
307 "gnt-node list --verbose --no-headers -o"
309 AssertEqual(compat.all(powered == "(unavail)"
310 for powered in result_output.splitlines()), True)
313 AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
314 # --all doesn't expect arguments
315 AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
317 # Unless --all is given master is not allowed to be in the list
318 AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
320 # This shouldn't fail
321 AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
323 # All instances should have been stopped now
324 result_output = GetCommandOutput(master.primary,
325 "gnt-instance list --no-headers -o status")
326 # ERROR_down because the instance is stopped but not recorded as such
327 AssertEqual(compat.all(status == "ERROR_down"
328 for status in result_output.splitlines()), True)
330 # Now start everything again
331 AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
333 # All instances should have been started now
334 result_output = GetCommandOutput(master.primary,
335 "gnt-instance list --no-headers -o status")
336 AssertEqual(compat.all(status == "running"
337 for status in result_output.splitlines()), True)
340 def TestClusterVerify():
341 """gnt-cluster verify"""
342 AssertCommand(_CLUSTER_VERIFY)
343 AssertCommand(["gnt-cluster", "verify-disks"])
347 """gnt-debug test-jobqueue"""
348 AssertCommand(["gnt-debug", "test-jobqueue"])
352 """gnt-debug delay"""
353 AssertCommand(["gnt-debug", "delay", "1"])
354 AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
355 AssertCommand(["gnt-debug", "delay", "--no-master",
356 "-n", node.primary, "1"])
359 def TestClusterReservedLvs():
360 """gnt-cluster reserved lvs"""
361 vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
362 lvname = _QA_LV_PREFIX + "test"
363 lvfullname = "/".join([vgname, lvname])
365 (False, _CLUSTER_VERIFY),
366 (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
367 (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
368 (True, _CLUSTER_VERIFY),
369 (False, ["gnt-cluster", "modify", "--reserved-lvs",
370 "%s,.*/other-test" % lvfullname]),
371 (False, _CLUSTER_VERIFY),
372 (False, ["gnt-cluster", "modify", "--reserved-lvs",
373 ".*/%s.*" % _QA_LV_PREFIX]),
374 (False, _CLUSTER_VERIFY),
375 (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
376 (True, _CLUSTER_VERIFY),
377 (False, ["lvremove", "-f", lvfullname]),
378 (False, _CLUSTER_VERIFY),
380 AssertCommand(cmd, fail=fail)
383 def TestClusterModifyEmpty():
384 """gnt-cluster modify"""
385 AssertCommand(["gnt-cluster", "modify"], fail=True)
388 def TestClusterModifyDisk():
389 """gnt-cluster modify -D"""
390 for param in _FAIL_PARAMS:
391 AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
394 def TestClusterModifyDiskTemplates():
395 """gnt-cluster modify --enabled-disk-templates=..."""
396 enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
397 default_disk_template = qa_config.GetDefaultDiskTemplate()
399 _TestClusterModifyDiskTemplatesArguments(default_disk_template,
400 enabled_disk_templates)
402 _RestoreEnabledDiskTemplates()
403 nodes = qa_config.AcquireManyNodes(2)
405 instance_template = enabled_disk_templates[0]
406 instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)
408 _TestClusterModifyUnusedDiskTemplate(instance_template)
409 _TestClusterModifyUsedDiskTemplate(instance_template,
410 enabled_disk_templates)
412 qa_instance.TestInstanceRemove(instance)
413 _RestoreEnabledDiskTemplates()
416 def _RestoreEnabledDiskTemplates():
417 """Sets the list of enabled disk templates back to the list of enabled disk
418 templates from the QA configuration. This can be used to make sure that
419 the tests that modify the list of disk templates do not interfere with
424 ["gnt-cluster", "modify",
425 "--enabled-disk-template=%s" %
426 ",".join(qa_config.GetEnabledDiskTemplates())],
430 def _TestClusterModifyDiskTemplatesArguments(default_disk_template,
431 enabled_disk_templates):
432 """Tests argument handling of 'gnt-cluster modify' with respect to
433 the parameter '--enabled-disk-templates'. This test is independent
438 ["gnt-cluster", "modify",
439 "--enabled-disk-template=%s" %
440 ",".join(enabled_disk_templates)],
444 AssertCommand(["gnt-cluster", "modify",
445 "--enabled-disk-templates=pinkbunny"],
448 # duplicate entries do no harm
450 ["gnt-cluster", "modify",
451 "--enabled-disk-templates=%s,%s" %
452 (default_disk_template, default_disk_template)],
456 def _TestClusterModifyUsedDiskTemplate(instance_template,
457 enabled_disk_templates):
458 """Tests that disk templates that are currently in use by instances cannot
459 be disabled on the cluster.
462 # If the list of enabled disk templates contains only one template
463 # we need to add some other templates, because the list of enabled disk
464 # templates can only be set to a non-empty list.
465 new_disk_templates = list(set(enabled_disk_templates)
466 - set([instance_template]))
467 if not new_disk_templates:
468 new_disk_templates = list(set(constants.DISK_TEMPLATES)
469 - set([instance_template]))
471 ["gnt-cluster", "modify",
472 "--enabled-disk-templates=%s" %
473 ",".join(new_disk_templates)],
477 def _TestClusterModifyUnusedDiskTemplate(instance_template):
478 """Tests that unused disk templates can be disabled safely."""
479 all_disk_templates = constants.DISK_TEMPLATES
481 ["gnt-cluster", "modify",
482 "--enabled-disk-templates=%s" %
483 ",".join(all_disk_templates)],
485 new_disk_templates = [instance_template]
487 ["gnt-cluster", "modify",
488 "--enabled-disk-templates=%s" %
489 ",".join(new_disk_templates)],
493 def TestClusterModifyBe():
494 """gnt-cluster modify -B"""
497 (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
498 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
499 (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
500 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
501 (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
502 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
503 (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
504 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
505 (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
506 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
507 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
509 (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
510 (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
511 (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
512 (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
513 (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
515 (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
516 (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
517 (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
518 (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
519 (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
521 AssertCommand(cmd, fail=fail)
523 # redo the original-requested BE parameters, if any
524 bep = qa_config.get("backend-parameters", "")
526 AssertCommand(["gnt-cluster", "modify", "-B", bep])
529 def _GetClusterIPolicy():
530 """Return the run-time values of the cluster-level instance policy.
533 @return: (policy, specs), where:
534 - policy is a dictionary of the policy values, instance specs excluded
535 - specs is a dictionary containing only the specs, using the internal
536 format (see L{constants.IPOLICY_DEFAULTS} for an example)
539 info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
540 policy = info["Instance policy - limits for instances"]
541 (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
544 assert "minmax" in ret_specs and "std" in ret_specs
545 assert len(ret_specs["minmax"]) > 0
546 assert len(ret_policy) > 0
547 return (ret_policy, ret_specs)
550 def TestClusterModifyIPolicy():
551 """gnt-cluster modify --ipolicy-*"""
552 basecmd = ["gnt-cluster", "modify"]
553 (old_policy, old_specs) = _GetClusterIPolicy()
554 for par in ["vcpu-ratio", "spindle-ratio"]:
555 curr_val = float(old_policy[par])
561 # Restore the old value
564 for (good, val) in test_values:
565 cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
566 AssertCommand(cmd, fail=not good)
569 # Check the affected parameter
570 (eff_policy, eff_specs) = _GetClusterIPolicy()
571 AssertEqual(float(eff_policy[par]), curr_val)
572 # Check everything else
573 AssertEqual(eff_specs, old_specs)
574 for p in eff_policy.keys():
577 AssertEqual(eff_policy[p], old_policy[p])
579 # Disk templates are treated slightly differently
580 par = "disk-templates"
581 disp_str = "allowed disk templates"
582 curr_val = old_policy[disp_str]
584 (True, constants.DT_PLAIN),
585 (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
586 (False, "thisisnotadisktemplate"),
588 # Restore the old value
589 (True, curr_val.replace(" ", "")),
591 for (good, val) in test_values:
592 cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
593 AssertCommand(cmd, fail=not good)
596 # Check the affected parameter
597 (eff_policy, eff_specs) = _GetClusterIPolicy()
598 AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
599 # Check everything else
600 AssertEqual(eff_specs, old_specs)
601 for p in eff_policy.keys():
604 AssertEqual(eff_policy[p], old_policy[p])
607 def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False,
609 """Change instance specs.
611 At most one of new_specs or diff_specs can be specified.
613 @type new_specs: dict
614 @param new_specs: new complete specs, in the same format returned by
615 L{_GetClusterIPolicy}
616 @type diff_specs: dict
617 @param diff_specs: partial specs, it can be an incomplete specifications, but
618 if min/max specs are specified, their number must match the number of the
621 @param fail: if the change is expected to fail
622 @type old_values: tuple
623 @param old_values: (old_policy, old_specs), as returned by
624 L{_GetClusterIPolicy}
625 @return: same as L{_GetClusterIPolicy}
628 build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
629 return qa_utils.TestSetISpecs(
630 new_specs=new_specs, diff_specs=diff_specs,
631 get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd,
632 fail=fail, old_values=old_values)
635 def TestClusterModifyISpecs():
636 """gnt-cluster modify --specs-*"""
637 params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
638 (cur_policy, cur_specs) = _GetClusterIPolicy()
639 # This test assumes that there is only one min/max bound
640 assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
655 # This is to restore the old values
657 cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
658 cur_specs[constants.ISPECS_STD][par],
659 cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
661 for (good, mn, st, mx) in test_values:
663 constants.ISPECS_MINMAX: [{
664 constants.ISPECS_MIN: {par: mn},
665 constants.ISPECS_MAX: {par: mx}
667 constants.ISPECS_STD: {par: st}
669 cur_state = (cur_policy, cur_specs)
670 # We update cur_specs, as we've copied the values to restore already
671 (cur_policy, cur_specs) = TestClusterSetISpecs(
672 diff_specs=new_vals, fail=not good, old_values=cur_state)
674 # Get the ipolicy command
675 mnode = qa_config.GetMasterNode()
676 initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
677 modcmd = ["gnt-cluster", "modify"]
678 opts = initcmd.split()
679 assert opts[0:2] == ["gnt-cluster", "init"]
680 for k in range(2, len(opts) - 1):
681 if opts[k].startswith("--ipolicy-"):
682 assert k + 2 <= len(opts)
683 modcmd.extend(opts[k:k + 2])
684 # Re-apply the ipolicy (this should be a no-op)
685 AssertCommand(modcmd)
686 new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
687 AssertEqual(initcmd, new_initcmd)
690 def TestClusterInfo():
691 """gnt-cluster info"""
692 AssertCommand(["gnt-cluster", "info"])
695 def TestClusterRedistConf():
696 """gnt-cluster redist-conf"""
697 AssertCommand(["gnt-cluster", "redist-conf"])
700 def TestClusterGetmaster():
701 """gnt-cluster getmaster"""
702 AssertCommand(["gnt-cluster", "getmaster"])
705 def TestClusterVersion():
706 """gnt-cluster version"""
707 AssertCommand(["gnt-cluster", "version"])
710 def TestClusterRenewCrypto():
711 """gnt-cluster renew-crypto"""
712 master = qa_config.GetMasterNode()
714 # Conflicting options
715 cmd = ["gnt-cluster", "renew-crypto", "--force",
716 "--new-cluster-certificate", "--new-confd-hmac-key"]
718 ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
719 ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
721 for i in conflicting:
722 AssertCommand(cmd + i, fail=True)
724 # Invalid RAPI certificate
725 cmd = ["gnt-cluster", "renew-crypto", "--force",
726 "--rapi-certificate=/dev/null"]
727 AssertCommand(cmd, fail=True)
729 rapi_cert_backup = qa_utils.BackupFile(master.primary,
730 pathutils.RAPI_CERT_FILE)
732 # Custom RAPI certificate
733 fh = tempfile.NamedTemporaryFile()
735 # Ensure certificate doesn't cause "gnt-cluster verify" to complain
736 validity = constants.SSL_CERT_EXPIRATION_WARN * 3
738 utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
740 tmpcert = qa_utils.UploadFile(master.primary, fh.name)
742 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
743 "--rapi-certificate=%s" % tmpcert])
745 AssertCommand(["rm", "-f", tmpcert])
747 # Custom cluster domain secret
748 cds_fh = tempfile.NamedTemporaryFile()
749 cds_fh.write(utils.GenerateSecret())
753 tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
755 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
756 "--cluster-domain-secret=%s" % tmpcds])
758 AssertCommand(["rm", "-f", tmpcds])
761 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
762 "--new-cluster-certificate", "--new-confd-hmac-key",
763 "--new-rapi-certificate", "--new-cluster-domain-secret"])
765 # Restore RAPI certificate
766 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
767 "--rapi-certificate=%s" % rapi_cert_backup])
769 AssertCommand(["rm", "-f", rapi_cert_backup])
772 def TestClusterBurnin():
774 master = qa_config.GetMasterNode()
776 options = qa_config.get("options", {})
777 disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
778 parallel = options.get("burnin-in-parallel", False)
779 check_inst = options.get("burnin-check-instances", False)
780 do_rename = options.get("burnin-rename", "")
781 do_reboot = options.get("burnin-reboot", True)
782 reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
784 # Get as many instances as we need
788 num = qa_config.get("options", {}).get("burnin-instances", 1)
789 for _ in range(0, num):
790 instances.append(qa_config.AcquireInstance())
791 except qa_error.OutOfInstancesError:
792 print "Not enough instances, continuing anyway."
794 if len(instances) < 1:
795 raise qa_error.Error("Burnin needs at least one instance")
797 script = qa_utils.UploadFile(master.primary, "../tools/burnin")
799 disks = qa_config.GetDiskOptions()
802 "--os=%s" % qa_config.get("os"),
803 "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
804 "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
805 "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
806 "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
807 "--disk-template=%s" % disk_template]
809 cmd.append("--parallel")
810 cmd.append("--early-release")
812 cmd.append("--http-check")
814 cmd.append("--rename=%s" % do_rename)
816 cmd.append("--no-reboot")
818 cmd.append("--reboot-types=%s" % ",".join(reboot_types))
819 cmd += [inst.name for inst in instances]
822 AssertCommand(["rm", "-f", script])
825 for inst in instances:
829 def TestClusterMasterFailover():
830 """gnt-cluster master-failover"""
831 master = qa_config.GetMasterNode()
832 failovermaster = qa_config.AcquireNode(exclude=master)
834 cmd = ["gnt-cluster", "master-failover"]
836 AssertCommand(cmd, node=failovermaster)
837 # Back to original master node
838 AssertCommand(cmd, node=master)
840 failovermaster.Release()
843 def _NodeQueueDrainFile(node):
844 """Returns path to queue drain file for a node.
847 return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE)
850 def _AssertDrainFile(node, **kwargs):
851 """Checks for the queue drain file.
854 AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs)
857 def TestClusterMasterFailoverWithDrainedQueue():
858 """gnt-cluster master-failover with drained queue"""
859 master = qa_config.GetMasterNode()
860 failovermaster = qa_config.AcquireNode(exclude=master)
862 # Ensure queue is not drained
863 for node in [master, failovermaster]:
864 _AssertDrainFile(node, fail=True)
866 # Drain queue on failover master
867 AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)],
870 cmd = ["gnt-cluster", "master-failover"]
872 _AssertDrainFile(failovermaster)
873 AssertCommand(cmd, node=failovermaster)
874 _AssertDrainFile(master, fail=True)
875 _AssertDrainFile(failovermaster, fail=True)
877 # Back to original master node
878 AssertCommand(cmd, node=master)
880 failovermaster.Release()
882 # Ensure queue is not drained
883 for node in [master, failovermaster]:
884 _AssertDrainFile(node, fail=True)
887 def TestClusterCopyfile():
888 """gnt-cluster copyfile"""
889 master = qa_config.GetMasterNode()
891 uniqueid = utils.NewUUID()
893 # Create temporary file
894 f = tempfile.NamedTemporaryFile()
899 # Upload file to master node
900 testname = qa_utils.UploadFile(master.primary, f.name)
902 # Copy file to all nodes
903 AssertCommand(["gnt-cluster", "copyfile", testname])
904 _CheckFileOnAllNodes(testname, uniqueid)
906 _RemoveFileFromAllNodes(testname)
909 def TestClusterCommand():
910 """gnt-cluster command"""
911 uniqueid = utils.NewUUID()
912 rfile = "/tmp/gnt%s" % utils.NewUUID()
913 rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
914 cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
915 "%s >%s" % (rcmd, rfile)])
919 _CheckFileOnAllNodes(rfile, uniqueid)
921 _RemoveFileFromAllNodes(rfile)
924 def TestClusterDestroy():
925 """gnt-cluster destroy"""
926 AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
929 def TestClusterRepairDiskSizes():
930 """gnt-cluster repair-disk-sizes"""
931 AssertCommand(["gnt-cluster", "repair-disk-sizes"])
934 def TestSetExclStorCluster(newvalue):
935 """Set the exclusive_storage node parameter at the cluster level.
938 @param newvalue: New value of exclusive_storage
940 @return: The old value of exclusive_storage
943 es_path = ["Default node parameters", "exclusive_storage"]
944 oldvalue = _GetClusterField(es_path)
945 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
946 "exclusive_storage=%s" % newvalue])
947 effvalue = _GetClusterField(es_path)
948 if effvalue != newvalue:
949 raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
950 " of %s" % (effvalue, newvalue))
951 qa_config.SetExclusiveStorage(newvalue)
955 def TestExclStorSharedPv(node):
956 """cluster-verify reports LVs that share the same PV with exclusive_storage.
959 vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
960 lvname1 = _QA_LV_PREFIX + "vol1"
961 lvname2 = _QA_LV_PREFIX + "vol2"
962 node_name = node.primary
963 AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
964 AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
965 AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
966 AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
967 constants.CV_ENODEORPHANLV])
968 AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
969 AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
970 AssertClusterVerify()