4 # Copyright (C) 2007, 2010, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Cluster related QA tests.
29 from ganeti import constants
30 from ganeti import compat
31 from ganeti import utils
32 from ganeti import pathutils
38 from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
41 #: cluster verify command
42 _CLUSTER_VERIFY = ["gnt-cluster", "verify"]
45 def _RemoveFileFromAllNodes(filename):
46 """Removes a file from all nodes.
49 for node in qa_config.get("nodes"):
50 AssertCommand(["rm", "-f", filename], node=node)
53 def _CheckFileOnAllNodes(filename, content):
54 """Verifies the content of the given file on all nodes.
57 cmd = utils.ShellQuoteArgs(["cat", filename])
58 for node in qa_config.get("nodes"):
59 AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content)
62 # data for testing failures due to bad keys/values for disk parameters
63 _FAIL_PARAMS = ["nonexistent:resync-rate=1",
65 "drbd:resync-rate=invalid",
69 def TestClusterInitDisk():
70 """gnt-cluster init -D"""
71 name = qa_config.get("name")
72 for param in _FAIL_PARAMS:
73 AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
76 def TestClusterInit(rapi_user, rapi_secret):
77 """gnt-cluster init"""
78 master = qa_config.GetMasterNode()
80 rapi_dir = os.path.dirname(pathutils.RAPI_USERS_FILE)
82 # First create the RAPI credentials
83 fh = tempfile.NamedTemporaryFile()
85 fh.write("%s %s write\n" % (rapi_user, rapi_secret))
88 tmpru = qa_utils.UploadFile(master["primary"], fh.name)
90 AssertCommand(["mkdir", "-p", rapi_dir])
91 AssertCommand(["mv", tmpru, pathutils.RAPI_USERS_FILE])
93 AssertCommand(["rm", "-f", tmpru])
99 "gnt-cluster", "init",
100 "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
101 "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
104 for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
106 for spec_val in ("min", "max", "std"):
107 spec = qa_config.get("ispec_%s_%s" %
108 (spec_type.replace('-', '_'), spec_val), None)
110 cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
112 if master.get("secondary", None):
113 cmd.append("--secondary-ip=%s" % master["secondary"])
115 bridge = qa_config.get("bridge", None)
117 cmd.append("--bridge=%s" % bridge)
118 cmd.append("--master-netdev=%s" % bridge)
120 cmd.append(qa_config.get("name"))
123 cmd = ["gnt-cluster", "modify"]
125 # hypervisor parameter modifications
126 hvp = qa_config.get("hypervisor-parameters", {})
127 for k, v in hvp.items():
128 cmd.extend(["-H", "%s:%s" % (k, v)])
129 # backend parameter modifications
130 bep = qa_config.get("backend-parameters", "")
132 cmd.extend(["-B", bep])
138 osp = qa_config.get("os-parameters", {})
139 for k, v in osp.items():
140 AssertCommand(["gnt-os", "modify", "-O", v, k])
142 # OS hypervisor parameters
143 os_hvp = qa_config.get("os-hvp", {})
144 for os_name in os_hvp:
145 for hv, hvp in os_hvp[os_name].items():
146 AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
149 def TestClusterRename():
150 """gnt-cluster rename"""
151 cmd = ["gnt-cluster", "rename", "-f"]
153 original_name = qa_config.get("name")
154 rename_target = qa_config.get("rename", None)
155 if rename_target is None:
156 print qa_utils.FormatError('"rename" entry is missing')
160 cmd + [rename_target],
162 cmd + [original_name],
168 def TestClusterOob():
169 """out-of-band framework"""
170 oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
172 AssertCommand(_CLUSTER_VERIFY)
173 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
174 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
177 AssertCommand(_CLUSTER_VERIFY, fail=True)
179 AssertCommand(["touch", oob_path_exists])
180 AssertCommand(["chmod", "0400", oob_path_exists])
181 AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
184 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
185 "oob_program=%s" % oob_path_exists])
187 AssertCommand(_CLUSTER_VERIFY, fail=True)
189 AssertCommand(["chmod", "0500", oob_path_exists])
190 AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
192 AssertCommand(_CLUSTER_VERIFY)
194 AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
196 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
200 def TestClusterEpo():
201 """gnt-cluster epo"""
202 master = qa_config.GetMasterNode()
204 # Assert that OOB is unavailable for all nodes
205 result_output = GetCommandOutput(master["primary"],
206 "gnt-node list --verbose --no-headers -o"
208 AssertEqual(compat.all(powered == "(unavail)"
209 for powered in result_output.splitlines()), True)
212 AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
213 # --all doesn't expect arguments
214 AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
216 # Unless --all is given master is not allowed to be in the list
217 AssertCommand(["gnt-cluster", "epo", "-f", master["primary"]], fail=True)
219 # This shouldn't fail
220 AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
222 # All instances should have been stopped now
223 result_output = GetCommandOutput(master["primary"],
224 "gnt-instance list --no-headers -o status")
225 # ERROR_down because the instance is stopped but not recorded as such
226 AssertEqual(compat.all(status == "ERROR_down"
227 for status in result_output.splitlines()), True)
229 # Now start everything again
230 AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
232 # All instances should have been started now
233 result_output = GetCommandOutput(master["primary"],
234 "gnt-instance list --no-headers -o status")
235 AssertEqual(compat.all(status == "running"
236 for status in result_output.splitlines()), True)
239 def TestClusterVerify():
240 """gnt-cluster verify"""
241 AssertCommand(_CLUSTER_VERIFY)
242 AssertCommand(["gnt-cluster", "verify-disks"])
246 """gnt-debug test-jobqueue"""
247 AssertCommand(["gnt-debug", "test-jobqueue"])
251 """gnt-debug delay"""
252 AssertCommand(["gnt-debug", "delay", "1"])
253 AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
254 AssertCommand(["gnt-debug", "delay", "--no-master",
255 "-n", node["primary"], "1"])
258 def TestClusterReservedLvs():
259 """gnt-cluster reserved lvs"""
261 (False, _CLUSTER_VERIFY),
262 (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
263 (False, ["lvcreate", "-L1G", "-nqa-test", "xenvg"]),
264 (True, _CLUSTER_VERIFY),
265 (False, ["gnt-cluster", "modify", "--reserved-lvs",
266 "xenvg/qa-test,.*/other-test"]),
267 (False, _CLUSTER_VERIFY),
268 (False, ["gnt-cluster", "modify", "--reserved-lvs", ".*/qa-.*"]),
269 (False, _CLUSTER_VERIFY),
270 (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
271 (True, _CLUSTER_VERIFY),
272 (False, ["lvremove", "-f", "xenvg/qa-test"]),
273 (False, _CLUSTER_VERIFY),
275 AssertCommand(cmd, fail=fail)
278 def TestClusterModifyEmpty():
279 """gnt-cluster modify"""
280 AssertCommand(["gnt-cluster", "modify"], fail=True)
283 def TestClusterModifyDisk():
284 """gnt-cluster modify -D"""
285 for param in _FAIL_PARAMS:
286 AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
289 def TestClusterModifyBe():
290 """gnt-cluster modify -B"""
293 (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
294 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
295 (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
296 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
297 (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
298 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
299 (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
300 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
301 (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
302 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
303 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
305 (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
306 (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
307 (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
308 (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
309 (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
311 (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
312 (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
313 (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
314 (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
315 (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
317 AssertCommand(cmd, fail=fail)
319 # redo the original-requested BE parameters, if any
320 bep = qa_config.get("backend-parameters", "")
322 AssertCommand(["gnt-cluster", "modify", "-B", bep])
325 def TestClusterInfo():
326 """gnt-cluster info"""
327 AssertCommand(["gnt-cluster", "info"])
330 def TestClusterRedistConf():
331 """gnt-cluster redist-conf"""
332 AssertCommand(["gnt-cluster", "redist-conf"])
335 def TestClusterGetmaster():
336 """gnt-cluster getmaster"""
337 AssertCommand(["gnt-cluster", "getmaster"])
340 def TestClusterVersion():
341 """gnt-cluster version"""
342 AssertCommand(["gnt-cluster", "version"])
345 def TestClusterRenewCrypto():
346 """gnt-cluster renew-crypto"""
347 master = qa_config.GetMasterNode()
349 # Conflicting options
350 cmd = ["gnt-cluster", "renew-crypto", "--force",
351 "--new-cluster-certificate", "--new-confd-hmac-key"]
353 ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
354 ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
356 for i in conflicting:
357 AssertCommand(cmd + i, fail=True)
359 # Invalid RAPI certificate
360 cmd = ["gnt-cluster", "renew-crypto", "--force",
361 "--rapi-certificate=/dev/null"]
362 AssertCommand(cmd, fail=True)
364 rapi_cert_backup = qa_utils.BackupFile(master["primary"],
365 pathutils.RAPI_CERT_FILE)
367 # Custom RAPI certificate
368 fh = tempfile.NamedTemporaryFile()
370 # Ensure certificate doesn't cause "gnt-cluster verify" to complain
371 validity = constants.SSL_CERT_EXPIRATION_WARN * 3
373 utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
375 tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
377 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
378 "--rapi-certificate=%s" % tmpcert])
380 AssertCommand(["rm", "-f", tmpcert])
382 # Custom cluster domain secret
383 cds_fh = tempfile.NamedTemporaryFile()
384 cds_fh.write(utils.GenerateSecret())
388 tmpcds = qa_utils.UploadFile(master["primary"], cds_fh.name)
390 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
391 "--cluster-domain-secret=%s" % tmpcds])
393 AssertCommand(["rm", "-f", tmpcds])
396 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
397 "--new-cluster-certificate", "--new-confd-hmac-key",
398 "--new-rapi-certificate", "--new-cluster-domain-secret"])
400 # Restore RAPI certificate
401 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
402 "--rapi-certificate=%s" % rapi_cert_backup])
404 AssertCommand(["rm", "-f", rapi_cert_backup])
407 def TestClusterBurnin():
409 master = qa_config.GetMasterNode()
411 options = qa_config.get("options", {})
412 disk_template = options.get("burnin-disk-template", "drbd")
413 parallel = options.get("burnin-in-parallel", False)
414 check_inst = options.get("burnin-check-instances", False)
415 do_rename = options.get("burnin-rename", "")
416 do_reboot = options.get("burnin-reboot", True)
417 reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
419 # Get as many instances as we need
423 num = qa_config.get("options", {}).get("burnin-instances", 1)
424 for _ in range(0, num):
425 instances.append(qa_config.AcquireInstance())
426 except qa_error.OutOfInstancesError:
427 print "Not enough instances, continuing anyway."
429 if len(instances) < 1:
430 raise qa_error.Error("Burnin needs at least one instance")
432 script = qa_utils.UploadFile(master["primary"], "../tools/burnin")
436 "--os=%s" % qa_config.get("os"),
437 "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
438 "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
439 "--disk-size=%s" % ",".join(qa_config.get("disk")),
440 "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
441 "--disk-template=%s" % disk_template]
443 cmd.append("--parallel")
444 cmd.append("--early-release")
446 cmd.append("--http-check")
448 cmd.append("--rename=%s" % do_rename)
450 cmd.append("--no-reboot")
452 cmd.append("--reboot-types=%s" % ",".join(reboot_types))
453 cmd += [inst["name"] for inst in instances]
456 AssertCommand(["rm", "-f", script])
459 for inst in instances:
460 qa_config.ReleaseInstance(inst)
463 def TestClusterMasterFailover():
464 """gnt-cluster master-failover"""
465 master = qa_config.GetMasterNode()
466 failovermaster = qa_config.AcquireNode(exclude=master)
468 cmd = ["gnt-cluster", "master-failover"]
470 AssertCommand(cmd, node=failovermaster)
471 # Back to original master node
472 AssertCommand(cmd, node=master)
474 qa_config.ReleaseNode(failovermaster)
477 def TestClusterMasterFailoverWithDrainedQueue():
478 """gnt-cluster master-failover with drained queue"""
479 drain_check = ["test", "-f", pathutils.JOB_QUEUE_DRAIN_FILE]
481 master = qa_config.GetMasterNode()
482 failovermaster = qa_config.AcquireNode(exclude=master)
484 # Ensure queue is not drained
485 for node in [master, failovermaster]:
486 AssertCommand(drain_check, node=node, fail=True)
488 # Drain queue on failover master
489 AssertCommand(["touch", pathutils.JOB_QUEUE_DRAIN_FILE], node=failovermaster)
491 cmd = ["gnt-cluster", "master-failover"]
493 AssertCommand(drain_check, node=failovermaster)
494 AssertCommand(cmd, node=failovermaster)
495 AssertCommand(drain_check, fail=True)
496 AssertCommand(drain_check, node=failovermaster, fail=True)
498 # Back to original master node
499 AssertCommand(cmd, node=master)
501 qa_config.ReleaseNode(failovermaster)
503 AssertCommand(drain_check, fail=True)
504 AssertCommand(drain_check, node=failovermaster, fail=True)
507 def TestClusterCopyfile():
508 """gnt-cluster copyfile"""
509 master = qa_config.GetMasterNode()
511 uniqueid = utils.NewUUID()
513 # Create temporary file
514 f = tempfile.NamedTemporaryFile()
519 # Upload file to master node
520 testname = qa_utils.UploadFile(master["primary"], f.name)
522 # Copy file to all nodes
523 AssertCommand(["gnt-cluster", "copyfile", testname])
524 _CheckFileOnAllNodes(testname, uniqueid)
526 _RemoveFileFromAllNodes(testname)
529 def TestClusterCommand():
530 """gnt-cluster command"""
531 uniqueid = utils.NewUUID()
532 rfile = "/tmp/gnt%s" % utils.NewUUID()
533 rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
534 cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
535 "%s >%s" % (rcmd, rfile)])
539 _CheckFileOnAllNodes(rfile, uniqueid)
541 _RemoveFileFromAllNodes(rfile)
544 def TestClusterDestroy():
545 """gnt-cluster destroy"""
546 AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
549 def TestClusterRepairDiskSizes():
550 """gnt-cluster repair-disk-sizes"""
551 AssertCommand(["gnt-cluster", "repair-disk-sizes"])