4 # Copyright (C) 2007, 2010, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Cluster related QA tests.
29 from ganeti import constants
30 from ganeti import compat
31 from ganeti import utils
37 from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
40 #: cluster verify command
41 _CLUSTER_VERIFY = ["gnt-cluster", "verify"]
44 def _RemoveFileFromAllNodes(filename):
45 """Removes a file from all nodes.
48 for node in qa_config.get("nodes"):
49 AssertCommand(["rm", "-f", filename], node=node)
52 def _CheckFileOnAllNodes(filename, content):
53 """Verifies the content of the given file on all nodes.
56 cmd = utils.ShellQuoteArgs(["cat", filename])
57 for node in qa_config.get("nodes"):
58 AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content)
61 # data for testing failures due to bad keys/values for disk parameters
62 _FAIL_PARAMS = ["nonexistent:resync-rate=1",
64 "drbd:resync-rate=invalid",
68 def TestClusterInitDisk():
69 """gnt-cluster init -D"""
70 name = qa_config.get("name")
71 for param in _FAIL_PARAMS:
72 AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
75 def TestClusterInit(rapi_user, rapi_secret):
76 """gnt-cluster init"""
77 master = qa_config.GetMasterNode()
79 rapi_dir = os.path.dirname(constants.RAPI_USERS_FILE)
81 # First create the RAPI credentials
82 fh = tempfile.NamedTemporaryFile()
84 fh.write("%s %s write\n" % (rapi_user, rapi_secret))
87 tmpru = qa_utils.UploadFile(master["primary"], fh.name)
89 AssertCommand(["mkdir", "-p", rapi_dir])
90 AssertCommand(["mv", tmpru, constants.RAPI_USERS_FILE])
92 AssertCommand(["rm", "-f", tmpru])
98 "gnt-cluster", "init",
99 "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
100 "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
103 for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
105 for spec_val in ("min", "max", "std"):
106 spec = qa_config.get("ispec_%s_%s" %
107 (spec_type.replace('-', '_'), spec_val), None)
109 cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
111 if master.get("secondary", None):
112 cmd.append("--secondary-ip=%s" % master["secondary"])
114 bridge = qa_config.get("bridge", None)
116 cmd.append("--bridge=%s" % bridge)
117 cmd.append("--master-netdev=%s" % bridge)
119 cmd.append(qa_config.get("name"))
122 cmd = ["gnt-cluster", "modify"]
124 # hypervisor parameter modifications
125 hvp = qa_config.get("hypervisor-parameters", {})
126 for k, v in hvp.items():
127 cmd.extend(["-H", "%s:%s" % (k, v)])
128 # backend parameter modifications
129 bep = qa_config.get("backend-parameters", "")
131 cmd.extend(["-B", bep])
137 osp = qa_config.get("os-parameters", {})
138 for k, v in osp.items():
139 AssertCommand(["gnt-os", "modify", "-O", v, k])
141 # OS hypervisor parameters
142 os_hvp = qa_config.get("os-hvp", {})
143 for os_name in os_hvp:
144 for hv, hvp in os_hvp[os_name].items():
145 AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
148 def TestClusterRename():
149 """gnt-cluster rename"""
150 cmd = ["gnt-cluster", "rename", "-f"]
152 original_name = qa_config.get("name")
153 rename_target = qa_config.get("rename", None)
154 if rename_target is None:
155 print qa_utils.FormatError('"rename" entry is missing')
159 cmd + [rename_target],
161 cmd + [original_name],
167 def TestClusterOob():
168 """out-of-band framework"""
169 oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
171 AssertCommand(_CLUSTER_VERIFY)
172 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
173 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
176 AssertCommand(_CLUSTER_VERIFY, fail=True)
178 AssertCommand(["touch", oob_path_exists])
179 AssertCommand(["chmod", "0400", oob_path_exists])
180 AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
183 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
184 "oob_program=%s" % oob_path_exists])
186 AssertCommand(_CLUSTER_VERIFY, fail=True)
188 AssertCommand(["chmod", "0500", oob_path_exists])
189 AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
191 AssertCommand(_CLUSTER_VERIFY)
193 AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
195 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
199 def TestClusterEpo():
200 """gnt-cluster epo"""
201 master = qa_config.GetMasterNode()
203 # Assert that OOB is unavailable for all nodes
204 result_output = GetCommandOutput(master["primary"],
205 "gnt-node list --verbose --no-headers -o"
207 AssertEqual(compat.all(powered == "(unavail)"
208 for powered in result_output.splitlines()), True)
211 AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
212 # --all doesn't expect arguments
213 AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
215 # Unless --all is given master is not allowed to be in the list
216 AssertCommand(["gnt-cluster", "epo", "-f", master["primary"]], fail=True)
218 # This shouldn't fail
219 AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
221 # All instances should have been stopped now
222 result_output = GetCommandOutput(master["primary"],
223 "gnt-instance list --no-headers -o status")
224 # ERROR_down because the instance is stopped but not recorded as such
225 AssertEqual(compat.all(status == "ERROR_down"
226 for status in result_output.splitlines()), True)
228 # Now start everything again
229 AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
231 # All instances should have been started now
232 result_output = GetCommandOutput(master["primary"],
233 "gnt-instance list --no-headers -o status")
234 AssertEqual(compat.all(status == "running"
235 for status in result_output.splitlines()), True)
238 def TestClusterVerify():
239 """gnt-cluster verify"""
240 AssertCommand(_CLUSTER_VERIFY)
241 AssertCommand(["gnt-cluster", "verify-disks"])
245 """gnt-debug test-jobqueue"""
246 AssertCommand(["gnt-debug", "test-jobqueue"])
250 """gnt-debug delay"""
251 AssertCommand(["gnt-debug", "delay", "1"])
252 AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
253 AssertCommand(["gnt-debug", "delay", "--no-master",
254 "-n", node["primary"], "1"])
257 def TestClusterReservedLvs():
258 """gnt-cluster reserved lvs"""
260 (False, _CLUSTER_VERIFY),
261 (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
262 (False, ["lvcreate", "-L1G", "-nqa-test", "xenvg"]),
263 (True, _CLUSTER_VERIFY),
264 (False, ["gnt-cluster", "modify", "--reserved-lvs",
265 "xenvg/qa-test,.*/other-test"]),
266 (False, _CLUSTER_VERIFY),
267 (False, ["gnt-cluster", "modify", "--reserved-lvs", ".*/qa-.*"]),
268 (False, _CLUSTER_VERIFY),
269 (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
270 (True, _CLUSTER_VERIFY),
271 (False, ["lvremove", "-f", "xenvg/qa-test"]),
272 (False, _CLUSTER_VERIFY),
274 AssertCommand(cmd, fail=fail)
277 def TestClusterModifyEmpty():
278 """gnt-cluster modify"""
279 AssertCommand(["gnt-cluster", "modify"], fail=True)
282 def TestClusterModifyDisk():
283 """gnt-cluster modify -D"""
284 for param in _FAIL_PARAMS:
285 AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
288 def TestClusterModifyBe():
289 """gnt-cluster modify -B"""
292 (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
293 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
294 (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
295 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
296 (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
297 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
298 (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
299 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
300 (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
301 (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
302 (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
304 (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
305 (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
306 (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
307 (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
308 (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
310 (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
311 (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
312 (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
313 (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
314 (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
316 AssertCommand(cmd, fail=fail)
318 # redo the original-requested BE parameters, if any
319 bep = qa_config.get("backend-parameters", "")
321 AssertCommand(["gnt-cluster", "modify", "-B", bep])
324 def TestClusterInfo():
325 """gnt-cluster info"""
326 AssertCommand(["gnt-cluster", "info"])
329 def TestClusterRedistConf():
330 """gnt-cluster redist-conf"""
331 AssertCommand(["gnt-cluster", "redist-conf"])
334 def TestClusterGetmaster():
335 """gnt-cluster getmaster"""
336 AssertCommand(["gnt-cluster", "getmaster"])
339 def TestClusterVersion():
340 """gnt-cluster version"""
341 AssertCommand(["gnt-cluster", "version"])
344 def TestClusterRenewCrypto():
345 """gnt-cluster renew-crypto"""
346 master = qa_config.GetMasterNode()
348 # Conflicting options
349 cmd = ["gnt-cluster", "renew-crypto", "--force",
350 "--new-cluster-certificate", "--new-confd-hmac-key"]
352 ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
353 ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
355 for i in conflicting:
356 AssertCommand(cmd + i, fail=True)
358 # Invalid RAPI certificate
359 cmd = ["gnt-cluster", "renew-crypto", "--force",
360 "--rapi-certificate=/dev/null"]
361 AssertCommand(cmd, fail=True)
363 rapi_cert_backup = qa_utils.BackupFile(master["primary"],
364 constants.RAPI_CERT_FILE)
366 # Custom RAPI certificate
367 fh = tempfile.NamedTemporaryFile()
369 # Ensure certificate doesn't cause "gnt-cluster verify" to complain
370 validity = constants.SSL_CERT_EXPIRATION_WARN * 3
372 utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
374 tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
376 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
377 "--rapi-certificate=%s" % tmpcert])
379 AssertCommand(["rm", "-f", tmpcert])
381 # Custom cluster domain secret
382 cds_fh = tempfile.NamedTemporaryFile()
383 cds_fh.write(utils.GenerateSecret())
387 tmpcds = qa_utils.UploadFile(master["primary"], cds_fh.name)
389 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
390 "--cluster-domain-secret=%s" % tmpcds])
392 AssertCommand(["rm", "-f", tmpcds])
395 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
396 "--new-cluster-certificate", "--new-confd-hmac-key",
397 "--new-rapi-certificate", "--new-cluster-domain-secret"])
399 # Restore RAPI certificate
400 AssertCommand(["gnt-cluster", "renew-crypto", "--force",
401 "--rapi-certificate=%s" % rapi_cert_backup])
403 AssertCommand(["rm", "-f", rapi_cert_backup])
406 def TestClusterBurnin():
408 master = qa_config.GetMasterNode()
410 options = qa_config.get("options", {})
411 disk_template = options.get("burnin-disk-template", "drbd")
412 parallel = options.get("burnin-in-parallel", False)
413 check_inst = options.get("burnin-check-instances", False)
414 do_rename = options.get("burnin-rename", "")
415 do_reboot = options.get("burnin-reboot", True)
416 reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
418 # Get as many instances as we need
422 num = qa_config.get("options", {}).get("burnin-instances", 1)
423 for _ in range(0, num):
424 instances.append(qa_config.AcquireInstance())
425 except qa_error.OutOfInstancesError:
426 print "Not enough instances, continuing anyway."
428 if len(instances) < 1:
429 raise qa_error.Error("Burnin needs at least one instance")
431 script = qa_utils.UploadFile(master["primary"], "../tools/burnin")
435 "--os=%s" % qa_config.get("os"),
436 "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
437 "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
438 "--disk-size=%s" % ",".join(qa_config.get("disk")),
439 "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
440 "--disk-template=%s" % disk_template]
442 cmd.append("--parallel")
443 cmd.append("--early-release")
445 cmd.append("--http-check")
447 cmd.append("--rename=%s" % do_rename)
449 cmd.append("--no-reboot")
451 cmd.append("--reboot-types=%s" % ",".join(reboot_types))
452 cmd += [inst["name"] for inst in instances]
455 AssertCommand(["rm", "-f", script])
458 for inst in instances:
459 qa_config.ReleaseInstance(inst)
462 def TestClusterMasterFailover():
463 """gnt-cluster master-failover"""
464 master = qa_config.GetMasterNode()
465 failovermaster = qa_config.AcquireNode(exclude=master)
467 cmd = ["gnt-cluster", "master-failover"]
469 AssertCommand(cmd, node=failovermaster)
470 # Back to original master node
471 AssertCommand(cmd, node=master)
473 qa_config.ReleaseNode(failovermaster)
476 def TestClusterMasterFailoverWithDrainedQueue():
477 """gnt-cluster master-failover with drained queue"""
478 drain_check = ["test", "-f", constants.JOB_QUEUE_DRAIN_FILE]
480 master = qa_config.GetMasterNode()
481 failovermaster = qa_config.AcquireNode(exclude=master)
483 # Ensure queue is not drained
484 for node in [master, failovermaster]:
485 AssertCommand(drain_check, node=node, fail=True)
487 # Drain queue on failover master
488 AssertCommand(["touch", constants.JOB_QUEUE_DRAIN_FILE], node=failovermaster)
490 cmd = ["gnt-cluster", "master-failover"]
492 AssertCommand(drain_check, node=failovermaster)
493 AssertCommand(cmd, node=failovermaster)
494 AssertCommand(drain_check, fail=True)
495 AssertCommand(drain_check, node=failovermaster, fail=True)
497 # Back to original master node
498 AssertCommand(cmd, node=master)
500 qa_config.ReleaseNode(failovermaster)
502 AssertCommand(drain_check, fail=True)
503 AssertCommand(drain_check, node=failovermaster, fail=True)
506 def TestClusterCopyfile():
507 """gnt-cluster copyfile"""
508 master = qa_config.GetMasterNode()
510 uniqueid = utils.NewUUID()
512 # Create temporary file
513 f = tempfile.NamedTemporaryFile()
518 # Upload file to master node
519 testname = qa_utils.UploadFile(master["primary"], f.name)
521 # Copy file to all nodes
522 AssertCommand(["gnt-cluster", "copyfile", testname])
523 _CheckFileOnAllNodes(testname, uniqueid)
525 _RemoveFileFromAllNodes(testname)
528 def TestClusterCommand():
529 """gnt-cluster command"""
530 uniqueid = utils.NewUUID()
531 rfile = "/tmp/gnt%s" % utils.NewUUID()
532 rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
533 cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
534 "%s >%s" % (rcmd, rfile)])
538 _CheckFileOnAllNodes(rfile, uniqueid)
540 _RemoveFileFromAllNodes(rfile)
543 def TestClusterDestroy():
544 """gnt-cluster destroy"""
545 AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
548 def TestClusterRepairDiskSizes():
549 """gnt-cluster repair-disk-sizes"""
550 AssertCommand(["gnt-cluster", "repair-disk-sizes"])