4 # Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Script for doing QA on Ganeti.
26 # pylint: disable=C0103
45 from ganeti import utils
46 from ganeti import rapi
47 from ganeti import constants
49 import ganeti.rapi.client # pylint: disable=W0611
52 def _FormatHeader(line, end=72):
53 """Fill a line up to the end column.
56 line = "---- " + line + " "
57 line += "-" * (end - len(line))
62 def _DescriptionOf(fn):
63 """Computes the description of an item.
67 desc = fn.__doc__.splitlines()[0].strip()
71 return desc.rstrip(".")
74 def RunTest(fn, *args):
75 """Runs a test after printing a header.
79 tstart = datetime.datetime.now()
81 desc = _DescriptionOf(fn)
84 print _FormatHeader("%s start %s" % (tstart, desc))
90 tstop = datetime.datetime.now()
91 tdelta = tstop - tstart
92 print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
95 def RunTestIf(testnames, fn, *args):
96 """Runs a test conditionally.
98 @param testnames: either a single test name in the configuration
99 file, or a list of testnames (which will be AND-ed together)
102 if qa_config.TestEnabled(testnames):
105 tstart = datetime.datetime.now()
106 desc = _DescriptionOf(fn)
107 print _FormatHeader("%s skipping %s, test(s) %s disabled" %
108 (tstart, desc, testnames))
112 """Run several environment tests.
115 RunTestIf("env", qa_env.TestSshConnection)
116 RunTestIf("env", qa_env.TestIcmpPing)
117 RunTestIf("env", qa_env.TestGanetiCommands)
120 def SetupCluster(rapi_user, rapi_secret):
121 """Initializes the cluster.
123 @param rapi_user: Login user for RAPI
124 @param rapi_secret: Login secret for RAPI
127 RunTestIf("create-cluster", qa_cluster.TestClusterInit,
128 rapi_user, rapi_secret)
130 # Test on empty cluster
131 RunTestIf("node-list", qa_node.TestNodeList)
132 RunTestIf("instance-list", qa_instance.TestInstanceList)
134 RunTestIf("create-cluster", qa_node.TestNodeAddAll)
135 if not qa_config.TestEnabled("create-cluster"):
136 # consider the nodes are already there
137 qa_node.MarkNodeAddedAll()
139 RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
141 # enable the watcher (unconditionally)
142 RunTest(qa_daemon.TestResumeWatcher)
144 RunTestIf("node-list", qa_node.TestNodeList)
146 # Test listing fields
147 RunTestIf("node-list", qa_node.TestNodeListFields)
148 RunTestIf("instance-list", qa_instance.TestInstanceListFields)
150 RunTestIf("node-info", qa_node.TestNodeInfo)
153 def RunClusterTests():
154 """Runs tests related to gnt-cluster.
158 ("create-cluster", qa_cluster.TestClusterInitDisk),
159 ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
160 ("cluster-verify", qa_cluster.TestClusterVerify),
161 ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
162 # TODO: add more cluster modify tests
163 ("cluster-modify", qa_cluster.TestClusterModifyBe),
164 ("cluster-modify", qa_cluster.TestClusterModifyDisk),
165 ("cluster-rename", qa_cluster.TestClusterRename),
166 ("cluster-info", qa_cluster.TestClusterVersion),
167 ("cluster-info", qa_cluster.TestClusterInfo),
168 ("cluster-info", qa_cluster.TestClusterGetmaster),
169 ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
170 ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
171 ("cluster-command", qa_cluster.TestClusterCommand),
172 ("cluster-burnin", qa_cluster.TestClusterBurnin),
173 ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
174 ("cluster-master-failover",
175 qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
176 ("cluster-oob", qa_cluster.TestClusterOob),
177 ("rapi", qa_rapi.TestVersion),
178 ("rapi", qa_rapi.TestEmptyCluster),
179 ("rapi", qa_rapi.TestRapiQuery),
184 def RunRepairDiskSizes():
185 """Run the repair disk-sizes test.
188 RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
192 """Runs all tests related to gnt-os.
195 if qa_config.TestEnabled("rapi"):
196 rapi_getos = qa_rapi.GetOperatingSystems
202 qa_os.TestOsDiagnose,
209 qa_os.TestOsPartiallyValid,
211 RunTestIf("os", fn, rapi_getos)
214 qa_os.TestOsModifyValid,
215 qa_os.TestOsModifyInvalid,
216 qa_os.TestOsStatesNonExisting,
221 def RunCommonInstanceTests(instance):
222 """Runs a few tests that are common to all disk types.
225 RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
226 RunTestIf(["instance-shutdown", "instance-console", "rapi"],
227 qa_rapi.TestRapiStoppedInstanceConsole, instance)
228 RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
230 # Test shutdown/start via RAPI
231 RunTestIf(["instance-shutdown", "rapi"],
232 qa_rapi.TestRapiInstanceShutdown, instance)
233 RunTestIf(["instance-shutdown", "rapi"],
234 qa_rapi.TestRapiInstanceStartup, instance)
236 RunTestIf("instance-list", qa_instance.TestInstanceList)
238 RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
240 RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
241 RunTestIf(["instance-modify", "rapi"],
242 qa_rapi.TestRapiInstanceModify, instance)
244 RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
245 RunTestIf(["instance-console", "rapi"],
246 qa_rapi.TestRapiInstanceConsole, instance)
248 RunTestIf("instance-reinstall", qa_instance.TestInstanceShutdown, instance)
249 RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
250 RunTestIf(["instance-reinstall", "rapi"],
251 qa_rapi.TestRapiInstanceReinstall, instance)
252 RunTestIf("instance-reinstall", qa_instance.TestInstanceStartup, instance)
254 RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
256 if qa_config.TestEnabled("instance-rename"):
257 rename_source = instance["name"]
258 rename_target = qa_config.get("rename", None)
259 RunTest(qa_instance.TestInstanceShutdown, instance)
260 # perform instance rename to the same name
261 RunTest(qa_instance.TestInstanceRename, rename_source, rename_source)
262 RunTestIf("rapi", qa_rapi.TestRapiInstanceRename,
263 rename_source, rename_source)
264 if rename_target is not None:
265 # perform instance rename to a different name, if we have one configured
266 RunTest(qa_instance.TestInstanceRename, rename_source, rename_target)
267 RunTest(qa_instance.TestInstanceRename, rename_target, rename_source)
268 RunTestIf("rapi", qa_rapi.TestRapiInstanceRename,
269 rename_source, rename_target)
270 RunTestIf("rapi", qa_rapi.TestRapiInstanceRename,
271 rename_target, rename_source)
272 RunTest(qa_instance.TestInstanceStartup, instance)
274 RunTestIf("tags", qa_tags.TestInstanceTags, instance)
276 RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
278 RunTestIf("rapi", qa_rapi.TestInstance, instance)
280 # Lists instances, too
281 RunTestIf("node-list", qa_node.TestNodeList)
284 def RunCommonNodeTests():
285 """Run a few common node tests.
288 RunTestIf("node-volumes", qa_node.TestNodeVolumes)
289 RunTestIf("node-storage", qa_node.TestNodeStorage)
290 RunTestIf("node-oob", qa_node.TestOutOfBand)
293 def RunGroupListTests():
294 """Run tests for listing node groups.
297 RunTestIf("group-list", qa_group.TestGroupList)
298 RunTestIf("group-list", qa_group.TestGroupListFields)
301 def RunGroupRwTests():
302 """Run tests for adding/removing/renaming groups.
305 RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
306 RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
307 RunTestIf("group-rwops", qa_group.TestGroupModify)
308 RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
309 RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
310 qa_group.GetDefaultGroup())
313 def RunExportImportTests(instance, pnode, snode):
314 """Tries to export and import the instance.
316 @param pnode: current primary node of the instance
317 @param snode: current secondary node of the instance, if any,
321 if qa_config.TestEnabled("instance-export"):
322 RunTest(qa_instance.TestInstanceExportNoTarget, instance)
324 expnode = qa_config.AcquireNode(exclude=pnode)
326 name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
328 RunTest(qa_instance.TestBackupList, expnode)
330 if qa_config.TestEnabled("instance-import"):
331 newinst = qa_config.AcquireInstance()
333 RunTest(qa_instance.TestInstanceImport, pnode, newinst,
335 RunTest(qa_instance.TestInstanceRemove, newinst)
337 qa_config.ReleaseInstance(newinst)
339 qa_config.ReleaseNode(expnode)
341 if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
342 newinst = qa_config.AcquireInstance()
347 excl = [pnode, snode]
348 tnode = qa_config.AcquireNode(exclude=excl)
350 RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
353 qa_config.ReleaseNode(tnode)
355 qa_config.ReleaseInstance(newinst)
358 def RunDaemonTests(instance):
359 """Test the ganeti-watcher script.
362 RunTest(qa_daemon.TestPauseWatcher)
364 RunTestIf("instance-automatic-restart",
365 qa_daemon.TestInstanceAutomaticRestart, instance)
366 RunTestIf("instance-consecutive-failures",
367 qa_daemon.TestInstanceConsecutiveFailures, instance)
369 RunTest(qa_daemon.TestResumeWatcher)
372 def RunHardwareFailureTests(instance, pnode, snode):
373 """Test cluster internal hardware failure recovery.
376 RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
377 RunTestIf(["instance-failover", "rapi"],
378 qa_rapi.TestRapiInstanceFailover, instance)
380 RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
381 RunTestIf(["instance-migrate", "rapi"],
382 qa_rapi.TestRapiInstanceMigrate, instance)
384 if qa_config.TestEnabled("instance-replace-disks"):
385 othernode = qa_config.AcquireNode(exclude=[pnode, snode])
387 RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
388 RunTest(qa_instance.TestReplaceDisks,
389 instance, pnode, snode, othernode)
391 qa_config.ReleaseNode(othernode)
393 RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, pnode, snode)
395 RunTestIf("node-failover", qa_node.TestNodeFailover, pnode, snode)
397 RunTestIf("instance-disk-failure", qa_instance.TestInstanceMasterDiskFailure,
398 instance, pnode, snode)
399 RunTestIf("instance-disk-failure",
400 qa_instance.TestInstanceSecondaryDiskFailure, instance,
408 rapi_user = "ganeti-qa"
409 rapi_secret = utils.GenerateSecret()
412 SetupCluster(rapi_user, rapi_secret)
414 # Load RAPI certificate
415 qa_rapi.Setup(rapi_user, rapi_secret)
420 RunTestIf("tags", qa_tags.TestClusterTags)
426 pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
428 RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
429 RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
430 RunTestIf("delay", qa_cluster.TestDelay, pnode)
432 qa_config.ReleaseNode(pnode)
434 pnode = qa_config.AcquireNode()
436 RunTestIf("tags", qa_tags.TestNodeTags, pnode)
438 if qa_rapi.Enabled():
439 RunTest(qa_rapi.TestNode, pnode)
441 if qa_config.TestEnabled("instance-add-plain-disk"):
442 for use_client in [True, False]:
443 rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
445 RunCommonInstanceTests(rapi_instance)
446 RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
449 if qa_config.TestEnabled("instance-add-plain-disk"):
450 instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
451 RunCommonInstanceTests(instance)
453 RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
454 RunExportImportTests(instance, pnode, None)
455 RunDaemonTests(instance)
457 RunTest(qa_instance.TestInstanceRemove, instance)
461 ("instance-add-drbd-disk",
462 qa_instance.TestInstanceAddWithDrbdDisk),
465 for name, func in multinode_tests:
466 if qa_config.TestEnabled(name):
467 snode = qa_config.AcquireNode(exclude=pnode)
469 instance = RunTest(func, pnode, snode)
470 RunCommonInstanceTests(instance)
472 RunTest(qa_group.TestAssignNodesIncludingSplit,
473 constants.INITIAL_NODE_GROUP_NAME,
474 pnode["primary"], snode["primary"])
475 if qa_config.TestEnabled("instance-convert-disk"):
476 RunTest(qa_instance.TestInstanceShutdown, instance)
477 RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
478 RunTest(qa_instance.TestInstanceStartup, instance)
479 RunExportImportTests(instance, pnode, snode)
480 RunHardwareFailureTests(instance, pnode, snode)
482 RunTest(qa_instance.TestInstanceRemove, instance)
485 qa_config.ReleaseNode(snode)
487 if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
488 for shutdown in [False, True]:
489 instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
490 expnode = qa_config.AcquireNode(exclude=pnode)
493 # Stop instance before exporting and removing it
494 RunTest(qa_instance.TestInstanceShutdown, instance)
495 RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
496 RunTest(qa_instance.TestBackupList, expnode)
498 qa_config.ReleaseNode(expnode)
503 qa_config.ReleaseNode(pnode)
505 RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
507 RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
510 @rapi.client.UsesRapiClient
515 parser = optparse.OptionParser(usage="%prog [options] <config-file>")
516 parser.add_option("--yes-do-it", dest="yes_do_it",
518 help="Really execute the tests")
519 (qa_config.options, args) = parser.parse_args()
522 (config_file, ) = args
524 parser.error("Wrong number of arguments.")
526 if not qa_config.options.yes_do_it:
527 print ("Executing this script irreversibly destroys any Ganeti\n"
528 "configuration on all nodes involved. If you really want\n"
529 "to start testing, supply the --yes-do-it option.")
532 qa_config.Load(config_file)
534 qa_utils.StartMultiplexer(qa_config.GetMasterNode()["primary"])
538 qa_utils.CloseMultiplexers()
540 if __name__ == "__main__":