4 # Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Script for doing QA on Ganeti.
26 # pylint: disable-msg=C0103
45 from ganeti import utils
46 from ganeti import rapi
47 from ganeti import constants
49 import ganeti.rapi.client # pylint: disable-msg=W0611
52 def _FormatHeader(line, end=72):
53 """Fill a line up to the end column.
56 line = "---- " + line + " "
57 line += "-" * (end-len(line))
62 def _DescriptionOf(fn):
63 """Computes the description of an item.
67 desc = fn.__doc__.splitlines()[0].strip()
71 return desc.rstrip(".")
74 def RunTest(fn, *args):
75 """Runs a test after printing a header.
79 tstart = datetime.datetime.now()
81 desc = _DescriptionOf(fn)
84 print _FormatHeader("%s start %s" % (tstart, desc))
90 tstop = datetime.datetime.now()
91 tdelta = tstop - tstart
92 print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
95 def RunTestIf(testnames, fn, *args):
96 """Runs a test conditionally.
98 @param testnames: either a single test name in the configuration
99 file, or a list of testnames (which will be AND-ed together)
102 if qa_config.TestEnabled(testnames):
105 tstart = datetime.datetime.now()
106 desc = _DescriptionOf(fn)
107 print _FormatHeader("%s skipping %s, test(s) %s disabled" %
108 (tstart, desc, testnames))
112 """Run several environment tests.
115 RunTestIf("env", qa_env.TestSshConnection)
116 RunTestIf("env", qa_env.TestIcmpPing)
117 RunTestIf("env", qa_env.TestGanetiCommands)
120 def SetupCluster(rapi_user, rapi_secret):
121 """Initializes the cluster.
123 @param rapi_user: Login user for RAPI
124 @param rapi_secret: Login secret for RAPI
127 RunTestIf("create-cluster", qa_cluster.TestClusterInit,
128 rapi_user, rapi_secret)
130 # Test on empty cluster
131 RunTestIf("node-list", qa_node.TestNodeList)
132 RunTestIf("instance-list", qa_instance.TestInstanceList)
134 RunTestIf("create-cluster", qa_node.TestNodeAddAll)
135 if not qa_config.TestEnabled("create-cluster"):
136 # consider the nodes are already there
137 qa_node.MarkNodeAddedAll()
139 RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
141 # enable the watcher (unconditionally)
142 RunTest(qa_daemon.TestResumeWatcher)
144 RunTestIf("node-list", qa_node.TestNodeList)
146 # Test listing fields
147 RunTestIf("node-list", qa_node.TestNodeListFields)
148 RunTestIf("instance-list", qa_instance.TestInstanceListFields)
150 RunTestIf("node-info", qa_node.TestNodeInfo)
153 def RunClusterTests():
154 """Runs tests related to gnt-cluster.
158 ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
159 ("cluster-verify", qa_cluster.TestClusterVerify),
160 ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
161 # TODO: add more cluster modify tests
162 ("cluster-modify", qa_cluster.TestClusterModifyBe),
163 ("cluster-rename", qa_cluster.TestClusterRename),
164 ("cluster-info", qa_cluster.TestClusterVersion),
165 ("cluster-info", qa_cluster.TestClusterInfo),
166 ("cluster-info", qa_cluster.TestClusterGetmaster),
167 ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
168 ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
169 ("cluster-command", qa_cluster.TestClusterCommand),
170 ("cluster-burnin", qa_cluster.TestClusterBurnin),
171 ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
172 ("cluster-master-failover",
173 qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
174 ("cluster-oob", qa_cluster.TestClusterOob),
175 ("rapi", qa_rapi.TestVersion),
176 ("rapi", qa_rapi.TestEmptyCluster),
177 ("rapi", qa_rapi.TestRapiQuery),
182 def RunRepairDiskSizes():
183 """Run the repair disk-sizes test.
186 RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
190 """Runs all tests related to gnt-os.
193 if qa_config.TestEnabled("rapi"):
194 rapi_getos = qa_rapi.GetOperatingSystems
200 qa_os.TestOsDiagnose,
207 qa_os.TestOsPartiallyValid,
209 RunTestIf("os", fn, rapi_getos)
212 qa_os.TestOsModifyValid,
213 qa_os.TestOsModifyInvalid,
214 qa_os.TestOsStatesNonExisting,
219 def RunCommonInstanceTests(instance):
220 """Runs a few tests that are common to all disk types.
223 RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
224 RunTestIf(["instance-shutdown", "instance-console", "rapi"],
225 qa_rapi.TestRapiStoppedInstanceConsole, instance)
226 RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
228 RunTestIf("instance-list", qa_instance.TestInstanceList)
230 RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
232 RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
233 RunTestIf(["instance-modify", "rapi"],
234 qa_rapi.TestRapiInstanceModify, instance)
236 RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
237 RunTestIf(["instance-console", "rapi"],
238 qa_rapi.TestRapiInstanceConsole, instance)
240 RunTestIf("instance-reinstall", qa_instance.TestInstanceShutdown, instance)
241 RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
242 RunTestIf(["instance-reinstall", "rapi"],
243 qa_rapi.TestRapiInstanceReinstall, instance)
244 RunTestIf("instance-reinstall", qa_instance.TestInstanceStartup, instance)
246 RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
248 if qa_config.TestEnabled("instance-rename"):
249 rename_source = instance["name"]
250 rename_target = qa_config.get("rename", None)
251 RunTest(qa_instance.TestInstanceShutdown, instance)
252 # perform instance rename to the same name
253 RunTest(qa_instance.TestInstanceRename, rename_source, rename_source)
254 RunTestIf("rapi", qa_rapi.TestRapiInstanceRename,
255 rename_source, rename_source)
256 if rename_target is not None:
257 # perform instance rename to a different name, if we have one configured
258 RunTest(qa_instance.TestInstanceRename, rename_source, rename_target)
259 RunTest(qa_instance.TestInstanceRename, rename_target, rename_source)
260 RunTestIf("rapi", qa_rapi.TestRapiInstanceRename,
261 rename_source, rename_target)
262 RunTestIf("rapi", qa_rapi.TestRapiInstanceRename,
263 rename_target, rename_source)
264 RunTest(qa_instance.TestInstanceStartup, instance)
266 RunTestIf("tags", qa_tags.TestInstanceTags, instance)
268 RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
270 RunTestIf("rapi", qa_rapi.TestInstance, instance)
272 # Lists instances, too
273 RunTestIf("node-list", qa_node.TestNodeList)
276 def RunCommonNodeTests():
277 """Run a few common node tests.
280 RunTestIf("node-volumes", qa_node.TestNodeVolumes)
281 RunTestIf("node-storage", qa_node.TestNodeStorage)
282 RunTestIf("node-oob", qa_node.TestOutOfBand)
285 def RunGroupListTests():
286 """Run tests for listing node groups.
289 RunTestIf("group-list", qa_group.TestGroupList)
290 RunTestIf("group-list", qa_group.TestGroupListFields)
293 def RunGroupRwTests():
294 """Run tests for adding/removing/renaming groups.
297 RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
298 RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
299 RunTestIf("group-rwops", qa_group.TestGroupModify)
300 RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
301 RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
302 qa_group.GetDefaultGroup())
305 def RunExportImportTests(instance, pnode, snode):
306 """Tries to export and import the instance.
308 @param pnode: current primary node of the instance
309 @param snode: current secondary node of the instance, if any,
313 if qa_config.TestEnabled("instance-export"):
314 RunTest(qa_instance.TestInstanceExportNoTarget, instance)
316 expnode = qa_config.AcquireNode(exclude=pnode)
318 name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
320 RunTest(qa_instance.TestBackupList, expnode)
322 if qa_config.TestEnabled("instance-import"):
323 newinst = qa_config.AcquireInstance()
325 RunTest(qa_instance.TestInstanceImport, pnode, newinst,
327 RunTest(qa_instance.TestInstanceRemove, newinst)
329 qa_config.ReleaseInstance(newinst)
331 qa_config.ReleaseNode(expnode)
333 if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
334 newinst = qa_config.AcquireInstance()
339 excl = [pnode, snode]
340 tnode = qa_config.AcquireNode(exclude=excl)
342 RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
345 qa_config.ReleaseNode(tnode)
347 qa_config.ReleaseInstance(newinst)
350 def RunDaemonTests(instance):
351 """Test the ganeti-watcher script.
354 RunTest(qa_daemon.TestPauseWatcher)
356 RunTestIf("instance-automatic-restart",
357 qa_daemon.TestInstanceAutomaticRestart, instance)
358 RunTestIf("instance-consecutive-failures",
359 qa_daemon.TestInstanceConsecutiveFailures, instance)
361 RunTest(qa_daemon.TestResumeWatcher)
364 def RunHardwareFailureTests(instance, pnode, snode):
365 """Test cluster internal hardware failure recovery.
368 RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
369 RunTestIf(["instance-failover", "rapi"],
370 qa_rapi.TestRapiInstanceFailover, instance)
372 RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
373 RunTestIf(["instance-migrate", "rapi"],
374 qa_rapi.TestRapiInstanceMigrate, instance)
376 if qa_config.TestEnabled("instance-replace-disks"):
377 othernode = qa_config.AcquireNode(exclude=[pnode, snode])
379 RunTest(qa_instance.TestReplaceDisks,
380 instance, pnode, snode, othernode)
382 qa_config.ReleaseNode(othernode)
384 RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, pnode, snode)
386 RunTestIf("node-failover", qa_node.TestNodeFailover, pnode, snode)
388 RunTestIf("instance-disk-failure", qa_instance.TestInstanceMasterDiskFailure,
389 instance, pnode, snode)
390 RunTestIf("instance-disk-failure",
391 qa_instance.TestInstanceSecondaryDiskFailure, instance,
399 rapi_user = "ganeti-qa"
400 rapi_secret = utils.GenerateSecret()
403 SetupCluster(rapi_user, rapi_secret)
405 # Load RAPI certificate
406 qa_rapi.Setup(rapi_user, rapi_secret)
411 RunTestIf("tags", qa_tags.TestClusterTags)
417 pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
419 RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
420 RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
422 qa_config.ReleaseNode(pnode)
424 pnode = qa_config.AcquireNode()
426 RunTestIf("tags", qa_tags.TestNodeTags, pnode)
428 if qa_rapi.Enabled():
429 RunTest(qa_rapi.TestNode, pnode)
431 if qa_config.TestEnabled("instance-add-plain-disk"):
432 for use_client in [True, False]:
433 rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
435 RunCommonInstanceTests(rapi_instance)
436 RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
439 if qa_config.TestEnabled("instance-add-plain-disk"):
440 instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
441 RunCommonInstanceTests(instance)
443 RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
444 RunExportImportTests(instance, pnode, None)
445 RunDaemonTests(instance)
447 RunTest(qa_instance.TestInstanceRemove, instance)
451 ("instance-add-drbd-disk",
452 qa_instance.TestInstanceAddWithDrbdDisk),
455 for name, func in multinode_tests:
456 if qa_config.TestEnabled(name):
457 snode = qa_config.AcquireNode(exclude=pnode)
459 instance = RunTest(func, pnode, snode)
460 RunCommonInstanceTests(instance)
462 RunTest(qa_group.TestAssignNodesIncludingSplit,
463 constants.INITIAL_NODE_GROUP_NAME,
464 pnode["primary"], snode["primary"])
465 if qa_config.TestEnabled("instance-convert-disk"):
466 RunTest(qa_instance.TestInstanceShutdown, instance)
467 RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
468 RunTest(qa_instance.TestInstanceStartup, instance)
469 RunExportImportTests(instance, pnode, snode)
470 RunHardwareFailureTests(instance, pnode, snode)
472 RunTest(qa_instance.TestInstanceRemove, instance)
475 qa_config.ReleaseNode(snode)
477 if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
478 for shutdown in [False, True]:
479 instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
480 expnode = qa_config.AcquireNode(exclude=pnode)
483 # Stop instance before exporting and removing it
484 RunTest(qa_instance.TestInstanceShutdown, instance)
485 RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
486 RunTest(qa_instance.TestBackupList, expnode)
488 qa_config.ReleaseNode(expnode)
493 qa_config.ReleaseNode(pnode)
495 RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
497 RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
500 @rapi.client.UsesRapiClient
505 parser = optparse.OptionParser(usage="%prog [options] <config-file>")
506 parser.add_option("--yes-do-it", dest="yes_do_it",
508 help="Really execute the tests")
509 (qa_config.options, args) = parser.parse_args()
512 (config_file, ) = args
514 parser.error("Wrong number of arguments.")
516 if not qa_config.options.yes_do_it:
517 print ("Executing this script irreversibly destroys any Ganeti\n"
518 "configuration on all nodes involved. If you really want\n"
519 "to start testing, supply the --yes-do-it option.")
522 qa_config.Load(config_file)
524 qa_utils.StartMultiplexer(qa_config.GetMasterNode()["primary"])
528 qa_utils.CloseMultiplexers()
530 if __name__ == "__main__":