X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/6623a5c144726430947802215315b2b1ae9d8d07..545d036273bb778751ac6de64a8c5dca9f7cb5cf:/qa/ganeti-qa.py diff --git a/qa/ganeti-qa.py b/qa/ganeti-qa.py index c68d0fb..a2a4eb0 100755 --- a/qa/ganeti-qa.py +++ b/qa/ganeti-qa.py @@ -1,7 +1,7 @@ -#!/usr/bin/python +#!/usr/bin/python -u # -# Copyright (C) 2007 Google Inc. +# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,146 +21,311 @@ """Script for doing QA on Ganeti. -You can create the required known_hosts file using ssh-keyscan. It's mandatory -to use the full name of a node (FQDN). For security reasons, verify the keys -before using them. -Example: ssh-keyscan -t rsa node{1,2,3,4}.example.com > known_hosts """ +# pylint: disable=C0103 +# due to invalid name + import sys -from datetime import datetime -from optparse import OptionParser +import datetime +import optparse import qa_cluster import qa_config import qa_daemon import qa_env +import qa_group import qa_instance import qa_node import qa_os -import qa_other +import qa_rapi import qa_tags import qa_utils +from ganeti import utils +from ganeti import rapi +from ganeti import constants -def RunTest(fn, *args): - """Runs a test after printing a header. +import ganeti.rapi.client # pylint: disable=W0611 + + +def _FormatHeader(line, end=72): + """Fill a line up to the end column. + + """ + line = "---- " + line + " " + line += "-" * (end - len(line)) + line = line.rstrip() + return line + + +def _DescriptionOf(fn): + """Computes the description of an item. """ if fn.__doc__: desc = fn.__doc__.splitlines()[0].strip() else: - desc = '%r' % fn + desc = "%r" % fn + + return desc.rstrip(".") + + +def RunTest(fn, *args): + """Runs a test after printing a header. + + """ + + tstart = datetime.datetime.now() - now = str(datetime.now()) + desc = _DescriptionOf(fn) print - print '---', now, ('-' * (55 - len(now))) - print desc - print '-' * 60 + print _FormatHeader("%s start %s" % (tstart, desc)) - return fn(*args) + try: + retval = fn(*args) + return retval + finally: + tstop = datetime.datetime.now() + tdelta = tstop - tstart + print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc)) + + +def RunTestIf(testnames, fn, *args): + """Runs a test conditionally. + + @param testnames: either a single test name in the configuration + file, or a list of testnames (which will be AND-ed together) + + """ + if qa_config.TestEnabled(testnames): + RunTest(fn, *args) + else: + tstart = datetime.datetime.now() + desc = _DescriptionOf(fn) + print _FormatHeader("%s skipping %s, test(s) %s disabled" % + (tstart, desc, testnames)) def RunEnvTests(): """Run several environment tests. """ - if not qa_config.TestEnabled('env'): - return + RunTestIf("env", qa_env.TestSshConnection) + RunTestIf("env", qa_env.TestIcmpPing) + RunTestIf("env", qa_env.TestGanetiCommands) - RunTest(qa_env.TestSshConnection) - RunTest(qa_env.TestIcmpPing) - RunTest(qa_env.TestGanetiCommands) - -def SetupCluster(): +def SetupCluster(rapi_user, rapi_secret): """Initializes the cluster. + @param rapi_user: Login user for RAPI + @param rapi_secret: Login secret for RAPI + """ - RunTest(qa_cluster.TestClusterInit) - RunTest(qa_node.TestNodeAddAll) - if qa_config.TestEnabled('node-info'): - RunTest(qa_node.TestNodeInfo) + RunTestIf("create-cluster", qa_cluster.TestClusterInit, + rapi_user, rapi_secret) + # Test on empty cluster + RunTestIf("node-list", qa_node.TestNodeList) + RunTestIf("instance-list", qa_instance.TestInstanceList) -def RunClusterTests(): - """Runs tests related to gnt-cluster. + RunTestIf("create-cluster", qa_node.TestNodeAddAll) + if not qa_config.TestEnabled("create-cluster"): + # consider the nodes are already there + qa_node.MarkNodeAddedAll() - """ - if qa_config.TestEnabled('cluster-verify'): - RunTest(qa_cluster.TestClusterVerify) + RunTestIf("test-jobqueue", qa_cluster.TestJobqueue) - if qa_config.TestEnabled('cluster-info'): - RunTest(qa_cluster.TestClusterVersion) - RunTest(qa_cluster.TestClusterInfo) - RunTest(qa_cluster.TestClusterGetmaster) + # enable the watcher (unconditionally) + RunTest(qa_daemon.TestResumeWatcher) - if qa_config.TestEnabled('cluster-copyfile'): - RunTest(qa_cluster.TestClusterCopyfile) + RunTestIf("node-list", qa_node.TestNodeList) - if qa_config.TestEnabled('cluster-command'): - RunTest(qa_cluster.TestClusterCommand) + # Test listing fields + RunTestIf("node-list", qa_node.TestNodeListFields) + RunTestIf("instance-list", qa_instance.TestInstanceListFields) - if qa_config.TestEnabled('cluster-burnin'): - RunTest(qa_cluster.TestClusterBurnin) + RunTestIf("node-info", qa_node.TestNodeInfo) - if qa_config.TestEnabled('cluster-master-failover'): - RunTest(qa_cluster.TestClusterMasterFailover) + +def RunClusterTests(): + """Runs tests related to gnt-cluster. + + """ + for test, fn in [ + ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto), + ("cluster-verify", qa_cluster.TestClusterVerify), + ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs), + # TODO: add more cluster modify tests + ("cluster-modify", qa_cluster.TestClusterModifyBe), + ("cluster-rename", qa_cluster.TestClusterRename), + ("cluster-info", qa_cluster.TestClusterVersion), + ("cluster-info", qa_cluster.TestClusterInfo), + ("cluster-info", qa_cluster.TestClusterGetmaster), + ("cluster-redist-conf", qa_cluster.TestClusterRedistConf), + ("cluster-copyfile", qa_cluster.TestClusterCopyfile), + ("cluster-command", qa_cluster.TestClusterCommand), + ("cluster-burnin", qa_cluster.TestClusterBurnin), + ("cluster-master-failover", qa_cluster.TestClusterMasterFailover), + ("cluster-master-failover", + qa_cluster.TestClusterMasterFailoverWithDrainedQueue), + ("cluster-oob", qa_cluster.TestClusterOob), + ("rapi", qa_rapi.TestVersion), + ("rapi", qa_rapi.TestEmptyCluster), + ("rapi", qa_rapi.TestRapiQuery), + ]: + RunTestIf(test, fn) + + +def RunRepairDiskSizes(): + """Run the repair disk-sizes test. + + """ + RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes) def RunOsTests(): """Runs all tests related to gnt-os. """ - if not qa_config.TestEnabled('os'): - return + if qa_config.TestEnabled("rapi"): + rapi_getos = qa_rapi.GetOperatingSystems + else: + rapi_getos = None + + for fn in [ + qa_os.TestOsList, + qa_os.TestOsDiagnose, + ]: + RunTestIf("os", fn) - RunTest(qa_os.TestOsList) - RunTest(qa_os.TestOsDiagnose) - RunTest(qa_os.TestOsValid) - RunTest(qa_os.TestOsInvalid) - RunTest(qa_os.TestOsPartiallyValid) + for fn in [ + qa_os.TestOsValid, + qa_os.TestOsInvalid, + qa_os.TestOsPartiallyValid, + ]: + RunTestIf("os", fn, rapi_getos) + + for fn in [ + qa_os.TestOsModifyValid, + qa_os.TestOsModifyInvalid, + qa_os.TestOsStatesNonExisting, + ]: + RunTestIf("os", fn) def RunCommonInstanceTests(instance): """Runs a few tests that are common to all disk types. """ - if qa_config.TestEnabled('instance-shutdown'): - RunTest(qa_instance.TestInstanceShutdown, instance) - RunTest(qa_instance.TestInstanceStartup, instance) + RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance) + RunTestIf(["instance-shutdown", "instance-console", "rapi"], + qa_rapi.TestRapiStoppedInstanceConsole, instance) + RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance) + + # Test shutdown/start via RAPI + RunTestIf(["instance-shutdown", "rapi"], + qa_rapi.TestRapiInstanceShutdown, instance) + RunTestIf(["instance-shutdown", "rapi"], + qa_rapi.TestRapiInstanceStartup, instance) + + RunTestIf("instance-list", qa_instance.TestInstanceList) + + RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance) - if qa_config.TestEnabled('instance-list'): - RunTest(qa_instance.TestInstanceList) + RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance) + RunTestIf(["instance-modify", "rapi"], + qa_rapi.TestRapiInstanceModify, instance) - if qa_config.TestEnabled('instance-info'): - RunTest(qa_instance.TestInstanceInfo, instance) + RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance) + RunTestIf(["instance-console", "rapi"], + qa_rapi.TestRapiInstanceConsole, instance) - if qa_config.TestEnabled('instance-reinstall'): + RunTestIf("instance-reinstall", qa_instance.TestInstanceShutdown, instance) + RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance) + RunTestIf(["instance-reinstall", "rapi"], + qa_rapi.TestRapiInstanceReinstall, instance) + RunTestIf("instance-reinstall", qa_instance.TestInstanceStartup, instance) + + RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance) + + if qa_config.TestEnabled("instance-rename"): + rename_source = instance["name"] + rename_target = qa_config.get("rename", None) RunTest(qa_instance.TestInstanceShutdown, instance) - RunTest(qa_instance.TestInstanceReinstall, instance) + # perform instance rename to the same name + RunTest(qa_instance.TestInstanceRename, rename_source, rename_source) + RunTestIf("rapi", qa_rapi.TestRapiInstanceRename, + rename_source, rename_source) + if rename_target is not None: + # perform instance rename to a different name, if we have one configured + RunTest(qa_instance.TestInstanceRename, rename_source, rename_target) + RunTest(qa_instance.TestInstanceRename, rename_target, rename_source) + RunTestIf("rapi", qa_rapi.TestRapiInstanceRename, + rename_source, rename_target) + RunTestIf("rapi", qa_rapi.TestRapiInstanceRename, + rename_target, rename_source) RunTest(qa_instance.TestInstanceStartup, instance) - if qa_config.TestEnabled('tags'): - RunTest(qa_tags.TestInstanceTags, instance) + RunTestIf("tags", qa_tags.TestInstanceTags, instance) + + RunTestIf("cluster-verify", qa_cluster.TestClusterVerify) + + RunTestIf("rapi", qa_rapi.TestInstance, instance) - if qa_config.TestEnabled('node-volumes'): - RunTest(qa_node.TestNodeVolumes) + # Lists instances, too + RunTestIf("node-list", qa_node.TestNodeList) -def RunExportImportTests(instance, pnode): +def RunCommonNodeTests(): + """Run a few common node tests. + + """ + RunTestIf("node-volumes", qa_node.TestNodeVolumes) + RunTestIf("node-storage", qa_node.TestNodeStorage) + RunTestIf("node-oob", qa_node.TestOutOfBand) + + +def RunGroupListTests(): + """Run tests for listing node groups. + + """ + RunTestIf("group-list", qa_group.TestGroupList) + RunTestIf("group-list", qa_group.TestGroupListFields) + + +def RunGroupRwTests(): + """Run tests for adding/removing/renaming groups. + + """ + RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename) + RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions) + RunTestIf("group-rwops", qa_group.TestGroupModify) + RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups) + RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags, + qa_group.GetDefaultGroup()) + + +def RunExportImportTests(instance, pnode, snode): """Tries to export and import the instance. + @param pnode: current primary node of the instance + @param snode: current secondary node of the instance, if any, + otherwise None + """ - if qa_config.TestEnabled('instance-export'): + if qa_config.TestEnabled("instance-export"): + RunTest(qa_instance.TestInstanceExportNoTarget, instance) + expnode = qa_config.AcquireNode(exclude=pnode) try: name = RunTest(qa_instance.TestInstanceExport, instance, expnode) RunTest(qa_instance.TestBackupList, expnode) - if qa_config.TestEnabled('instance-import'): + if qa_config.TestEnabled("instance-import"): newinst = qa_config.AcquireInstance() try: RunTest(qa_instance.TestInstanceImport, pnode, newinst, @@ -171,125 +336,203 @@ def RunExportImportTests(instance, pnode): finally: qa_config.ReleaseNode(expnode) + if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]): + newinst = qa_config.AcquireInstance() + try: + if snode is None: + excl = [pnode] + else: + excl = [pnode, snode] + tnode = qa_config.AcquireNode(exclude=excl) + try: + RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst, + pnode, snode, tnode) + finally: + qa_config.ReleaseNode(tnode) + finally: + qa_config.ReleaseInstance(newinst) -def RunDaemonTests(instance, pnode): + +def RunDaemonTests(instance): """Test the ganeti-watcher script. """ - automatic_restart = \ - qa_config.TestEnabled('instance-automatic-restart') - consecutive_failures = \ - qa_config.TestEnabled('instance-consecutive-failures') - - if automatic_restart or consecutive_failures: - qa_daemon.PrintCronWarning() + RunTest(qa_daemon.TestPauseWatcher) - if automatic_restart: - RunTest(qa_daemon.TestInstanceAutomaticRestart, pnode, instance) + RunTestIf("instance-automatic-restart", + qa_daemon.TestInstanceAutomaticRestart, instance) + RunTestIf("instance-consecutive-failures", + qa_daemon.TestInstanceConsecutiveFailures, instance) - if consecutive_failures: - RunTest(qa_daemon.TestInstanceConsecutiveFailures, pnode, instance) + RunTest(qa_daemon.TestResumeWatcher) def RunHardwareFailureTests(instance, pnode, snode): """Test cluster internal hardware failure recovery. """ - if qa_config.TestEnabled('instance-failover'): - RunTest(qa_instance.TestInstanceFailover, instance) + RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance) + RunTestIf(["instance-failover", "rapi"], + qa_rapi.TestRapiInstanceFailover, instance) - if qa_config.TestEnabled('node-evacuate'): - RunTest(qa_node.TestNodeEvacuate, pnode, snode) + RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance) + RunTestIf(["instance-migrate", "rapi"], + qa_rapi.TestRapiInstanceMigrate, instance) - if qa_config.TestEnabled('node-failover'): - RunTest(qa_node.TestNodeFailover, pnode, snode) + if qa_config.TestEnabled("instance-replace-disks"): + othernode = qa_config.AcquireNode(exclude=[pnode, snode]) + try: + RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance) + RunTest(qa_instance.TestReplaceDisks, + instance, pnode, snode, othernode) + finally: + qa_config.ReleaseNode(othernode) - if qa_config.TestEnabled('instance-disk-failure'): - RunTest(qa_instance.TestInstanceMasterDiskFailure, - instance, pnode, snode) - RunTest(qa_instance.TestInstanceSecondaryDiskFailure, - instance, pnode, snode) + RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, pnode, snode) + RunTestIf("node-failover", qa_node.TestNodeFailover, pnode, snode) -def main(): - """Main program. + RunTestIf("instance-disk-failure", qa_instance.TestInstanceMasterDiskFailure, + instance, pnode, snode) + RunTestIf("instance-disk-failure", + qa_instance.TestInstanceSecondaryDiskFailure, instance, + pnode, snode) - """ - parser = OptionParser(usage="%prog [options] " - "") - parser.add_option('--dry-run', dest='dry_run', - action="store_true", - help="Show what would be done") - parser.add_option('--yes-do-it', dest='yes_do_it', - action="store_true", - help="Really execute the tests") - (qa_config.options, args) = parser.parse_args() - if len(args) == 2: - (config_file, known_hosts_file) = args - else: - parser.error("Not enough arguments.") +def RunQa(): + """Main QA body. - if not qa_config.options.yes_do_it: - print ("Executing this script irreversibly destroys any Ganeti\n" - "configuration on all nodes involved. If you really want\n" - "to start testing, supply the --yes-do-it option.") - sys.exit(1) + """ + rapi_user = "ganeti-qa" + rapi_secret = utils.GenerateSecret() - qa_config.Load(config_file) - qa_utils.LoadHooks() + RunEnvTests() + SetupCluster(rapi_user, rapi_secret) - RunTest(qa_other.UploadKnownHostsFile, known_hosts_file) + # Load RAPI certificate + qa_rapi.Setup(rapi_user, rapi_secret) - RunEnvTests() - SetupCluster() RunClusterTests() RunOsTests() - if qa_config.TestEnabled('tags'): - RunTest(qa_tags.TestClusterTags) + RunTestIf("tags", qa_tags.TestClusterTags) + + RunCommonNodeTests() + RunGroupListTests() + RunGroupRwTests() + + pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode()) + try: + RunTestIf("node-readd", qa_node.TestNodeReadd, pnode) + RunTestIf("node-modify", qa_node.TestNodeModify, pnode) + finally: + qa_config.ReleaseNode(pnode) pnode = qa_config.AcquireNode() try: - if qa_config.TestEnabled('tags'): - RunTest(qa_tags.TestNodeTags, pnode) + RunTestIf("tags", qa_tags.TestNodeTags, pnode) + + if qa_rapi.Enabled(): + RunTest(qa_rapi.TestNode, pnode) - if qa_config.TestEnabled('instance-add-plain-disk'): + if qa_config.TestEnabled("instance-add-plain-disk"): + for use_client in [True, False]: + rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode, + use_client) + RunCommonInstanceTests(rapi_instance) + RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client) + del rapi_instance + + if qa_config.TestEnabled("instance-add-plain-disk"): instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode) RunCommonInstanceTests(instance) - RunExportImportTests(instance, pnode) - RunDaemonTests(instance, pnode) + RunGroupListTests() + RunTestIf("cluster-epo", qa_cluster.TestClusterEpo) + RunExportImportTests(instance, pnode, None) + RunDaemonTests(instance) + RunRepairDiskSizes() RunTest(qa_instance.TestInstanceRemove, instance) del instance - if qa_config.TestEnabled('instance-add-local-mirror-disk'): - instance = RunTest(qa_instance.TestInstanceAddWithLocalMirrorDisk, pnode) - RunCommonInstanceTests(instance) - RunExportImportTests(instance, pnode) - RunTest(qa_instance.TestInstanceRemove, instance) - del instance + multinode_tests = [ + ("instance-add-drbd-disk", + qa_instance.TestInstanceAddWithDrbdDisk), + ] - if qa_config.TestEnabled('instance-add-remote-raid-disk'): - snode = qa_config.AcquireNode(exclude=pnode) - try: - instance = RunTest(qa_instance.TestInstanceAddWithRemoteRaidDisk, - pnode, snode) - RunCommonInstanceTests(instance) - RunExportImportTests(instance, pnode) - RunHardwareFailureTests(instance, pnode, snode) - RunTest(qa_instance.TestInstanceRemove, instance) + for name, func in multinode_tests: + if qa_config.TestEnabled(name): + snode = qa_config.AcquireNode(exclude=pnode) + try: + instance = RunTest(func, pnode, snode) + RunCommonInstanceTests(instance) + RunGroupListTests() + RunTest(qa_group.TestAssignNodesIncludingSplit, + constants.INITIAL_NODE_GROUP_NAME, + pnode["primary"], snode["primary"]) + if qa_config.TestEnabled("instance-convert-disk"): + RunTest(qa_instance.TestInstanceShutdown, instance) + RunTest(qa_instance.TestInstanceConvertDisk, instance, snode) + RunTest(qa_instance.TestInstanceStartup, instance) + RunExportImportTests(instance, pnode, snode) + RunHardwareFailureTests(instance, pnode, snode) + RunRepairDiskSizes() + RunTest(qa_instance.TestInstanceRemove, instance) + del instance + finally: + qa_config.ReleaseNode(snode) + + if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]): + for shutdown in [False, True]: + instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode) + expnode = qa_config.AcquireNode(exclude=pnode) + try: + if shutdown: + # Stop instance before exporting and removing it + RunTest(qa_instance.TestInstanceShutdown, instance) + RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode) + RunTest(qa_instance.TestBackupList, expnode) + finally: + qa_config.ReleaseNode(expnode) + del expnode del instance - finally: - qa_config.ReleaseNode(snode) finally: qa_config.ReleaseNode(pnode) - RunTest(qa_node.TestNodeRemoveAll) + RunTestIf("create-cluster", qa_node.TestNodeRemoveAll) + + RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy) + + +@rapi.client.UsesRapiClient +def main(): + """Main program. + + """ + parser = optparse.OptionParser(usage="%prog [options] ") + parser.add_option("--yes-do-it", dest="yes_do_it", + action="store_true", + help="Really execute the tests") + (qa_config.options, args) = parser.parse_args() + + if len(args) == 1: + (config_file, ) = args + else: + parser.error("Wrong number of arguments.") + + if not qa_config.options.yes_do_it: + print ("Executing this script irreversibly destroys any Ganeti\n" + "configuration on all nodes involved. If you really want\n" + "to start testing, supply the --yes-do-it option.") + sys.exit(1) - if qa_config.TestEnabled('cluster-destroy'): - RunTest(qa_cluster.TestClusterDestroy) + qa_config.Load(config_file) + qa_utils.StartMultiplexer(qa_config.GetMasterNode()["primary"]) + try: + RunQa() + finally: + qa_utils.CloseMultiplexers() -if __name__ == '__main__': +if __name__ == "__main__": main()