X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/8947cf2b1fb0c219f1549f5c736fa4bc4a0cf5e8..610e39da1ebb3c3fe94f07adb1c4f383cac34f03:/qa/ganeti-qa.py?ds=sidebyside

diff --git a/qa/ganeti-qa.py b/qa/ganeti-qa.py
index 64c0006..4c2a2fc 100755
--- a/qa/ganeti-qa.py
+++ b/qa/ganeti-qa.py
@@ -21,15 +21,11 @@
 
 """Script for doing QA on Ganeti.
 
-You can create the required known_hosts file using ssh-keyscan. It's mandatory
-to use the full name of a node (FQDN). For security reasons, verify the keys
-before using them.
-Example: ssh-keyscan -t rsa node{1,2,3,4}.example.com > known_hosts
 """
 
 import sys
-from datetime import datetime
-from optparse import OptionParser
+import datetime
+import optparse
 
 import qa_cluster
 import qa_config
@@ -38,7 +34,9 @@ import qa_env
 import qa_instance
 import qa_node
 import qa_os
-import qa_other
+import qa_rapi
+import qa_tags
+import qa_utils
 
 
 def RunTest(fn, *args):
@@ -50,7 +48,7 @@ def RunTest(fn, *args):
   else:
     desc = '%r' % fn
 
-  now = str(datetime.now())
+  now = str(datetime.datetime.now())
 
   print
   print '---', now, ('-' * (55 - len(now)))
@@ -60,61 +58,49 @@ def RunTest(fn, *args):
   return fn(*args)
 
 
-def main():
-  """Main program.
+def RunEnvTests():
+  """Run several environment tests.
 
   """
-  parser = OptionParser(usage="%prog [options] <config-file> "
-                              "<known-hosts-file>")
-  parser.add_option('--dry-run', dest='dry_run',
-      action="store_true",
-      help="Show what would be done")
-  parser.add_option('--yes-do-it', dest='yes_do_it',
-      action="store_true",
-      help="Really execute the tests")
-  (qa_config.options, args) = parser.parse_args()
+  if not qa_config.TestEnabled('env'):
+    return
 
-  if len(args) == 2:
-    (config_file, known_hosts_file) = args
-  else:
-    parser.error("Not enough arguments.")
-
-  if not qa_config.options.yes_do_it:
-    print ("Executing this script irreversibly destroys any Ganeti\n"
-           "configuration on all nodes involved. If you really want\n"
-           "to start testing, supply the --yes-do-it option.")
-    sys.exit(1)
+  RunTest(qa_env.TestSshConnection)
+  RunTest(qa_env.TestIcmpPing)
+  RunTest(qa_env.TestGanetiCommands)
 
-  qa_config.Load(config_file)
 
-  RunTest(qa_other.TestUploadKnownHostsFile, known_hosts_file)
+def SetupCluster():
+  """Initializes the cluster.
 
-  if qa_config.TestEnabled('env'):
-    RunTest(qa_env.TestSshConnection)
-    RunTest(qa_env.TestIcmpPing)
-    RunTest(qa_env.TestGanetiCommands)
+  """
+  if qa_config.TestEnabled('create-cluster'):
+    RunTest(qa_cluster.TestClusterInit)
+    RunTest(qa_node.TestNodeAddAll)
+  if qa_config.TestEnabled('node-info'):
+    RunTest(qa_node.TestNodeInfo)
 
-  RunTest(qa_cluster.TestClusterInit)
 
-  RunTest(qa_node.TestNodeAddAll)
+def RunClusterTests():
+  """Runs tests related to gnt-cluster.
 
+  """
   if qa_config.TestEnabled('cluster-verify'):
     RunTest(qa_cluster.TestClusterVerify)
 
+  if qa_config.TestEnabled('cluster-rename'):
+    RunTest(qa_cluster.TestClusterRename)
+
   if qa_config.TestEnabled('cluster-info'):
+    RunTest(qa_cluster.TestClusterVersion)
     RunTest(qa_cluster.TestClusterInfo)
-
-  if qa_config.TestEnabled('cluster-getmaster'):
     RunTest(qa_cluster.TestClusterGetmaster)
 
-  if qa_config.TestEnabled('cluster-version'):
-    RunTest(qa_cluster.TestClusterVersion)
-
   if qa_config.TestEnabled('cluster-copyfile'):
     RunTest(qa_cluster.TestClusterCopyfile)
 
-  if qa_config.TestEnabled('node-info'):
-    RunTest(qa_node.TestNodeInfo)
+  if qa_config.TestEnabled('cluster-command'):
+    RunTest(qa_cluster.TestClusterCommand)
 
   if qa_config.TestEnabled('cluster-burnin'):
     RunTest(qa_cluster.TestClusterBurnin)
@@ -122,121 +108,209 @@ def main():
   if qa_config.TestEnabled('cluster-master-failover'):
     RunTest(qa_cluster.TestClusterMasterFailover)
 
-  if qa_config.TestEnabled('os'):
-    RunTest(qa_os.TestOsList)
-    RunTest(qa_os.TestOsDiagnose)
-    RunTest(qa_os.TestOsValid)
-    RunTest(qa_os.TestOsInvalid)
-    RunTest(qa_os.TestOsPartiallyValid)
+  if qa_rapi.Enabled():
+    RunTest(qa_rapi.TestVersion)
+    RunTest(qa_rapi.TestEmptyCluster)
 
-  node = qa_config.AcquireNode()
-  try:
-    if qa_config.TestEnabled('instance-add-plain-disk'):
-      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, node)
+def RunOsTests():
+  """Runs all tests related to gnt-os.
+
+  """
+  if not qa_config.TestEnabled('os'):
+    return
+
+  RunTest(qa_os.TestOsList)
+  RunTest(qa_os.TestOsDiagnose)
+  RunTest(qa_os.TestOsValid)
+  RunTest(qa_os.TestOsInvalid)
+  RunTest(qa_os.TestOsPartiallyValid)
+
+
+def RunCommonInstanceTests(instance):
+  """Runs a few tests that are common to all disk types.
+
+  """
+  if qa_config.TestEnabled('instance-shutdown'):
+    RunTest(qa_instance.TestInstanceShutdown, instance)
+    RunTest(qa_instance.TestInstanceStartup, instance)
+
+  if qa_config.TestEnabled('instance-list'):
+    RunTest(qa_instance.TestInstanceList)
+
+  if qa_config.TestEnabled('instance-info'):
+    RunTest(qa_instance.TestInstanceInfo, instance)
+
+  if qa_config.TestEnabled('instance-modify'):
+    RunTest(qa_instance.TestInstanceModify, instance)
+
+  if qa_config.TestEnabled('instance-console'):
+    RunTest(qa_instance.TestInstanceConsole, instance)
 
-      if qa_config.TestEnabled('instance-shutdown'):
-        RunTest(qa_instance.TestInstanceShutdown, instance)
-        RunTest(qa_instance.TestInstanceStartup, instance)
+  if qa_config.TestEnabled('instance-reinstall'):
+    RunTest(qa_instance.TestInstanceShutdown, instance)
+    RunTest(qa_instance.TestInstanceReinstall, instance)
+    RunTest(qa_instance.TestInstanceStartup, instance)
 
-      if qa_config.TestEnabled('instance-list'):
-        RunTest(qa_instance.TestInstanceList)
+  if qa_config.TestEnabled('instance-reboot'):
+    RunTest(qa_instance.TestInstanceReboot, instance)
 
-      if qa_config.TestEnabled('instance-info'):
-        RunTest(qa_instance.TestInstanceInfo, instance)
+  if qa_config.TestEnabled('tags'):
+    RunTest(qa_tags.TestInstanceTags, instance)
 
-      automatic_restart = \
-        qa_config.TestEnabled('instance-automatic-restart')
-      consecutive_failures = \
-        qa_config.TestEnabled('instance-consecutive-failures')
+  if qa_config.TestEnabled('node-volumes'):
+    RunTest(qa_node.TestNodeVolumes)
 
-      if automatic_restart or consecutive_failures:
-        qa_daemon.PrintCronWarning()
+  if qa_rapi.Enabled():
+    RunTest(qa_rapi.TestInstance, instance)
 
-        if automatic_restart:
-          RunTest(qa_daemon.TestInstanceAutomaticRestart, node, instance)
+def RunExportImportTests(instance, pnode):
+  """Tries to export and import the instance.
 
-        if consecutive_failures:
-          RunTest(qa_daemon.TestInstanceConsecutiveFailures, node, instance)
+  """
+  if qa_config.TestEnabled('instance-export'):
+    expnode = qa_config.AcquireNode(exclude=pnode)
+    try:
+      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
+
+      RunTest(qa_instance.TestBackupList, expnode)
 
-      if qa_config.TestEnabled('instance-export'):
-        expnode = qa_config.AcquireNode(exclude=node)
+      if qa_config.TestEnabled('instance-import'):
+        newinst = qa_config.AcquireInstance()
         try:
-          name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
-
-          RunTest(qa_instance.TestBackupList, expnode)
-
-          if qa_config.TestEnabled('instance-import'):
-            newinst = qa_config.AcquireInstance()
-            try:
-              RunTest(qa_instance.TestInstanceImport, node, newinst,
-                      expnode, name)
-              RunTest(qa_instance.TestInstanceRemove, newinst)
-            finally:
-              qa_config.ReleaseInstance(newinst)
+          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
+                  expnode, name)
+          RunTest(qa_instance.TestInstanceRemove, newinst)
         finally:
-          qa_config.ReleaseNode(expnode)
+          qa_config.ReleaseInstance(newinst)
+    finally:
+      qa_config.ReleaseNode(expnode)
 
-      if qa_config.TestEnabled('instance-reinstall'):
-        RunTest(qa_instance.TestInstanceShutdown, instance)
-        RunTest(qa_instance.TestInstanceReinstall, instance)
-        RunTest(qa_instance.TestInstanceStartup, instance)
 
-      if qa_config.TestEnabled('node-volumes'):
-        RunTest(qa_node.TestNodeVolumes)
+def RunDaemonTests(instance, pnode):
+  """Test the ganeti-watcher script.
 
-      RunTest(qa_instance.TestInstanceRemove, instance)
-      del instance
+  """
+  automatic_restart = \
+    qa_config.TestEnabled('instance-automatic-restart')
+  consecutive_failures = \
+    qa_config.TestEnabled('instance-consecutive-failures')
 
-    if qa_config.TestEnabled('instance-add-local-mirror-disk'):
-      instance = RunTest(qa_instance.TestInstanceAddWithLocalMirrorDisk, node)
+  if automatic_restart or consecutive_failures:
+    qa_daemon.PrintCronWarning()
 
-      if qa_config.TestEnabled('instance-shutdown'):
-        RunTest(qa_instance.TestInstanceShutdown, instance)
-        RunTest(qa_instance.TestInstanceStartup, instance)
+    if automatic_restart:
+      RunTest(qa_daemon.TestInstanceAutomaticRestart, pnode, instance)
 
-      if qa_config.TestEnabled('instance-info'):
-        RunTest(qa_instance.TestInstanceInfo, instance)
+    if consecutive_failures:
+      RunTest(qa_daemon.TestInstanceConsecutiveFailures, pnode, instance)
 
-      if qa_config.TestEnabled('node-volumes'):
-        RunTest(qa_node.TestNodeVolumes)
 
-      RunTest(qa_instance.TestInstanceRemove, instance)
-      del instance
+def RunHardwareFailureTests(instance, pnode, snode):
+  """Test cluster internal hardware failure recovery.
+
+  """
+  if qa_config.TestEnabled('instance-failover'):
+    RunTest(qa_instance.TestInstanceFailover, instance)
+
+  if qa_config.TestEnabled('instance-replace-disks'):
+    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
+    try:
+      RunTest(qa_instance.TestReplaceDisks,
+              instance, pnode, snode, othernode)
+    finally:
+      qa_config.ReleaseNode(othernode)
 
-    if qa_config.TestEnabled('instance-add-remote-raid-disk'):
-      node2 = qa_config.AcquireNode(exclude=node)
-      try:
-        instance = RunTest(qa_instance.TestInstanceAddWithRemoteRaidDisk,
-                           node, node2)
+  if qa_config.TestEnabled('node-evacuate'):
+    RunTest(qa_node.TestNodeEvacuate, pnode, snode)
 
-        if qa_config.TestEnabled('instance-shutdown'):
-          RunTest(qa_instance.TestInstanceShutdown, instance)
-          RunTest(qa_instance.TestInstanceStartup, instance)
+  if qa_config.TestEnabled('node-failover'):
+    RunTest(qa_node.TestNodeFailover, pnode, snode)
 
-        if qa_config.TestEnabled('instance-info'):
-          RunTest(qa_instance.TestInstanceInfo, instance)
+  if qa_config.TestEnabled('instance-disk-failure'):
+    RunTest(qa_instance.TestInstanceMasterDiskFailure,
+            instance, pnode, snode)
+    RunTest(qa_instance.TestInstanceSecondaryDiskFailure,
+            instance, pnode, snode)
 
-        if qa_config.TestEnabled('instance-failover'):
-          RunTest(qa_instance.TestInstanceFailover, instance)
 
-        if qa_config.TestEnabled('node-evacuate'):
-          RunTest(qa_node.TestNodeEvacuate, node, node2)
+def main():
+  """Main program.
 
-        if qa_config.TestEnabled('node-failover'):
-          RunTest(qa_node.TestNodeFailover, node, node2)
+  """
+  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
+  parser.add_option('--yes-do-it', dest='yes_do_it',
+      action="store_true",
+      help="Really execute the tests")
+  (qa_config.options, args) = parser.parse_args()
 
-        if qa_config.TestEnabled('node-volumes'):
-          RunTest(qa_node.TestNodeVolumes)
+  if len(args) == 1:
+    (config_file, ) = args
+  else:
+    parser.error("Wrong number of arguments.")
 
-        RunTest(qa_instance.TestInstanceRemove, instance)
-        del instance
-      finally:
-        qa_config.ReleaseNode(node2)
+  if not qa_config.options.yes_do_it:
+    print ("Executing this script irreversibly destroys any Ganeti\n"
+           "configuration on all nodes involved. If you really want\n"
+           "to start testing, supply the --yes-do-it option.")
+    sys.exit(1)
+
+  qa_config.Load(config_file)
+
+  RunEnvTests()
+  SetupCluster()
+  RunClusterTests()
+  RunOsTests()
+
+  if qa_config.TestEnabled('tags'):
+    RunTest(qa_tags.TestClusterTags)
+
+  if qa_config.TestEnabled('node-readd'):
+    master = qa_config.GetMasterNode()
+    pnode = qa_config.AcquireNode(exclude=master)
+    try:
+      RunTest(qa_node.TestNodeReadd, pnode)
+    finally:
+      qa_config.ReleaseNode(pnode)
+
+  pnode = qa_config.AcquireNode()
+  try:
+    if qa_config.TestEnabled('tags'):
+      RunTest(qa_tags.TestNodeTags, pnode)
+
+    if qa_rapi.Enabled():
+      RunTest(qa_rapi.TestNode, pnode)
+
+    if qa_config.TestEnabled('instance-add-plain-disk'):
+      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
+      RunCommonInstanceTests(instance)
+      RunExportImportTests(instance, pnode)
+      RunDaemonTests(instance, pnode)
+      RunTest(qa_instance.TestInstanceRemove, instance)
+      del instance
+
+    multinode_tests = [
+      ('instance-add-drbd-disk',
+       qa_instance.TestInstanceAddWithDrbdDisk),
+    ]
+
+    for name, func in multinode_tests:
+      if qa_config.TestEnabled(name):
+        snode = qa_config.AcquireNode(exclude=pnode)
+        try:
+          instance = RunTest(func, pnode, snode)
+          RunCommonInstanceTests(instance)
+          RunExportImportTests(instance, pnode)
+          RunHardwareFailureTests(instance, pnode, snode)
+          RunTest(qa_instance.TestInstanceRemove, instance)
+          del instance
+        finally:
+          qa_config.ReleaseNode(snode)
 
   finally:
-    qa_config.ReleaseNode(node)
+    qa_config.ReleaseNode(pnode)
 
-  RunTest(qa_node.TestNodeRemoveAll)
+  if qa_config.TestEnabled('create-cluster'):
+    RunTest(qa_node.TestNodeRemoveAll)
 
   if qa_config.TestEnabled('cluster-destroy'):
     RunTest(qa_cluster.TestClusterDestroy)