Fix DTS_EXT_MIRROR migration

[ganeti-local] / qa / qa_daemon.py

diff --git a/qa/qa_daemon.py b/qa/qa_daemon.py
index 8b40edf..1f47357 100644 (file)
--- a/qa/qa_daemon.py
+++ b/qa/qa_daemon.py
@@ -1,4 +1,7 @@
-# Copyright (C) 2007 Google Inc.
+#
+#
+
+# Copyright (C) 2007, 2008, 2009, 2010 Google Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -29,15 +32,15 @@ import qa_config
 import qa_utils
 import qa_error
 
-from qa_utils import AssertEqual, StartSSH
+from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput
 
 
 def _InstanceRunning(node, name):
   """Checks whether an instance is running.
 
-  Args:
-    node: Node the instance runs on
-    name: Full name of Xen instance
+  @param node: node the instance runs on
+  @param name: full name of the Xen instance
+
   """
   cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
   ret = StartSSH(node['primary'], cmd).wait()
@@ -47,15 +50,11 @@ def _InstanceRunning(node, name):
 def _XmShutdownInstance(node, name):
   """Shuts down instance using "xm" and waits for completion.
 
-  Args:
-    node: Node the instance runs on
-    name: Full name of Xen instance
-  """
-  master = qa_config.GetMasterNode()
+  @param node: node the instance runs on
+  @param name: full name of Xen instance
 
-  cmd = ['xm', 'shutdown', name]
-  AssertEqual(StartSSH(node['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+  """
+  AssertCommand(["xm", "shutdown", name], node=node)
 
   # Wait up to a minute
   end = time.time() + 60
@@ -67,79 +66,84 @@ def _XmShutdownInstance(node, name):
     raise qa_error.Error("xm shutdown failed")
 
 
-def _ResetWatcherDaemon(node):
+def _ResetWatcherDaemon():
   """Removes the watcher daemon's state file.
 
-  Args:
-    node: Node to be reset
   """
-  cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
-  AssertEqual(StartSSH(node['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+  AssertCommand(["rm", "-f", constants.WATCHER_STATEFILE])
+
+
+def _RunWatcherDaemon():
+  """Runs the ganeti-watcher daemon on the master node.
+
+  """
+  AssertCommand(["ganeti-watcher", "-d", "--ignore-pause"])
+
+
+def TestPauseWatcher():
+  """Tests and pauses the watcher.
+
+  """
+  master = qa_config.GetMasterNode()
+
+  AssertCommand(["gnt-cluster", "watcher", "pause", "4h"])
+
+  cmd = ["gnt-cluster", "watcher", "info"]
+  output = GetCommandOutput(master["primary"],
+                            utils.ShellQuoteArgs(cmd))
+  AssertMatch(output, r"^.*\bis paused\b.*")
 
 
-def PrintCronWarning():
-  """Shows a warning about the required cron job.
+def TestResumeWatcher():
+  """Tests and unpauses the watcher.
 
   """
-  print
-  qa_utils.PrintWarning("The following tests require the cron script for"
-                        " ganeti-watcher to be set up.")
+  master = qa_config.GetMasterNode()
+
+  AssertCommand(["gnt-cluster", "watcher", "continue"])
+
+  cmd = ["gnt-cluster", "watcher", "info"]
+  output = GetCommandOutput(master["primary"],
+                            utils.ShellQuoteArgs(cmd))
+  AssertMatch(output, r"^.*\bis not paused\b.*")
 
 
 def TestInstanceAutomaticRestart(node, instance):
   """Test automatic restart of instance by ganeti-watcher.
 
-  Note: takes up to 6 minutes to complete.
   """
-  master = qa_config.GetMasterNode()
-  inst_name = qa_utils.ResolveInstanceName(instance)
+  inst_name = qa_utils.ResolveInstanceName(instance["name"])
 
-  _ResetWatcherDaemon(node)
+  _ResetWatcherDaemon()
   _XmShutdownInstance(node, inst_name)
 
-  # Give it a bit more than five minutes to start again
-  restart_at = time.time() + 330
+  _RunWatcherDaemon()
+  time.sleep(5)
 
-  # Wait until it's running again
-  while time.time() <= restart_at:
-    if _InstanceRunning(node, inst_name):
-      break
-    time.sleep(15)
-  else:
-    raise qa_error.Error("Daemon didn't restart instance in time")
+  if not _InstanceRunning(node, inst_name):
+    raise qa_error.Error("Daemon didn't restart instance")
 
-  cmd = ['gnt-instance', 'info', inst_name]
-  AssertEqual(StartSSH(master['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+  AssertCommand(["gnt-instance", "info", inst_name])
 
 
 def TestInstanceConsecutiveFailures(node, instance):
   """Test five consecutive instance failures.
 
-  Note: takes at least 35 minutes to complete.
   """
-  master = qa_config.GetMasterNode()
-  inst_name = qa_utils.ResolveInstanceName(instance)
+  inst_name = qa_utils.ResolveInstanceName(instance["name"])
 
-  _ResetWatcherDaemon(node)
-  _XmShutdownInstance(node, inst_name)
-
-  # Do shutdowns for 30 minutes
-  finished_at = time.time() + (35 * 60)
+  _ResetWatcherDaemon()
 
-  while time.time() <= finished_at:
-    if _InstanceRunning(node, inst_name):
-      _XmShutdownInstance(node, inst_name)
-    time.sleep(30)
+  for should_start in ([True] * 5) + [False]:
+    _XmShutdownInstance(node, inst_name)
+    _RunWatcherDaemon()
+    time.sleep(5)
 
-  # Check for some time whether the instance doesn't start again
-  check_until = time.time() + 330
-  while time.time() <= check_until:
-    if _InstanceRunning(node, inst_name):
-      raise qa_error.Error("Instance started when it shouldn't")
-    time.sleep(30)
+    if bool(_InstanceRunning(node, inst_name)) != should_start:
+      if should_start:
+        msg = "Instance not started when it should"
+      else:
+        msg = "Instance started when it shouldn't"
+      raise qa_error.Error(msg)
 
-  cmd = ['gnt-instance', 'info', inst_name]
-  AssertEqual(StartSSH(master['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+  AssertCommand(["gnt-instance", "info", inst_name])