-# Copyright (C) 2007 Google Inc.
+#
+#
+
+# Copyright (C) 2007, 2008, 2009, 2010 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
"""
import time
-import subprocess
from ganeti import utils
from ganeti import constants
import qa_utils
import qa_error
-from qa_utils import AssertEqual, StartSSH
-
-
-def _ResolveInstanceName(instance):
- """Gets the full Xen name of an instance.
-
- """
- master = qa_config.GetMasterNode()
-
- info_cmd = utils.ShellQuoteArgs(['gnt-instance', 'info', instance['name']])
- sed_cmd = utils.ShellQuoteArgs(['sed', '-n', '-e', 's/^Instance name: *//p'])
-
- cmd = '%s | %s' % (info_cmd, sed_cmd)
- ssh_cmd = qa_utils.GetSSHCommand(master['primary'], cmd)
- p = subprocess.Popen(ssh_cmd, shell=False, stdout=subprocess.PIPE)
- AssertEqual(p.wait(), 0)
-
- return p.stdout.read().strip()
+from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput
def _InstanceRunning(node, name):
"""Checks whether an instance is running.
- Args:
- node: Node the instance runs on
- name: Full name of Xen instance
+ @param node: node the instance runs on
+ @param name: full name of the Xen instance
+
"""
cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
ret = StartSSH(node['primary'], cmd).wait()
def _XmShutdownInstance(node, name):
"""Shuts down instance using "xm" and waits for completion.
- Args:
- node: Node the instance runs on
- name: Full name of Xen instance
- """
- master = qa_config.GetMasterNode()
+ @param node: node the instance runs on
+ @param name: full name of Xen instance
- cmd = ['xm', 'shutdown', name]
- AssertEqual(StartSSH(master['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ """
+ AssertCommand(["xm", "shutdown", name], node=node)
# Wait up to a minute
end = time.time() + 60
raise qa_error.Error("xm shutdown failed")
-def _ResetWatcherDaemon(node):
+def _ResetWatcherDaemon():
"""Removes the watcher daemon's state file.
- Args:
- node: Node to be reset
"""
- cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
- AssertEqual(StartSSH(node['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ AssertCommand(["rm", "-f", constants.WATCHER_STATEFILE])
+
+
+def _RunWatcherDaemon():
+ """Runs the ganeti-watcher daemon on the master node.
+
+ """
+ AssertCommand(["ganeti-watcher", "-d", "--ignore-pause"])
+
+
+def TestPauseWatcher():
+ """Tests and pauses the watcher.
+
+ """
+ master = qa_config.GetMasterNode()
+
+ AssertCommand(["gnt-cluster", "watcher", "pause", "4h"])
+
+ cmd = ["gnt-cluster", "watcher", "info"]
+ output = GetCommandOutput(master["primary"],
+ utils.ShellQuoteArgs(cmd))
+ AssertMatch(output, r"^.*\bis paused\b.*")
+
+
+def TestResumeWatcher():
+ """Tests and unpauses the watcher.
+
+ """
+ master = qa_config.GetMasterNode()
+
+ AssertCommand(["gnt-cluster", "watcher", "continue"])
+
+ cmd = ["gnt-cluster", "watcher", "info"]
+ output = GetCommandOutput(master["primary"],
+ utils.ShellQuoteArgs(cmd))
+ AssertMatch(output, r"^.*\bis not paused\b.*")
def TestInstanceAutomaticRestart(node, instance):
"""Test automatic restart of instance by ganeti-watcher.
- Note: takes up to 6 minutes to complete.
"""
- master = qa_config.GetMasterNode()
- inst_name = _ResolveInstanceName(instance)
+ inst_name = qa_utils.ResolveInstanceName(instance["name"])
- _ResetWatcherDaemon(node)
+ _ResetWatcherDaemon()
_XmShutdownInstance(node, inst_name)
- # Give it a bit more than five minutes to start again
- restart_at = time.time() + 330
+ _RunWatcherDaemon()
+ time.sleep(5)
- # Wait until it's running again
- while time.time() <= restart_at:
- if _InstanceRunning(node, inst_name):
- break
- time.sleep(15)
- else:
- raise qa_error.Error("Daemon didn't restart instance in time")
+ if not _InstanceRunning(node, inst_name):
+ raise qa_error.Error("Daemon didn't restart instance")
- cmd = ['gnt-instance', 'info', inst_name]
- AssertEqual(StartSSH(master['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ AssertCommand(["gnt-instance", "info", inst_name])
def TestInstanceConsecutiveFailures(node, instance):
"""Test five consecutive instance failures.
- Note: takes at least 35 minutes to complete.
"""
- master = qa_config.GetMasterNode()
- inst_name = _ResolveInstanceName(instance)
+ inst_name = qa_utils.ResolveInstanceName(instance["name"])
- _ResetWatcherDaemon(node)
- _XmShutdownInstance(node, inst_name)
+ _ResetWatcherDaemon()
- # Do shutdowns for 30 minutes
- finished_at = time.time() + (35 * 60)
-
- while time.time() <= finished_at:
- if _InstanceRunning(node, inst_name):
- _XmShutdownInstance(node, inst_name)
- time.sleep(30)
+ for should_start in ([True] * 5) + [False]:
+ _XmShutdownInstance(node, inst_name)
+ _RunWatcherDaemon()
+ time.sleep(5)
- # Check for some time whether the instance doesn't start again
- check_until = time.time() + 330
- while time.time() <= check_until:
- if _InstanceRunning(node, inst_name):
- raise qa_error.Error("Instance started when it shouldn't")
- time.sleep(30)
+ if bool(_InstanceRunning(node, inst_name)) != should_start:
+ if should_start:
+ msg = "Instance not started when it should"
+ else:
+ msg = "Instance started when it shouldn't"
+ raise qa_error.Error(msg)
- cmd = ['gnt-instance', 'info', inst_name]
- AssertEqual(StartSSH(master['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ AssertCommand(["gnt-instance", "info", inst_name])