#
#
-# Copyright (C) 2007 Google Inc.
+# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
import time
from ganeti import utils
-from ganeti import constants
+from ganeti import pathutils
import qa_config
import qa_utils
import qa_error
-from qa_utils import AssertEqual, StartSSH
+from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput
-def _InstanceRunning(node, name):
+def _InstanceRunning(name):
"""Checks whether an instance is running.
- Args:
- node: Node the instance runs on
- name: Full name of Xen instance
+ @param name: full name of the instance
+
"""
- cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
- ret = StartSSH(node['primary'], cmd).wait()
+ master = qa_config.GetMasterNode()
+
+ cmd = (utils.ShellQuoteArgs(["gnt-instance", "list", "-o", "status", name]) +
+ ' | grep running')
+ ret = StartSSH(master.primary, cmd).wait()
return ret == 0
-def _XmShutdownInstance(node, name):
- """Shuts down instance using "xm" and waits for completion.
+def _ShutdownInstance(name):
+ """Shuts down instance without recording state and waits for completion.
+
+ @param name: full name of the instance
- Args:
- node: Node the instance runs on
- name: Full name of Xen instance
"""
- master = qa_config.GetMasterNode()
+ AssertCommand(["gnt-instance", "shutdown", "--no-remember", name])
- cmd = ['xm', 'shutdown', name]
- AssertEqual(StartSSH(node['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ if _InstanceRunning(name):
+ raise qa_error.Error("instance shutdown failed")
- # Wait up to a minute
- end = time.time() + 60
- while time.time() <= end:
- if not _InstanceRunning(node, name):
- break
- time.sleep(5)
- else:
- raise qa_error.Error("xm shutdown failed")
+
+def _StartInstance(name):
+ """Starts instance and waits for completion.
+
+ @param name: full name of the instance
+
+ """
+ AssertCommand(["gnt-instance", "start", name])
+
+ if not bool(_InstanceRunning(name)):
+ raise qa_error.Error("instance start failed")
def _ResetWatcherDaemon():
"""Removes the watcher daemon's state file.
- Args:
- node: Node to be reset
"""
- master = qa_config.GetMasterNode()
+ path = \
+ qa_utils.MakeNodePath(qa_config.GetMasterNode(),
+ pathutils.WATCHER_GROUP_STATE_FILE % "*-*-*-*")
- cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
- AssertEqual(StartSSH(master['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ AssertCommand(["bash", "-c", "rm -vf %s" % path])
def _RunWatcherDaemon():
"""Runs the ganeti-watcher daemon on the master node.
"""
+ AssertCommand(["ganeti-watcher", "-d", "--ignore-pause", "--wait-children"])
+
+
+def TestPauseWatcher():
+ """Tests and pauses the watcher.
+
+ """
master = qa_config.GetMasterNode()
- cmd = ['ganeti-watcher', '-d']
- AssertEqual(StartSSH(master['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ AssertCommand(["gnt-cluster", "watcher", "pause", "4h"])
+
+ cmd = ["gnt-cluster", "watcher", "info"]
+ output = GetCommandOutput(master.primary,
+ utils.ShellQuoteArgs(cmd))
+ AssertMatch(output, r"^.*\bis paused\b.*")
-def PrintCronWarning():
- """Shows a warning about the cron job.
+def TestResumeWatcher():
+ """Tests and unpauses the watcher.
"""
- msg = ("For the following tests it's recommended to turn off the"
- " ganeti-watcher cronjob.")
- print
- print qa_utils.FormatWarning(msg)
+ master = qa_config.GetMasterNode()
+ AssertCommand(["gnt-cluster", "watcher", "continue"])
-@qa_utils.DefineHook('daemon-automatic-restart')
-def TestInstanceAutomaticRestart(node, instance):
+ cmd = ["gnt-cluster", "watcher", "info"]
+ output = GetCommandOutput(master.primary,
+ utils.ShellQuoteArgs(cmd))
+ AssertMatch(output, r"^.*\bis not paused\b.*")
+
+
+def TestInstanceAutomaticRestart(instance):
"""Test automatic restart of instance by ganeti-watcher.
"""
- master = qa_config.GetMasterNode()
- inst_name = qa_utils.ResolveInstanceName(instance)
+ inst_name = qa_utils.ResolveInstanceName(instance.name)
_ResetWatcherDaemon()
- _XmShutdownInstance(node, inst_name)
+ _ShutdownInstance(inst_name)
_RunWatcherDaemon()
time.sleep(5)
- if not _InstanceRunning(node, inst_name):
+ if not _InstanceRunning(inst_name):
raise qa_error.Error("Daemon didn't restart instance")
- cmd = ['gnt-instance', 'info', inst_name]
- AssertEqual(StartSSH(master['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ AssertCommand(["gnt-instance", "info", inst_name])
-@qa_utils.DefineHook('daemon-consecutive-failures')
-def TestInstanceConsecutiveFailures(node, instance):
+def TestInstanceConsecutiveFailures(instance):
"""Test five consecutive instance failures.
"""
- master = qa_config.GetMasterNode()
- inst_name = qa_utils.ResolveInstanceName(instance)
+ inst_name = qa_utils.ResolveInstanceName(instance.name)
+ inst_was_running = bool(_InstanceRunning(inst_name))
_ResetWatcherDaemon()
for should_start in ([True] * 5) + [False]:
- _XmShutdownInstance(node, inst_name)
+ _ShutdownInstance(inst_name)
_RunWatcherDaemon()
time.sleep(5)
- if bool(_InstanceRunning(node, inst_name)) != should_start:
+ if bool(_InstanceRunning(inst_name)) != should_start:
if should_start:
msg = "Instance not started when it should"
else:
msg = "Instance started when it shouldn't"
raise qa_error.Error(msg)
- cmd = ['gnt-instance', 'info', inst_name]
- AssertEqual(StartSSH(master['primary'],
- utils.ShellQuoteArgs(cmd)).wait(), 0)
+ AssertCommand(["gnt-instance", "info", inst_name])
+
+ if inst_was_running:
+ _StartInstance(inst_name)