| 1 |
1 |
#
|
| 2 |
2 |
#
|
| 3 |
3 |
|
| 4 |
|
# Copyright (C) 2007, 2008, 2009, 2010 Google Inc.
|
|
4 |
# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
|
| 5 |
5 |
#
|
| 6 |
6 |
# This program is free software; you can redistribute it and/or modify
|
| 7 |
7 |
# it under the terms of the GNU General Public License as published by
|
| ... | ... | |
| 35 |
35 |
from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput
|
| 36 |
36 |
|
| 37 |
37 |
|
| 38 |
|
def _InstanceRunning(node, name):
|
|
38 |
def _InstanceRunning(name):
|
| 39 |
39 |
"""Checks whether an instance is running.
|
| 40 |
40 |
|
| 41 |
|
@param node: node the instance runs on
|
| 42 |
|
@param name: full name of the Xen instance
|
|
41 |
@param name: full name of the instance
|
| 43 |
42 |
|
| 44 |
43 |
"""
|
| 45 |
|
cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
|
| 46 |
|
ret = StartSSH(node['primary'], cmd).wait()
|
|
44 |
master = qa_config.GetMasterNode()
|
|
45 |
|
|
46 |
cmd = (utils.ShellQuoteArgs(["gnt-instance", "list", "-o", "status", name]) +
|
|
47 |
' | grep running')
|
|
48 |
ret = StartSSH(master["primary"], cmd).wait()
|
| 47 |
49 |
return ret == 0
|
| 48 |
50 |
|
| 49 |
51 |
|
| 50 |
|
def _XmShutdownInstance(node, name):
|
| 51 |
|
"""Shuts down instance using "xm" and waits for completion.
|
|
52 |
def _ShutdownInstance(name):
|
|
53 |
"""Shuts down instance without recording state and waits for completion.
|
| 52 |
54 |
|
| 53 |
|
@param node: node the instance runs on
|
| 54 |
|
@param name: full name of Xen instance
|
|
55 |
@param name: full name of the instance
|
| 55 |
56 |
|
| 56 |
57 |
"""
|
| 57 |
|
AssertCommand(["xm", "shutdown", name], node=node)
|
|
58 |
AssertCommand(["gnt-instance", "shutdown", "--no-remember", name])
|
| 58 |
59 |
|
| 59 |
|
# Wait up to a minute
|
| 60 |
|
end = time.time() + 60
|
| 61 |
|
while time.time() <= end:
|
| 62 |
|
if not _InstanceRunning(node, name):
|
| 63 |
|
break
|
| 64 |
|
time.sleep(5)
|
| 65 |
|
else:
|
| 66 |
|
raise qa_error.Error("xm shutdown failed")
|
|
60 |
if _InstanceRunning(name):
|
|
61 |
raise qa_error.Error("instance shutdown failed")
|
| 67 |
62 |
|
| 68 |
63 |
|
| 69 |
64 |
def _ResetWatcherDaemon():
|
| ... | ... | |
| 108 |
103 |
AssertMatch(output, r"^.*\bis not paused\b.*")
|
| 109 |
104 |
|
| 110 |
105 |
|
| 111 |
|
def TestInstanceAutomaticRestart(node, instance):
|
|
106 |
def TestInstanceAutomaticRestart(instance):
|
| 112 |
107 |
"""Test automatic restart of instance by ganeti-watcher.
|
| 113 |
108 |
|
| 114 |
109 |
"""
|
| 115 |
110 |
inst_name = qa_utils.ResolveInstanceName(instance["name"])
|
| 116 |
111 |
|
| 117 |
112 |
_ResetWatcherDaemon()
|
| 118 |
|
_XmShutdownInstance(node, inst_name)
|
|
113 |
_ShutdownInstance(inst_name)
|
| 119 |
114 |
|
| 120 |
115 |
_RunWatcherDaemon()
|
| 121 |
116 |
time.sleep(5)
|
| 122 |
117 |
|
| 123 |
|
if not _InstanceRunning(node, inst_name):
|
|
118 |
if not _InstanceRunning(inst_name):
|
| 124 |
119 |
raise qa_error.Error("Daemon didn't restart instance")
|
| 125 |
120 |
|
| 126 |
121 |
AssertCommand(["gnt-instance", "info", inst_name])
|
| 127 |
122 |
|
| 128 |
123 |
|
| 129 |
|
def TestInstanceConsecutiveFailures(node, instance):
|
|
124 |
def TestInstanceConsecutiveFailures(instance):
|
| 130 |
125 |
"""Test five consecutive instance failures.
|
| 131 |
126 |
|
| 132 |
127 |
"""
|
| ... | ... | |
| 135 |
130 |
_ResetWatcherDaemon()
|
| 136 |
131 |
|
| 137 |
132 |
for should_start in ([True] * 5) + [False]:
|
| 138 |
|
_XmShutdownInstance(node, inst_name)
|
|
133 |
_ShutdownInstance(inst_name)
|
| 139 |
134 |
_RunWatcherDaemon()
|
| 140 |
135 |
time.sleep(5)
|
| 141 |
136 |
|
| 142 |
|
if bool(_InstanceRunning(node, inst_name)) != should_start:
|
|
137 |
if bool(_InstanceRunning(inst_name)) != should_start:
|
| 143 |
138 |
if should_start:
|
| 144 |
139 |
msg = "Instance not started when it should"
|
| 145 |
140 |
else:
|