Revision 2f4b4f78 qa/qa_daemon.py
| b/qa/qa_daemon.py | ||
|---|---|---|
| 32 | 32 |
import qa_utils |
| 33 | 33 |
import qa_error |
| 34 | 34 |
|
| 35 |
from qa_utils import AssertEqual, AssertMatch, StartSSH, GetCommandOutput
|
|
| 35 |
from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput
|
|
| 36 | 36 |
|
| 37 | 37 |
|
| 38 | 38 |
def _InstanceRunning(node, name): |
| 39 | 39 |
"""Checks whether an instance is running. |
| 40 | 40 |
|
| 41 |
Args:
|
|
| 42 |
node: Node the instance runs on
|
|
| 43 |
name: Full name of Xen instance |
|
| 41 |
@param node: node the instance runs on
|
|
| 42 |
@param name: full name of the Xen instance
|
|
| 43 |
|
|
| 44 | 44 |
""" |
| 45 | 45 |
cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null' |
| 46 | 46 |
ret = StartSSH(node['primary'], cmd).wait() |
| ... | ... | |
| 50 | 50 |
def _XmShutdownInstance(node, name): |
| 51 | 51 |
"""Shuts down instance using "xm" and waits for completion. |
| 52 | 52 |
|
| 53 |
Args: |
|
| 54 |
node: Node the instance runs on |
|
| 55 |
name: Full name of Xen instance |
|
| 56 |
""" |
|
| 57 |
master = qa_config.GetMasterNode() |
|
| 53 |
@param node: node the instance runs on |
|
| 54 |
@param name: full name of Xen instance |
|
| 58 | 55 |
|
| 59 |
cmd = ['xm', 'shutdown', name] |
|
| 60 |
AssertEqual(StartSSH(node['primary'], |
|
| 61 |
utils.ShellQuoteArgs(cmd)).wait(), 0) |
|
| 56 |
""" |
|
| 57 |
AssertCommand(["xm", "shutdown", name], node=node) |
|
| 62 | 58 |
|
| 63 | 59 |
# Wait up to a minute |
| 64 | 60 |
end = time.time() + 60 |
| ... | ... | |
| 73 | 69 |
def _ResetWatcherDaemon(): |
| 74 | 70 |
"""Removes the watcher daemon's state file. |
| 75 | 71 |
|
| 76 |
Args: |
|
| 77 |
node: Node to be reset |
|
| 78 | 72 |
""" |
| 79 |
master = qa_config.GetMasterNode() |
|
| 80 |
|
|
| 81 |
cmd = ['rm', '-f', constants.WATCHER_STATEFILE] |
|
| 82 |
AssertEqual(StartSSH(master['primary'], |
|
| 83 |
utils.ShellQuoteArgs(cmd)).wait(), 0) |
|
| 73 |
AssertCommand(["rm", "-f", constants.WATCHER_STATEFILE]) |
|
| 84 | 74 |
|
| 85 | 75 |
|
| 86 | 76 |
def _RunWatcherDaemon(): |
| 87 | 77 |
"""Runs the ganeti-watcher daemon on the master node. |
| 88 | 78 |
|
| 89 | 79 |
""" |
| 90 |
master = qa_config.GetMasterNode() |
|
| 91 |
|
|
| 92 |
cmd = ["ganeti-watcher", "-d", "--ignore-pause"] |
|
| 93 |
AssertEqual(StartSSH(master["primary"], |
|
| 94 |
utils.ShellQuoteArgs(cmd)).wait(), 0) |
|
| 80 |
AssertCommand(["ganeti-watcher", "-d", "--ignore-pause"]) |
|
| 95 | 81 |
|
| 96 | 82 |
|
| 97 | 83 |
def TestPauseWatcher(): |
| ... | ... | |
| 100 | 86 |
""" |
| 101 | 87 |
master = qa_config.GetMasterNode() |
| 102 | 88 |
|
| 103 |
cmd = ["gnt-cluster", "watcher", "pause", "4h"] |
|
| 104 |
AssertEqual(StartSSH(master["primary"], |
|
| 105 |
utils.ShellQuoteArgs(cmd)).wait(), 0) |
|
| 89 |
AssertCommand(["gnt-cluster", "watcher", "pause", "4h"]) |
|
| 106 | 90 |
|
| 107 | 91 |
cmd = ["gnt-cluster", "watcher", "info"] |
| 108 | 92 |
output = GetCommandOutput(master["primary"], |
| ... | ... | |
| 116 | 100 |
""" |
| 117 | 101 |
master = qa_config.GetMasterNode() |
| 118 | 102 |
|
| 119 |
cmd = ["gnt-cluster", "watcher", "continue"] |
|
| 120 |
AssertEqual(StartSSH(master["primary"], |
|
| 121 |
utils.ShellQuoteArgs(cmd)).wait(), 0) |
|
| 103 |
AssertCommand(["gnt-cluster", "watcher", "continue"]) |
|
| 122 | 104 |
|
| 123 | 105 |
cmd = ["gnt-cluster", "watcher", "info"] |
| 124 | 106 |
output = GetCommandOutput(master["primary"], |
| ... | ... | |
| 130 | 112 |
"""Test automatic restart of instance by ganeti-watcher. |
| 131 | 113 |
|
| 132 | 114 |
""" |
| 133 |
master = qa_config.GetMasterNode() |
|
| 134 | 115 |
inst_name = qa_utils.ResolveInstanceName(instance["name"]) |
| 135 | 116 |
|
| 136 | 117 |
_ResetWatcherDaemon() |
| ... | ... | |
| 142 | 123 |
if not _InstanceRunning(node, inst_name): |
| 143 | 124 |
raise qa_error.Error("Daemon didn't restart instance")
|
| 144 | 125 |
|
| 145 |
cmd = ['gnt-instance', 'info', inst_name] |
|
| 146 |
AssertEqual(StartSSH(master['primary'], |
|
| 147 |
utils.ShellQuoteArgs(cmd)).wait(), 0) |
|
| 126 |
AssertCommand(["gnt-instance", "info", inst_name]) |
|
| 148 | 127 |
|
| 149 | 128 |
|
| 150 | 129 |
def TestInstanceConsecutiveFailures(node, instance): |
| 151 | 130 |
"""Test five consecutive instance failures. |
| 152 | 131 |
|
| 153 | 132 |
""" |
| 154 |
master = qa_config.GetMasterNode() |
|
| 155 | 133 |
inst_name = qa_utils.ResolveInstanceName(instance["name"]) |
| 156 | 134 |
|
| 157 | 135 |
_ResetWatcherDaemon() |
| ... | ... | |
| 168 | 146 |
msg = "Instance started when it shouldn't" |
| 169 | 147 |
raise qa_error.Error(msg) |
| 170 | 148 |
|
| 171 |
cmd = ['gnt-instance', 'info', inst_name] |
|
| 172 |
AssertEqual(StartSSH(master['primary'], |
|
| 173 |
utils.ShellQuoteArgs(cmd)).wait(), 0) |
|
| 149 |
AssertCommand(["gnt-instance", "info", inst_name]) |
|
Also available in: Unified diff