Statistics
| Branch: | Tag: | Revision:

root / qa / qa_daemon.py @ e8b919a1

History | View | Annotate | Download (3.6 kB)

1 c68d1f43 Michael Hanselmann
#
2 c68d1f43 Michael Hanselmann
#
3 c68d1f43 Michael Hanselmann
4 b998270c Iustin Pop
# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
5 cec9845c Michael Hanselmann
#
6 cec9845c Michael Hanselmann
# This program is free software; you can redistribute it and/or modify
7 cec9845c Michael Hanselmann
# it under the terms of the GNU General Public License as published by
8 cec9845c Michael Hanselmann
# the Free Software Foundation; either version 2 of the License, or
9 cec9845c Michael Hanselmann
# (at your option) any later version.
10 cec9845c Michael Hanselmann
#
11 cec9845c Michael Hanselmann
# This program is distributed in the hope that it will be useful, but
12 cec9845c Michael Hanselmann
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 cec9845c Michael Hanselmann
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 cec9845c Michael Hanselmann
# General Public License for more details.
15 cec9845c Michael Hanselmann
#
16 cec9845c Michael Hanselmann
# You should have received a copy of the GNU General Public License
17 cec9845c Michael Hanselmann
# along with this program; if not, write to the Free Software
18 cec9845c Michael Hanselmann
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 cec9845c Michael Hanselmann
# 02110-1301, USA.
20 cec9845c Michael Hanselmann
21 cec9845c Michael Hanselmann
22 cec9845c Michael Hanselmann
"""Daemon related QA tests.
23 cec9845c Michael Hanselmann

24 cec9845c Michael Hanselmann
"""
25 cec9845c Michael Hanselmann
26 cec9845c Michael Hanselmann
import time
27 cec9845c Michael Hanselmann
28 cec9845c Michael Hanselmann
from ganeti import utils
29 304d9f02 Michael Hanselmann
from ganeti import pathutils
30 cec9845c Michael Hanselmann
31 cec9845c Michael Hanselmann
import qa_config
32 cec9845c Michael Hanselmann
import qa_utils
33 cec9845c Michael Hanselmann
import qa_error
34 cec9845c Michael Hanselmann
35 2f4b4f78 Iustin Pop
from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput
36 cec9845c Michael Hanselmann
37 cec9845c Michael Hanselmann
38 b998270c Iustin Pop
def _InstanceRunning(name):
39 cec9845c Michael Hanselmann
  """Checks whether an instance is running.
40 cec9845c Michael Hanselmann

41 b998270c Iustin Pop
  @param name: full name of the instance
42 2f4b4f78 Iustin Pop

43 cec9845c Michael Hanselmann
  """
44 b998270c Iustin Pop
  master = qa_config.GetMasterNode()
45 b998270c Iustin Pop
46 b998270c Iustin Pop
  cmd = (utils.ShellQuoteArgs(["gnt-instance", "list", "-o", "status", name]) +
47 b998270c Iustin Pop
         ' | grep running')
48 b998270c Iustin Pop
  ret = StartSSH(master["primary"], cmd).wait()
49 cec9845c Michael Hanselmann
  return ret == 0
50 cec9845c Michael Hanselmann
51 cec9845c Michael Hanselmann
52 b998270c Iustin Pop
def _ShutdownInstance(name):
53 b998270c Iustin Pop
  """Shuts down instance without recording state and waits for completion.
54 cec9845c Michael Hanselmann

55 b998270c Iustin Pop
  @param name: full name of the instance
56 cec9845c Michael Hanselmann

57 2f4b4f78 Iustin Pop
  """
58 b998270c Iustin Pop
  AssertCommand(["gnt-instance", "shutdown", "--no-remember", name])
59 cec9845c Michael Hanselmann
60 b998270c Iustin Pop
  if _InstanceRunning(name):
61 b998270c Iustin Pop
    raise qa_error.Error("instance shutdown failed")
62 cec9845c Michael Hanselmann
63 cec9845c Michael Hanselmann
64 28a6fbc8 Michael Hanselmann
def _ResetWatcherDaemon():
65 cec9845c Michael Hanselmann
  """Removes the watcher daemon's state file.
66 cec9845c Michael Hanselmann

67 cec9845c Michael Hanselmann
  """
68 9d7b1e53 Michael Hanselmann
  AssertCommand([
69 9d7b1e53 Michael Hanselmann
    "bash", "-c",
70 3c286190 Dimitris Aragiorgis
    "rm -vf %s" % (pathutils.WATCHER_GROUP_STATE_FILE % "*-*-*-*"),
71 9d7b1e53 Michael Hanselmann
    ])
72 28a6fbc8 Michael Hanselmann
73 28a6fbc8 Michael Hanselmann
74 28a6fbc8 Michael Hanselmann
def _RunWatcherDaemon():
75 28a6fbc8 Michael Hanselmann
  """Runs the ganeti-watcher daemon on the master node.
76 28a6fbc8 Michael Hanselmann

77 28a6fbc8 Michael Hanselmann
  """
78 16e0b9c9 Michael Hanselmann
  AssertCommand(["ganeti-watcher", "-d", "--ignore-pause", "--wait-children"])
79 cec9845c Michael Hanselmann
80 cec9845c Michael Hanselmann
81 8201b996 Iustin Pop
def TestPauseWatcher():
82 8201b996 Iustin Pop
  """Tests and pauses the watcher.
83 23269c5b Michael Hanselmann

84 23269c5b Michael Hanselmann
  """
85 8201b996 Iustin Pop
  master = qa_config.GetMasterNode()
86 8201b996 Iustin Pop
87 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-cluster", "watcher", "pause", "4h"])
88 8201b996 Iustin Pop
89 8201b996 Iustin Pop
  cmd = ["gnt-cluster", "watcher", "info"]
90 8201b996 Iustin Pop
  output = GetCommandOutput(master["primary"],
91 8201b996 Iustin Pop
                            utils.ShellQuoteArgs(cmd))
92 8201b996 Iustin Pop
  AssertMatch(output, r"^.*\bis paused\b.*")
93 8201b996 Iustin Pop
94 8201b996 Iustin Pop
95 8201b996 Iustin Pop
def TestResumeWatcher():
96 8201b996 Iustin Pop
  """Tests and unpauses the watcher.
97 8201b996 Iustin Pop

98 8201b996 Iustin Pop
  """
99 8201b996 Iustin Pop
  master = qa_config.GetMasterNode()
100 8201b996 Iustin Pop
101 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-cluster", "watcher", "continue"])
102 8201b996 Iustin Pop
103 8201b996 Iustin Pop
  cmd = ["gnt-cluster", "watcher", "info"]
104 8201b996 Iustin Pop
  output = GetCommandOutput(master["primary"],
105 8201b996 Iustin Pop
                            utils.ShellQuoteArgs(cmd))
106 8201b996 Iustin Pop
  AssertMatch(output, r"^.*\bis not paused\b.*")
107 23269c5b Michael Hanselmann
108 23269c5b Michael Hanselmann
109 b998270c Iustin Pop
def TestInstanceAutomaticRestart(instance):
110 cec9845c Michael Hanselmann
  """Test automatic restart of instance by ganeti-watcher.
111 cec9845c Michael Hanselmann

112 cec9845c Michael Hanselmann
  """
113 46f9a948 Michael Hanselmann
  inst_name = qa_utils.ResolveInstanceName(instance["name"])
114 cec9845c Michael Hanselmann
115 28a6fbc8 Michael Hanselmann
  _ResetWatcherDaemon()
116 b998270c Iustin Pop
  _ShutdownInstance(inst_name)
117 cec9845c Michael Hanselmann
118 28a6fbc8 Michael Hanselmann
  _RunWatcherDaemon()
119 28a6fbc8 Michael Hanselmann
  time.sleep(5)
120 cec9845c Michael Hanselmann
121 b998270c Iustin Pop
  if not _InstanceRunning(inst_name):
122 28a6fbc8 Michael Hanselmann
    raise qa_error.Error("Daemon didn't restart instance")
123 cec9845c Michael Hanselmann
124 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-instance", "info", inst_name])
125 cec9845c Michael Hanselmann
126 cec9845c Michael Hanselmann
127 b998270c Iustin Pop
def TestInstanceConsecutiveFailures(instance):
128 cec9845c Michael Hanselmann
  """Test five consecutive instance failures.
129 cec9845c Michael Hanselmann

130 cec9845c Michael Hanselmann
  """
131 46f9a948 Michael Hanselmann
  inst_name = qa_utils.ResolveInstanceName(instance["name"])
132 cec9845c Michael Hanselmann
133 28a6fbc8 Michael Hanselmann
  _ResetWatcherDaemon()
134 cec9845c Michael Hanselmann
135 28a6fbc8 Michael Hanselmann
  for should_start in ([True] * 5) + [False]:
136 b998270c Iustin Pop
    _ShutdownInstance(inst_name)
137 28a6fbc8 Michael Hanselmann
    _RunWatcherDaemon()
138 28a6fbc8 Michael Hanselmann
    time.sleep(5)
139 cec9845c Michael Hanselmann
140 b998270c Iustin Pop
    if bool(_InstanceRunning(inst_name)) != should_start:
141 28a6fbc8 Michael Hanselmann
      if should_start:
142 28a6fbc8 Michael Hanselmann
        msg = "Instance not started when it should"
143 28a6fbc8 Michael Hanselmann
      else:
144 28a6fbc8 Michael Hanselmann
        msg = "Instance started when it shouldn't"
145 28a6fbc8 Michael Hanselmann
      raise qa_error.Error(msg)
146 cec9845c Michael Hanselmann
147 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-instance", "info", inst_name])