Statistics
| Branch: | Tag: | Revision:

root / qa / qa_daemon.py @ 2e5eb96a

History | View | Annotate | Download (3.6 kB)

1 c68d1f43 Michael Hanselmann
#
2 c68d1f43 Michael Hanselmann
#
3 c68d1f43 Michael Hanselmann
4 b998270c Iustin Pop
# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
5 cec9845c Michael Hanselmann
#
6 cec9845c Michael Hanselmann
# This program is free software; you can redistribute it and/or modify
7 cec9845c Michael Hanselmann
# it under the terms of the GNU General Public License as published by
8 cec9845c Michael Hanselmann
# the Free Software Foundation; either version 2 of the License, or
9 cec9845c Michael Hanselmann
# (at your option) any later version.
10 cec9845c Michael Hanselmann
#
11 cec9845c Michael Hanselmann
# This program is distributed in the hope that it will be useful, but
12 cec9845c Michael Hanselmann
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 cec9845c Michael Hanselmann
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 cec9845c Michael Hanselmann
# General Public License for more details.
15 cec9845c Michael Hanselmann
#
16 cec9845c Michael Hanselmann
# You should have received a copy of the GNU General Public License
17 cec9845c Michael Hanselmann
# along with this program; if not, write to the Free Software
18 cec9845c Michael Hanselmann
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 cec9845c Michael Hanselmann
# 02110-1301, USA.
20 cec9845c Michael Hanselmann
21 cec9845c Michael Hanselmann
22 cec9845c Michael Hanselmann
"""Daemon related QA tests.
23 cec9845c Michael Hanselmann

24 cec9845c Michael Hanselmann
"""
25 cec9845c Michael Hanselmann
26 cec9845c Michael Hanselmann
import time
27 cec9845c Michael Hanselmann
28 cec9845c Michael Hanselmann
from ganeti import utils
29 cec9845c Michael Hanselmann
from ganeti import constants
30 cec9845c Michael Hanselmann
31 cec9845c Michael Hanselmann
import qa_config
32 cec9845c Michael Hanselmann
import qa_utils
33 cec9845c Michael Hanselmann
import qa_error
34 cec9845c Michael Hanselmann
35 2f4b4f78 Iustin Pop
from qa_utils import AssertMatch, AssertCommand, StartSSH, GetCommandOutput
36 cec9845c Michael Hanselmann
37 cec9845c Michael Hanselmann
38 b998270c Iustin Pop
def _InstanceRunning(name):
39 cec9845c Michael Hanselmann
  """Checks whether an instance is running.
40 cec9845c Michael Hanselmann

41 b998270c Iustin Pop
  @param name: full name of the instance
42 2f4b4f78 Iustin Pop

43 cec9845c Michael Hanselmann
  """
44 b998270c Iustin Pop
  master = qa_config.GetMasterNode()
45 b998270c Iustin Pop
46 b998270c Iustin Pop
  cmd = (utils.ShellQuoteArgs(["gnt-instance", "list", "-o", "status", name]) +
47 b998270c Iustin Pop
         ' | grep running')
48 b998270c Iustin Pop
  ret = StartSSH(master["primary"], cmd).wait()
49 cec9845c Michael Hanselmann
  return ret == 0
50 cec9845c Michael Hanselmann
51 cec9845c Michael Hanselmann
52 b998270c Iustin Pop
def _ShutdownInstance(name):
53 b998270c Iustin Pop
  """Shuts down instance without recording state and waits for completion.
54 cec9845c Michael Hanselmann

55 b998270c Iustin Pop
  @param name: full name of the instance
56 cec9845c Michael Hanselmann

57 2f4b4f78 Iustin Pop
  """
58 b998270c Iustin Pop
  AssertCommand(["gnt-instance", "shutdown", "--no-remember", name])
59 cec9845c Michael Hanselmann
60 b998270c Iustin Pop
  if _InstanceRunning(name):
61 b998270c Iustin Pop
    raise qa_error.Error("instance shutdown failed")
62 cec9845c Michael Hanselmann
63 cec9845c Michael Hanselmann
64 28a6fbc8 Michael Hanselmann
def _ResetWatcherDaemon():
65 cec9845c Michael Hanselmann
  """Removes the watcher daemon's state file.
66 cec9845c Michael Hanselmann

67 cec9845c Michael Hanselmann
  """
68 2f4b4f78 Iustin Pop
  AssertCommand(["rm", "-f", constants.WATCHER_STATEFILE])
69 28a6fbc8 Michael Hanselmann
70 28a6fbc8 Michael Hanselmann
71 28a6fbc8 Michael Hanselmann
def _RunWatcherDaemon():
72 28a6fbc8 Michael Hanselmann
  """Runs the ganeti-watcher daemon on the master node.
73 28a6fbc8 Michael Hanselmann

74 28a6fbc8 Michael Hanselmann
  """
75 2f4b4f78 Iustin Pop
  AssertCommand(["ganeti-watcher", "-d", "--ignore-pause"])
76 cec9845c Michael Hanselmann
77 cec9845c Michael Hanselmann
78 8201b996 Iustin Pop
def TestPauseWatcher():
79 8201b996 Iustin Pop
  """Tests and pauses the watcher.
80 23269c5b Michael Hanselmann

81 23269c5b Michael Hanselmann
  """
82 8201b996 Iustin Pop
  master = qa_config.GetMasterNode()
83 8201b996 Iustin Pop
84 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-cluster", "watcher", "pause", "4h"])
85 8201b996 Iustin Pop
86 8201b996 Iustin Pop
  cmd = ["gnt-cluster", "watcher", "info"]
87 8201b996 Iustin Pop
  output = GetCommandOutput(master["primary"],
88 8201b996 Iustin Pop
                            utils.ShellQuoteArgs(cmd))
89 8201b996 Iustin Pop
  AssertMatch(output, r"^.*\bis paused\b.*")
90 8201b996 Iustin Pop
91 8201b996 Iustin Pop
92 8201b996 Iustin Pop
def TestResumeWatcher():
93 8201b996 Iustin Pop
  """Tests and unpauses the watcher.
94 8201b996 Iustin Pop

95 8201b996 Iustin Pop
  """
96 8201b996 Iustin Pop
  master = qa_config.GetMasterNode()
97 8201b996 Iustin Pop
98 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-cluster", "watcher", "continue"])
99 8201b996 Iustin Pop
100 8201b996 Iustin Pop
  cmd = ["gnt-cluster", "watcher", "info"]
101 8201b996 Iustin Pop
  output = GetCommandOutput(master["primary"],
102 8201b996 Iustin Pop
                            utils.ShellQuoteArgs(cmd))
103 8201b996 Iustin Pop
  AssertMatch(output, r"^.*\bis not paused\b.*")
104 23269c5b Michael Hanselmann
105 23269c5b Michael Hanselmann
106 b998270c Iustin Pop
def TestInstanceAutomaticRestart(instance):
107 cec9845c Michael Hanselmann
  """Test automatic restart of instance by ganeti-watcher.
108 cec9845c Michael Hanselmann

109 cec9845c Michael Hanselmann
  """
110 46f9a948 Michael Hanselmann
  inst_name = qa_utils.ResolveInstanceName(instance["name"])
111 cec9845c Michael Hanselmann
112 28a6fbc8 Michael Hanselmann
  _ResetWatcherDaemon()
113 b998270c Iustin Pop
  _ShutdownInstance(inst_name)
114 cec9845c Michael Hanselmann
115 28a6fbc8 Michael Hanselmann
  _RunWatcherDaemon()
116 28a6fbc8 Michael Hanselmann
  time.sleep(5)
117 cec9845c Michael Hanselmann
118 b998270c Iustin Pop
  if not _InstanceRunning(inst_name):
119 28a6fbc8 Michael Hanselmann
    raise qa_error.Error("Daemon didn't restart instance")
120 cec9845c Michael Hanselmann
121 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-instance", "info", inst_name])
122 cec9845c Michael Hanselmann
123 cec9845c Michael Hanselmann
124 b998270c Iustin Pop
def TestInstanceConsecutiveFailures(instance):
125 cec9845c Michael Hanselmann
  """Test five consecutive instance failures.
126 cec9845c Michael Hanselmann

127 cec9845c Michael Hanselmann
  """
128 46f9a948 Michael Hanselmann
  inst_name = qa_utils.ResolveInstanceName(instance["name"])
129 cec9845c Michael Hanselmann
130 28a6fbc8 Michael Hanselmann
  _ResetWatcherDaemon()
131 cec9845c Michael Hanselmann
132 28a6fbc8 Michael Hanselmann
  for should_start in ([True] * 5) + [False]:
133 b998270c Iustin Pop
    _ShutdownInstance(inst_name)
134 28a6fbc8 Michael Hanselmann
    _RunWatcherDaemon()
135 28a6fbc8 Michael Hanselmann
    time.sleep(5)
136 cec9845c Michael Hanselmann
137 b998270c Iustin Pop
    if bool(_InstanceRunning(inst_name)) != should_start:
138 28a6fbc8 Michael Hanselmann
      if should_start:
139 28a6fbc8 Michael Hanselmann
        msg = "Instance not started when it should"
140 28a6fbc8 Michael Hanselmann
      else:
141 28a6fbc8 Michael Hanselmann
        msg = "Instance started when it shouldn't"
142 28a6fbc8 Michael Hanselmann
      raise qa_error.Error(msg)
143 cec9845c Michael Hanselmann
144 2f4b4f78 Iustin Pop
  AssertCommand(["gnt-instance", "info", inst_name])