Statistics
| Branch: | Tag: | Revision:

root / qa / qa_daemon.py @ 8201b996

History | View | Annotate | Download (4.5 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Daemon related QA tests.
23

24
"""
25

    
26
import time
27

    
28
from ganeti import utils
29
from ganeti import constants
30

    
31
import qa_config
32
import qa_utils
33
import qa_error
34

    
35
from qa_utils import AssertEqual, AssertMatch, StartSSH, GetCommandOutput
36

    
37

    
38
def _InstanceRunning(node, name):
39
  """Checks whether an instance is running.
40

41
  Args:
42
    node: Node the instance runs on
43
    name: Full name of Xen instance
44
  """
45
  cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
46
  ret = StartSSH(node['primary'], cmd).wait()
47
  return ret == 0
48

    
49

    
50
def _XmShutdownInstance(node, name):
51
  """Shuts down instance using "xm" and waits for completion.
52

53
  Args:
54
    node: Node the instance runs on
55
    name: Full name of Xen instance
56
  """
57
  master = qa_config.GetMasterNode()
58

    
59
  cmd = ['xm', 'shutdown', name]
60
  AssertEqual(StartSSH(node['primary'],
61
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
62

    
63
  # Wait up to a minute
64
  end = time.time() + 60
65
  while time.time() <= end:
66
    if not _InstanceRunning(node, name):
67
      break
68
    time.sleep(5)
69
  else:
70
    raise qa_error.Error("xm shutdown failed")
71

    
72

    
73
def _ResetWatcherDaemon():
74
  """Removes the watcher daemon's state file.
75

76
  Args:
77
    node: Node to be reset
78
  """
79
  master = qa_config.GetMasterNode()
80

    
81
  cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
82
  AssertEqual(StartSSH(master['primary'],
83
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
84

    
85

    
86
def _RunWatcherDaemon():
87
  """Runs the ganeti-watcher daemon on the master node.
88

89
  """
90
  master = qa_config.GetMasterNode()
91

    
92
  cmd = ["ganeti-watcher", "-d", "--ignore-pause"]
93
  AssertEqual(StartSSH(master["primary"],
94
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
95

    
96

    
97
def TestPauseWatcher():
98
  """Tests and pauses the watcher.
99

100
  """
101
  master = qa_config.GetMasterNode()
102

    
103
  cmd = ["gnt-cluster", "watcher", "pause", "4h"]
104
  AssertEqual(StartSSH(master["primary"],
105
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
106

    
107
  cmd = ["gnt-cluster", "watcher", "info"]
108
  output = GetCommandOutput(master["primary"],
109
                            utils.ShellQuoteArgs(cmd))
110
  AssertMatch(output, r"^.*\bis paused\b.*")
111

    
112

    
113
def TestResumeWatcher():
114
  """Tests and unpauses the watcher.
115

116
  """
117
  master = qa_config.GetMasterNode()
118

    
119
  cmd = ["gnt-cluster", "watcher", "continue"]
120
  AssertEqual(StartSSH(master["primary"],
121
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
122

    
123
  cmd = ["gnt-cluster", "watcher", "info"]
124
  output = GetCommandOutput(master["primary"],
125
                            utils.ShellQuoteArgs(cmd))
126
  AssertMatch(output, r"^.*\bis not paused\b.*")
127

    
128

    
129
def TestInstanceAutomaticRestart(node, instance):
130
  """Test automatic restart of instance by ganeti-watcher.
131

132
  """
133
  master = qa_config.GetMasterNode()
134
  inst_name = qa_utils.ResolveInstanceName(instance)
135

    
136
  _ResetWatcherDaemon()
137
  _XmShutdownInstance(node, inst_name)
138

    
139
  _RunWatcherDaemon()
140
  time.sleep(5)
141

    
142
  if not _InstanceRunning(node, inst_name):
143
    raise qa_error.Error("Daemon didn't restart instance")
144

    
145
  cmd = ['gnt-instance', 'info', inst_name]
146
  AssertEqual(StartSSH(master['primary'],
147
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
148

    
149

    
150
def TestInstanceConsecutiveFailures(node, instance):
151
  """Test five consecutive instance failures.
152

153
  """
154
  master = qa_config.GetMasterNode()
155
  inst_name = qa_utils.ResolveInstanceName(instance)
156

    
157
  _ResetWatcherDaemon()
158

    
159
  for should_start in ([True] * 5) + [False]:
160
    _XmShutdownInstance(node, inst_name)
161
    _RunWatcherDaemon()
162
    time.sleep(5)
163

    
164
    if bool(_InstanceRunning(node, inst_name)) != should_start:
165
      if should_start:
166
        msg = "Instance not started when it should"
167
      else:
168
        msg = "Instance started when it shouldn't"
169
      raise qa_error.Error(msg)
170

    
171
  cmd = ['gnt-instance', 'info', inst_name]
172
  AssertEqual(StartSSH(master['primary'],
173
                       utils.ShellQuoteArgs(cmd)).wait(), 0)