Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ 6b7d5878

History | View | Annotate | Download (8.9 kB)

1 c68d1f43 Michael Hanselmann
#
2 c68d1f43 Michael Hanselmann
#
3 c68d1f43 Michael Hanselmann
4 cec9845c Michael Hanselmann
# Copyright (C) 2007 Google Inc.
5 cec9845c Michael Hanselmann
#
6 cec9845c Michael Hanselmann
# This program is free software; you can redistribute it and/or modify
7 cec9845c Michael Hanselmann
# it under the terms of the GNU General Public License as published by
8 cec9845c Michael Hanselmann
# the Free Software Foundation; either version 2 of the License, or
9 cec9845c Michael Hanselmann
# (at your option) any later version.
10 cec9845c Michael Hanselmann
#
11 cec9845c Michael Hanselmann
# This program is distributed in the hope that it will be useful, but
12 cec9845c Michael Hanselmann
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 cec9845c Michael Hanselmann
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 cec9845c Michael Hanselmann
# General Public License for more details.
15 cec9845c Michael Hanselmann
#
16 cec9845c Michael Hanselmann
# You should have received a copy of the GNU General Public License
17 cec9845c Michael Hanselmann
# along with this program; if not, write to the Free Software
18 cec9845c Michael Hanselmann
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 cec9845c Michael Hanselmann
# 02110-1301, USA.
20 cec9845c Michael Hanselmann
21 cec9845c Michael Hanselmann
22 cec9845c Michael Hanselmann
"""Cluster related QA tests.
23 cec9845c Michael Hanselmann

24 cec9845c Michael Hanselmann
"""
25 cec9845c Michael Hanselmann
26 cec9845c Michael Hanselmann
import tempfile
27 cec9845c Michael Hanselmann
28 6d4a1656 Michael Hanselmann
from ganeti import constants
29 6d4a1656 Michael Hanselmann
from ganeti import bootstrap
30 cec9845c Michael Hanselmann
from ganeti import utils
31 cec9845c Michael Hanselmann
32 cec9845c Michael Hanselmann
import qa_config
33 cec9845c Michael Hanselmann
import qa_utils
34 cec9845c Michael Hanselmann
import qa_error
35 cec9845c Michael Hanselmann
36 6d4a1656 Michael Hanselmann
from qa_utils import AssertEqual, AssertNotEqual, StartSSH
37 cec9845c Michael Hanselmann
38 cec9845c Michael Hanselmann
39 830da270 Michael Hanselmann
def _RemoveFileFromAllNodes(filename):
40 830da270 Michael Hanselmann
  """Removes a file from all nodes.
41 830da270 Michael Hanselmann

42 830da270 Michael Hanselmann
  """
43 830da270 Michael Hanselmann
  for node in qa_config.get('nodes'):
44 830da270 Michael Hanselmann
    cmd = ['rm', '-f', filename]
45 830da270 Michael Hanselmann
    AssertEqual(StartSSH(node['primary'],
46 830da270 Michael Hanselmann
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
47 830da270 Michael Hanselmann
48 830da270 Michael Hanselmann
49 830da270 Michael Hanselmann
def _CheckFileOnAllNodes(filename, content):
50 830da270 Michael Hanselmann
  """Verifies the content of the given file on all nodes.
51 830da270 Michael Hanselmann

52 830da270 Michael Hanselmann
  """
53 830da270 Michael Hanselmann
  cmd = utils.ShellQuoteArgs(["cat", filename])
54 830da270 Michael Hanselmann
  for node in qa_config.get('nodes'):
55 830da270 Michael Hanselmann
    AssertEqual(qa_utils.GetCommandOutput(node['primary'], cmd),
56 830da270 Michael Hanselmann
                content)
57 830da270 Michael Hanselmann
58 830da270 Michael Hanselmann
59 cec9845c Michael Hanselmann
def TestClusterInit():
60 cec9845c Michael Hanselmann
  """gnt-cluster init"""
61 cec9845c Michael Hanselmann
  master = qa_config.GetMasterNode()
62 cec9845c Michael Hanselmann
63 cec9845c Michael Hanselmann
  cmd = ['gnt-cluster', 'init']
64 cec9845c Michael Hanselmann
65 cec9845c Michael Hanselmann
  if master.get('secondary', None):
66 cec9845c Michael Hanselmann
    cmd.append('--secondary-ip=%s' % master['secondary'])
67 cec9845c Michael Hanselmann
68 cec9845c Michael Hanselmann
  bridge = qa_config.get('bridge', None)
69 cec9845c Michael Hanselmann
  if bridge:
70 cec9845c Michael Hanselmann
    cmd.append('--bridge=%s' % bridge)
71 cec9845c Michael Hanselmann
    cmd.append('--master-netdev=%s' % bridge)
72 cec9845c Michael Hanselmann
73 2d6db53a Guido Trotter
  htype = qa_config.get('enabled-hypervisors', None)
74 b32f9859 Iustin Pop
  if htype:
75 2d6db53a Guido Trotter
    cmd.append('--enabled-hypervisors=%s' % htype)
76 b32f9859 Iustin Pop
77 cec9845c Michael Hanselmann
  cmd.append(qa_config.get('name'))
78 cec9845c Michael Hanselmann
79 cec9845c Michael Hanselmann
  AssertEqual(StartSSH(master['primary'],
80 cec9845c Michael Hanselmann
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
81 cec9845c Michael Hanselmann
82 cec9845c Michael Hanselmann
83 caea3b32 Iustin Pop
def TestClusterRename():
84 caea3b32 Iustin Pop
  """gnt-cluster rename"""
85 caea3b32 Iustin Pop
  master = qa_config.GetMasterNode()
86 caea3b32 Iustin Pop
87 caea3b32 Iustin Pop
  cmd = ['gnt-cluster', 'rename', '-f']
88 caea3b32 Iustin Pop
89 caea3b32 Iustin Pop
  original_name = qa_config.get('name')
90 caea3b32 Iustin Pop
  rename_target = qa_config.get('rename', None)
91 caea3b32 Iustin Pop
  if rename_target is None:
92 caea3b32 Iustin Pop
    print qa_utils.FormatError('"rename" entry is missing')
93 caea3b32 Iustin Pop
    return
94 caea3b32 Iustin Pop
95 caea3b32 Iustin Pop
  cmd_1 = cmd + [rename_target]
96 caea3b32 Iustin Pop
  cmd_2 = cmd + [original_name]
97 caea3b32 Iustin Pop
98 caea3b32 Iustin Pop
  cmd_verify = ['gnt-cluster', 'verify']
99 caea3b32 Iustin Pop
100 caea3b32 Iustin Pop
  AssertEqual(StartSSH(master['primary'],
101 caea3b32 Iustin Pop
                       utils.ShellQuoteArgs(cmd_1)).wait(), 0)
102 caea3b32 Iustin Pop
103 caea3b32 Iustin Pop
  AssertEqual(StartSSH(master['primary'],
104 caea3b32 Iustin Pop
                       utils.ShellQuoteArgs(cmd_verify)).wait(), 0)
105 caea3b32 Iustin Pop
106 caea3b32 Iustin Pop
  AssertEqual(StartSSH(master['primary'],
107 caea3b32 Iustin Pop
                       utils.ShellQuoteArgs(cmd_2)).wait(), 0)
108 caea3b32 Iustin Pop
109 caea3b32 Iustin Pop
  AssertEqual(StartSSH(master['primary'],
110 caea3b32 Iustin Pop
                       utils.ShellQuoteArgs(cmd_verify)).wait(), 0)
111 caea3b32 Iustin Pop
112 caea3b32 Iustin Pop
113 cec9845c Michael Hanselmann
def TestClusterVerify():
114 cec9845c Michael Hanselmann
  """gnt-cluster verify"""
115 cec9845c Michael Hanselmann
  master = qa_config.GetMasterNode()
116 cec9845c Michael Hanselmann
117 283f9d4c Michael Hanselmann
  cmd = ['gnt-cluster', 'verify']
118 cec9845c Michael Hanselmann
  AssertEqual(StartSSH(master['primary'],
119 cec9845c Michael Hanselmann
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
120 cec9845c Michael Hanselmann
121 cec9845c Michael Hanselmann
122 cec9845c Michael Hanselmann
def TestClusterInfo():
123 cec9845c Michael Hanselmann
  """gnt-cluster info"""
124 283f9d4c Michael Hanselmann
  master = qa_config.GetMasterNode()
125 283f9d4c Michael Hanselmann
126 cec9845c Michael Hanselmann
  cmd = ['gnt-cluster', 'info']
127 283f9d4c Michael Hanselmann
  AssertEqual(StartSSH(master['primary'],
128 283f9d4c Michael Hanselmann
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
129 283f9d4c Michael Hanselmann
130 283f9d4c Michael Hanselmann
131 283f9d4c Michael Hanselmann
def TestClusterGetmaster():
132 283f9d4c Michael Hanselmann
  """gnt-cluster getmaster"""
133 cec9845c Michael Hanselmann
  master = qa_config.GetMasterNode()
134 cec9845c Michael Hanselmann
135 283f9d4c Michael Hanselmann
  cmd = ['gnt-cluster', 'getmaster']
136 283f9d4c Michael Hanselmann
  AssertEqual(StartSSH(master['primary'],
137 283f9d4c Michael Hanselmann
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
138 283f9d4c Michael Hanselmann
139 283f9d4c Michael Hanselmann
140 283f9d4c Michael Hanselmann
def TestClusterVersion():
141 283f9d4c Michael Hanselmann
  """gnt-cluster version"""
142 283f9d4c Michael Hanselmann
  master = qa_config.GetMasterNode()
143 283f9d4c Michael Hanselmann
144 283f9d4c Michael Hanselmann
  cmd = ['gnt-cluster', 'version']
145 cec9845c Michael Hanselmann
  AssertEqual(StartSSH(master['primary'],
146 cec9845c Michael Hanselmann
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
147 cec9845c Michael Hanselmann
148 cec9845c Michael Hanselmann
149 6d4a1656 Michael Hanselmann
def TestClusterRenewCrypto():
150 6d4a1656 Michael Hanselmann
  """gnt-cluster renew-crypto"""
151 6d4a1656 Michael Hanselmann
  master = qa_config.GetMasterNode()
152 6d4a1656 Michael Hanselmann
153 6d4a1656 Michael Hanselmann
  # Conflicting options
154 6d4a1656 Michael Hanselmann
  cmd = ["gnt-cluster", "renew-crypto", "--force",
155 6b7d5878 Michael Hanselmann
         "--new-cluster-certificate", "--new-confd-hmac-key",
156 6d4a1656 Michael Hanselmann
         "--new-rapi-certificate", "--rapi-certificate=/dev/null"]
157 6d4a1656 Michael Hanselmann
  AssertNotEqual(StartSSH(master["primary"],
158 6d4a1656 Michael Hanselmann
                          utils.ShellQuoteArgs(cmd)).wait(), 0)
159 6d4a1656 Michael Hanselmann
160 6d4a1656 Michael Hanselmann
  # Invalid RAPI certificate
161 6d4a1656 Michael Hanselmann
  cmd = ["gnt-cluster", "renew-crypto", "--force",
162 6d4a1656 Michael Hanselmann
         "--rapi-certificate=/dev/null"]
163 6d4a1656 Michael Hanselmann
  AssertNotEqual(StartSSH(master["primary"],
164 6d4a1656 Michael Hanselmann
                          utils.ShellQuoteArgs(cmd)).wait(), 0)
165 6d4a1656 Michael Hanselmann
166 6d4a1656 Michael Hanselmann
  # Custom RAPI certificate
167 6d4a1656 Michael Hanselmann
  fh = tempfile.NamedTemporaryFile()
168 6d4a1656 Michael Hanselmann
169 6d4a1656 Michael Hanselmann
  # Ensure certificate doesn't cause "gnt-cluster verify" to complain
170 6d4a1656 Michael Hanselmann
  validity = constants.SSL_CERT_EXPIRATION_WARN * 3
171 6d4a1656 Michael Hanselmann
172 6d4a1656 Michael Hanselmann
  bootstrap.GenerateSelfSignedSslCert(fh.name, validity=validity)
173 6d4a1656 Michael Hanselmann
174 6d4a1656 Michael Hanselmann
  tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
175 6d4a1656 Michael Hanselmann
  try:
176 6d4a1656 Michael Hanselmann
    cmd = ["gnt-cluster", "renew-crypto", "--force",
177 6d4a1656 Michael Hanselmann
           "--rapi-certificate=%s" % tmpcert]
178 6d4a1656 Michael Hanselmann
    AssertEqual(StartSSH(master["primary"],
179 6d4a1656 Michael Hanselmann
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
180 6d4a1656 Michael Hanselmann
  finally:
181 6d4a1656 Michael Hanselmann
    cmd = ["rm", "-f", tmpcert]
182 6d4a1656 Michael Hanselmann
    AssertEqual(StartSSH(master["primary"],
183 6d4a1656 Michael Hanselmann
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
184 6d4a1656 Michael Hanselmann
185 6d4a1656 Michael Hanselmann
  # Normal case
186 6d4a1656 Michael Hanselmann
  cmd = ["gnt-cluster", "renew-crypto", "--force",
187 6b7d5878 Michael Hanselmann
         "--new-cluster-certificate", "--new-confd-hmac-key",
188 6d4a1656 Michael Hanselmann
         "--new-rapi-certificate"]
189 6d4a1656 Michael Hanselmann
  AssertEqual(StartSSH(master["primary"],
190 6d4a1656 Michael Hanselmann
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
191 6d4a1656 Michael Hanselmann
192 6d4a1656 Michael Hanselmann
193 cec9845c Michael Hanselmann
def TestClusterBurnin():
194 cec9845c Michael Hanselmann
  """Burnin"""
195 cec9845c Michael Hanselmann
  master = qa_config.GetMasterNode()
196 cec9845c Michael Hanselmann
197 0b0a150a Iustin Pop
  options = qa_config.get('options', {})
198 0b0a150a Iustin Pop
  disk_template = options.get('burnin-disk-template', 'drbd')
199 0b0a150a Iustin Pop
  parallel = options.get('burnin-in-parallel', False)
200 0b0a150a Iustin Pop
  check_inst = options.get('burnin-check-instances', False)
201 4dc76b24 Iustin Pop
  do_rename = options.get('burnin-rename', '')
202 23103544 Michael Hanselmann
203 cec9845c Michael Hanselmann
  # Get as many instances as we need
204 cec9845c Michael Hanselmann
  instances = []
205 cec9845c Michael Hanselmann
  try:
206 23103544 Michael Hanselmann
    try:
207 23103544 Michael Hanselmann
      num = qa_config.get('options', {}).get('burnin-instances', 1)
208 f1501b3f Michael Hanselmann
      for _ in range(0, num):
209 23103544 Michael Hanselmann
        instances.append(qa_config.AcquireInstance())
210 23103544 Michael Hanselmann
    except qa_error.OutOfInstancesError:
211 23103544 Michael Hanselmann
      print "Not enough instances, continuing anyway."
212 cec9845c Michael Hanselmann
213 23103544 Michael Hanselmann
    if len(instances) < 1:
214 23103544 Michael Hanselmann
      raise qa_error.Error("Burnin needs at least one instance")
215 cec9845c Michael Hanselmann
216 cec9845c Michael Hanselmann
    script = qa_utils.UploadFile(master['primary'], '../tools/burnin')
217 cec9845c Michael Hanselmann
    try:
218 23103544 Michael Hanselmann
      # Run burnin
219 cec9845c Michael Hanselmann
      cmd = [script,
220 0b0a150a Iustin Pop
             '-p',
221 cec9845c Michael Hanselmann
             '--os=%s' % qa_config.get('os'),
222 1d693311 Michael Hanselmann
             '--disk-size=%s' % ",".join(qa_config.get('disk')),
223 1d693311 Michael Hanselmann
             '--disk-growth=%s' % ",".join(qa_config.get('disk-growth')),
224 23103544 Michael Hanselmann
             '--disk-template=%s' % disk_template]
225 0b0a150a Iustin Pop
      if parallel:
226 0b0a150a Iustin Pop
        cmd.append('--parallel')
227 0b0a150a Iustin Pop
      if check_inst:
228 0b0a150a Iustin Pop
        cmd.append('--http-check')
229 4dc76b24 Iustin Pop
      if do_rename:
230 4dc76b24 Iustin Pop
        cmd.append('--rename=%s' % do_rename)
231 cec9845c Michael Hanselmann
      cmd += [inst['name'] for inst in instances]
232 cec9845c Michael Hanselmann
      AssertEqual(StartSSH(master['primary'],
233 cec9845c Michael Hanselmann
                           utils.ShellQuoteArgs(cmd)).wait(), 0)
234 cec9845c Michael Hanselmann
    finally:
235 cec9845c Michael Hanselmann
      cmd = ['rm', '-f', script]
236 cec9845c Michael Hanselmann
      AssertEqual(StartSSH(master['primary'],
237 cec9845c Michael Hanselmann
                           utils.ShellQuoteArgs(cmd)).wait(), 0)
238 cec9845c Michael Hanselmann
  finally:
239 cec9845c Michael Hanselmann
    for inst in instances:
240 cec9845c Michael Hanselmann
      qa_config.ReleaseInstance(inst)
241 cec9845c Michael Hanselmann
242 cec9845c Michael Hanselmann
243 cec9845c Michael Hanselmann
def TestClusterMasterFailover():
244 cec9845c Michael Hanselmann
  """gnt-cluster masterfailover"""
245 cec9845c Michael Hanselmann
  master = qa_config.GetMasterNode()
246 cec9845c Michael Hanselmann
247 cec9845c Michael Hanselmann
  failovermaster = qa_config.AcquireNode(exclude=master)
248 cec9845c Michael Hanselmann
  try:
249 cec9845c Michael Hanselmann
    cmd = ['gnt-cluster', 'masterfailover']
250 cec9845c Michael Hanselmann
    AssertEqual(StartSSH(failovermaster['primary'],
251 cec9845c Michael Hanselmann
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
252 cec9845c Michael Hanselmann
253 cec9845c Michael Hanselmann
    cmd = ['gnt-cluster', 'masterfailover']
254 cec9845c Michael Hanselmann
    AssertEqual(StartSSH(master['primary'],
255 cec9845c Michael Hanselmann
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
256 cec9845c Michael Hanselmann
  finally:
257 cec9845c Michael Hanselmann
    qa_config.ReleaseNode(failovermaster)
258 cec9845c Michael Hanselmann
259 cec9845c Michael Hanselmann
260 cec9845c Michael Hanselmann
def TestClusterCopyfile():
261 cec9845c Michael Hanselmann
  """gnt-cluster copyfile"""
262 cec9845c Michael Hanselmann
  master = qa_config.GetMasterNode()
263 cec9845c Michael Hanselmann
264 24818e8f Michael Hanselmann
  uniqueid = utils.NewUUID()
265 830da270 Michael Hanselmann
266 cec9845c Michael Hanselmann
  # Create temporary file
267 cec9845c Michael Hanselmann
  f = tempfile.NamedTemporaryFile()
268 830da270 Michael Hanselmann
  f.write(uniqueid)
269 cec9845c Michael Hanselmann
  f.flush()
270 cec9845c Michael Hanselmann
  f.seek(0)
271 cec9845c Michael Hanselmann
272 cec9845c Michael Hanselmann
  # Upload file to master node
273 cec9845c Michael Hanselmann
  testname = qa_utils.UploadFile(master['primary'], f.name)
274 cec9845c Michael Hanselmann
  try:
275 cec9845c Michael Hanselmann
    # Copy file to all nodes
276 cec9845c Michael Hanselmann
    cmd = ['gnt-cluster', 'copyfile', testname]
277 cec9845c Michael Hanselmann
    AssertEqual(StartSSH(master['primary'],
278 cec9845c Michael Hanselmann
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
279 830da270 Michael Hanselmann
    _CheckFileOnAllNodes(testname, uniqueid)
280 cec9845c Michael Hanselmann
  finally:
281 830da270 Michael Hanselmann
    _RemoveFileFromAllNodes(testname)
282 830da270 Michael Hanselmann
283 830da270 Michael Hanselmann
284 830da270 Michael Hanselmann
def TestClusterCommand():
285 830da270 Michael Hanselmann
  """gnt-cluster command"""
286 830da270 Michael Hanselmann
  master = qa_config.GetMasterNode()
287 830da270 Michael Hanselmann
288 24818e8f Michael Hanselmann
  uniqueid = utils.NewUUID()
289 24818e8f Michael Hanselmann
  rfile = "/tmp/gnt%s" % utils.NewUUID()
290 830da270 Michael Hanselmann
  rcmd = utils.ShellQuoteArgs(['echo', '-n', uniqueid])
291 830da270 Michael Hanselmann
  cmd = utils.ShellQuoteArgs(['gnt-cluster', 'command',
292 830da270 Michael Hanselmann
                              "%s >%s" % (rcmd, rfile)])
293 830da270 Michael Hanselmann
294 830da270 Michael Hanselmann
  try:
295 830da270 Michael Hanselmann
    AssertEqual(StartSSH(master['primary'], cmd).wait(), 0)
296 830da270 Michael Hanselmann
    _CheckFileOnAllNodes(rfile, uniqueid)
297 830da270 Michael Hanselmann
  finally:
298 830da270 Michael Hanselmann
    _RemoveFileFromAllNodes(rfile)
299 cec9845c Michael Hanselmann
300 cec9845c Michael Hanselmann
301 cec9845c Michael Hanselmann
def TestClusterDestroy():
302 cec9845c Michael Hanselmann
  """gnt-cluster destroy"""
303 cec9845c Michael Hanselmann
  master = qa_config.GetMasterNode()
304 cec9845c Michael Hanselmann
305 283f9d4c Michael Hanselmann
  cmd = ['gnt-cluster', 'destroy', '--yes-do-it']
306 cec9845c Michael Hanselmann
  AssertEqual(StartSSH(master['primary'],
307 cec9845c Michael Hanselmann
                       utils.ShellQuoteArgs(cmd)).wait(), 0)