Statistics
| Branch: | Tag: | Revision:

root / qa / qa_utils.py @ 58ea8d17

History | View | Annotate | Download (13.4 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Utilities for QA tests.
23

24
"""
25

    
26
import os
27
import re
28
import sys
29
import subprocess
30
import random
31
import tempfile
32

    
33
from ganeti import utils
34
from ganeti import compat
35
from ganeti import constants
36

    
37
import qa_config
38
import qa_error
39

    
40

    
41
_INFO_SEQ = None
42
_WARNING_SEQ = None
43
_ERROR_SEQ = None
44
_RESET_SEQ = None
45

    
46
_MULTIPLEXERS = {}
47

    
48

    
49
def _SetupColours():
50
  """Initializes the colour constants.
51

52
  """
53
  # pylint: disable=W0603
54
  # due to global usage
55
  global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
56

    
57
  # Don't use colours if stdout isn't a terminal
58
  if not sys.stdout.isatty():
59
    return
60

    
61
  try:
62
    import curses
63
  except ImportError:
64
    # Don't use colours if curses module can't be imported
65
    return
66

    
67
  curses.setupterm()
68

    
69
  _RESET_SEQ = curses.tigetstr("op")
70

    
71
  setaf = curses.tigetstr("setaf")
72
  _INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
73
  _WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
74
  _ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
75

    
76

    
77
_SetupColours()
78

    
79

    
80
def AssertIn(item, sequence):
81
  """Raises an error when item is not in sequence.
82

83
  """
84
  if item not in sequence:
85
    raise qa_error.Error("%r not in %r" % (item, sequence))
86

    
87

    
88
def AssertNotIn(item, sequence):
89
  """Raises an error when item is in sequence.
90

91
  """
92
  if item in sequence:
93
    raise qa_error.Error("%r in %r" % (item, sequence))
94

    
95

    
96
def AssertEqual(first, second):
97
  """Raises an error when values aren't equal.
98

99
  """
100
  if not first == second:
101
    raise qa_error.Error("%r == %r" % (first, second))
102

    
103

    
104
def AssertNotEqual(first, second):
105
  """Raises an error when values are equal.
106

107
  """
108
  if not first != second:
109
    raise qa_error.Error("%r != %r" % (first, second))
110

    
111

    
112
def AssertMatch(string, pattern):
113
  """Raises an error when string doesn't match regexp pattern.
114

115
  """
116
  if not re.match(pattern, string):
117
    raise qa_error.Error("%r doesn't match /%r/" % (string, pattern))
118

    
119

    
120
def AssertCommand(cmd, fail=False, node=None):
121
  """Checks that a remote command succeeds.
122

123
  @param cmd: either a string (the command to execute) or a list (to
124
      be converted using L{utils.ShellQuoteArgs} into a string)
125
  @type fail: boolean
126
  @param fail: if the command is expected to fail instead of succeeding
127
  @param node: if passed, it should be the node on which the command
128
      should be executed, instead of the master node (can be either a
129
      dict or a string)
130

131
  """
132
  if node is None:
133
    node = qa_config.GetMasterNode()
134

    
135
  if isinstance(node, basestring):
136
    nodename = node
137
  else:
138
    nodename = node["primary"]
139

    
140
  if isinstance(cmd, basestring):
141
    cmdstr = cmd
142
  else:
143
    cmdstr = utils.ShellQuoteArgs(cmd)
144

    
145
  rcode = StartSSH(nodename, cmdstr).wait()
146

    
147
  if fail:
148
    if rcode == 0:
149
      raise qa_error.Error("Command '%s' on node %s was expected to fail but"
150
                           " didn't" % (cmdstr, nodename))
151
  else:
152
    if rcode != 0:
153
      raise qa_error.Error("Command '%s' on node %s failed, exit code %s" %
154
                           (cmdstr, nodename, rcode))
155

    
156
  return rcode
157

    
158

    
159
def GetSSHCommand(node, cmd, strict=True, opts=None, tty=True):
160
  """Builds SSH command to be executed.
161

162
  @type node: string
163
  @param node: node the command should run on
164
  @type cmd: string
165
  @param cmd: command to be executed in the node; if None or empty
166
      string, no command will be executed
167
  @type strict: boolean
168
  @param strict: whether to enable strict host key checking
169
  @type opts: list
170
  @param opts: list of additional options
171
  @type tty: Bool
172
  @param tty: If we should use tty
173

174
  """
175
  args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-l", "root"]
176

    
177
  if tty:
178
    args.append("-t")
179

    
180
  if strict:
181
    tmp = "yes"
182
  else:
183
    tmp = "no"
184
  args.append("-oStrictHostKeyChecking=%s" % tmp)
185
  args.append("-oClearAllForwardings=yes")
186
  args.append("-oForwardAgent=yes")
187
  if opts:
188
    args.extend(opts)
189
  if node in _MULTIPLEXERS:
190
    spath = _MULTIPLEXERS[node][0]
191
    args.append("-oControlPath=%s" % spath)
192
    args.append("-oControlMaster=no")
193
  args.append(node)
194
  if cmd:
195
    args.append(cmd)
196

    
197
  return args
198

    
199

    
200
def StartLocalCommand(cmd, **kwargs):
201
  """Starts a local command.
202

203
  """
204
  print "Command: %s" % utils.ShellQuoteArgs(cmd)
205
  return subprocess.Popen(cmd, shell=False, **kwargs)
206

    
207

    
208
def StartSSH(node, cmd, strict=True):
209
  """Starts SSH.
210

211
  """
212
  return StartLocalCommand(GetSSHCommand(node, cmd, strict=strict))
213

    
214

    
215
def StartMultiplexer(node):
216
  """Starts a multiplexer command.
217

218
  @param node: the node for which to open the multiplexer
219

220
  """
221
  if node in _MULTIPLEXERS:
222
    return
223

    
224
  # Note: yes, we only need mktemp, since we'll remove the file anyway
225
  sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
226
  utils.RemoveFile(sname)
227
  opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
228
  print "Created socket at %s" % sname
229
  child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
230
  _MULTIPLEXERS[node] = (sname, child)
231

    
232

    
233
def CloseMultiplexers():
234
  """Closes all current multiplexers and cleans up.
235

236
  """
237
  for node in _MULTIPLEXERS.keys():
238
    (sname, child) = _MULTIPLEXERS.pop(node)
239
    utils.KillProcess(child.pid, timeout=10, waitpid=True)
240
    utils.RemoveFile(sname)
241

    
242

    
243
def GetCommandOutput(node, cmd, tty=True):
244
  """Returns the output of a command executed on the given node.
245

246
  """
247
  p = StartLocalCommand(GetSSHCommand(node, cmd, tty=tty),
248
                        stdout=subprocess.PIPE)
249
  AssertEqual(p.wait(), 0)
250
  return p.stdout.read()
251

    
252

    
253
def UploadFile(node, src):
254
  """Uploads a file to a node and returns the filename.
255

256
  Caller needs to remove the returned file on the node when it's not needed
257
  anymore.
258

259
  """
260
  # Make sure nobody else has access to it while preserving local permissions
261
  mode = os.stat(src).st_mode & 0700
262

    
263
  cmd = ('tmp=$(tempfile --mode %o --prefix gnt) && '
264
         '[[ -f "${tmp}" ]] && '
265
         'cat > "${tmp}" && '
266
         'echo "${tmp}"') % mode
267

    
268
  f = open(src, "r")
269
  try:
270
    p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
271
                         stdout=subprocess.PIPE)
272
    AssertEqual(p.wait(), 0)
273

    
274
    # Return temporary filename
275
    return p.stdout.read().strip()
276
  finally:
277
    f.close()
278

    
279

    
280
def UploadData(node, data, mode=0600, filename=None):
281
  """Uploads data to a node and returns the filename.
282

283
  Caller needs to remove the returned file on the node when it's not needed
284
  anymore.
285

286
  """
287
  if filename:
288
    tmp = "tmp=%s" % utils.ShellQuote(filename)
289
  else:
290
    tmp = "tmp=$(tempfile --mode %o --prefix gnt)" % mode
291
  cmd = ("%s && "
292
         "[[ -f \"${tmp}\" ]] && "
293
         "cat > \"${tmp}\" && "
294
         "echo \"${tmp}\"") % tmp
295

    
296
  p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False,
297
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE)
298
  p.stdin.write(data)
299
  p.stdin.close()
300
  AssertEqual(p.wait(), 0)
301

    
302
  # Return temporary filename
303
  return p.stdout.read().strip()
304

    
305

    
306
def BackupFile(node, path):
307
  """Creates a backup of a file on the node and returns the filename.
308

309
  Caller needs to remove the returned file on the node when it's not needed
310
  anymore.
311

312
  """
313
  cmd = ("tmp=$(tempfile --prefix .gnt --directory=$(dirname %s)) && "
314
         "[[ -f \"$tmp\" ]] && "
315
         "cp %s $tmp && "
316
         "echo $tmp") % (utils.ShellQuote(path), utils.ShellQuote(path))
317

    
318
  # Return temporary filename
319
  return GetCommandOutput(node, cmd).strip()
320

    
321

    
322
def _ResolveName(cmd, key):
323
  """Helper function.
324

325
  """
326
  master = qa_config.GetMasterNode()
327

    
328
  output = GetCommandOutput(master["primary"], utils.ShellQuoteArgs(cmd))
329
  for line in output.splitlines():
330
    (lkey, lvalue) = line.split(":", 1)
331
    if lkey == key:
332
      return lvalue.lstrip()
333
  raise KeyError("Key not found")
334

    
335

    
336
def ResolveInstanceName(instance):
337
  """Gets the full name of an instance.
338

339
  @type instance: string
340
  @param instance: Instance name
341

342
  """
343
  return _ResolveName(["gnt-instance", "info", instance],
344
                      "Instance name")
345

    
346

    
347
def ResolveNodeName(node):
348
  """Gets the full name of a node.
349

350
  """
351
  return _ResolveName(["gnt-node", "info", node["primary"]],
352
                      "Node name")
353

    
354

    
355
def GetNodeInstances(node, secondaries=False):
356
  """Gets a list of instances on a node.
357

358
  """
359
  master = qa_config.GetMasterNode()
360
  node_name = ResolveNodeName(node)
361

    
362
  # Get list of all instances
363
  cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
364
         "--output=name,pnode,snodes"]
365
  output = GetCommandOutput(master["primary"], utils.ShellQuoteArgs(cmd))
366

    
367
  instances = []
368
  for line in output.splitlines():
369
    (name, pnode, snodes) = line.split(":", 2)
370
    if ((not secondaries and pnode == node_name) or
371
        (secondaries and node_name in snodes.split(","))):
372
      instances.append(name)
373

    
374
  return instances
375

    
376

    
377
def _SelectQueryFields(rnd, fields):
378
  """Generates a list of fields for query tests.
379

380
  """
381
  # Create copy for shuffling
382
  fields = list(fields)
383
  rnd.shuffle(fields)
384

    
385
  # Check all fields
386
  yield fields
387
  yield sorted(fields)
388

    
389
  # Duplicate fields
390
  yield fields + fields
391

    
392
  # Check small groups of fields
393
  while fields:
394
    yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields]
395

    
396

    
397
def _List(listcmd, fields, names):
398
  """Runs a list command.
399

400
  """
401
  master = qa_config.GetMasterNode()
402

    
403
  cmd = [listcmd, "list", "--separator=|", "--no-headers",
404
         "--output", ",".join(fields)]
405

    
406
  if names:
407
    cmd.extend(names)
408

    
409
  return GetCommandOutput(master["primary"],
410
                          utils.ShellQuoteArgs(cmd)).splitlines()
411

    
412

    
413
def GenericQueryTest(cmd, fields):
414
  """Runs a number of tests on query commands.
415

416
  @param cmd: Command name
417
  @param fields: List of field names
418

419
  """
420
  rnd = random.Random(hash(cmd))
421

    
422
  fields = list(fields)
423
  rnd.shuffle(fields)
424

    
425
  # Test a number of field combinations
426
  for testfields in _SelectQueryFields(rnd, fields):
427
    AssertCommand([cmd, "list", "--output", ",".join(testfields)])
428

    
429
  namelist_fn = compat.partial(_List, cmd, ["name"])
430

    
431
  # When no names were requested, the list must be sorted
432
  names = namelist_fn(None)
433
  AssertEqual(names, utils.NiceSort(names))
434

    
435
  # When requesting specific names, the order must be kept
436
  revnames = list(reversed(names))
437
  AssertEqual(namelist_fn(revnames), revnames)
438

    
439
  randnames = list(names)
440
  rnd.shuffle(randnames)
441
  AssertEqual(namelist_fn(randnames), randnames)
442

    
443
  # Listing unknown items must fail
444
  AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"], fail=True)
445

    
446
  # Check exit code for listing unknown field
447
  AssertEqual(AssertCommand([cmd, "list", "--output=field/does/not/exist"],
448
                            fail=True),
449
              constants.EXIT_UNKNOWN_FIELD)
450

    
451

    
452
def GenericQueryFieldsTest(cmd, fields):
453
  master = qa_config.GetMasterNode()
454

    
455
  # Listing fields
456
  AssertCommand([cmd, "list-fields"])
457
  AssertCommand([cmd, "list-fields"] + fields)
458

    
459
  # Check listed fields (all, must be sorted)
460
  realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
461
  output = GetCommandOutput(master["primary"],
462
                            utils.ShellQuoteArgs(realcmd)).splitlines()
463
  AssertEqual([line.split("|", 1)[0] for line in output],
464
              utils.NiceSort(fields))
465

    
466
  # Check exit code for listing unknown field
467
  AssertEqual(AssertCommand([cmd, "list-fields", "field/does/not/exist"],
468
                            fail=True),
469
              constants.EXIT_UNKNOWN_FIELD)
470

    
471

    
472
def _FormatWithColor(text, seq):
473
  if not seq:
474
    return text
475
  return "%s%s%s" % (seq, text, _RESET_SEQ)
476

    
477

    
478
FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
479
FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
480
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
481

    
482

    
483
def AddToEtcHosts(hostnames):
484
  """Adds hostnames to /etc/hosts.
485

486
  @param hostnames: List of hostnames first used A records, all other CNAMEs
487

488
  """
489
  master = qa_config.GetMasterNode()
490
  tmp_hosts = UploadData(master["primary"], "", mode=0644)
491

    
492
  quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
493
  data = []
494
  for localhost in ("::1", "127.0.0.1"):
495
    data.append("%s %s" % (localhost, " ".join(hostnames)))
496

    
497
  try:
498
    AssertCommand(("cat /etc/hosts > %s && echo -e '%s' >> %s && mv %s"
499
                   " /etc/hosts") % (quoted_tmp_hosts, "\\n".join(data),
500
                                     quoted_tmp_hosts, quoted_tmp_hosts))
501
  except qa_error.Error:
502
    AssertCommand(["rm", tmp_hosts])
503

    
504

    
505
def RemoveFromEtcHosts(hostnames):
506
  """Remove hostnames from /etc/hosts.
507

508
  @param hostnames: List of hostnames first used A records, all other CNAMEs
509

510
  """
511
  master = qa_config.GetMasterNode()
512
  tmp_hosts = UploadData(master["primary"], "", mode=0644)
513
  quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
514

    
515
  sed_data = " ".join(hostnames)
516
  try:
517
    AssertCommand(("sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' /etc/hosts > %s"
518
                   " && mv %s /etc/hosts") % (sed_data, quoted_tmp_hosts,
519
                                              quoted_tmp_hosts))
520
  except qa_error.Error:
521
    AssertCommand(["rm", tmp_hosts])