Statistics
| Branch: | Tag: | Revision:

root / qa / qa_utils.py @ afd5ca04

History | View | Annotate | Download (19 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Utilities for QA tests.
23

24
"""
25

    
26
import os
27
import re
28
import sys
29
import subprocess
30
import random
31
import tempfile
32

    
33
try:
34
  import functools
35
except ImportError, err:
36
  raise ImportError("Python 2.5 or higher is required: %s" % err)
37

    
38
from ganeti import utils
39
from ganeti import compat
40
from ganeti import constants
41
from ganeti import ht
42
from ganeti import pathutils
43

    
44
import qa_config
45
import qa_error
46

    
47

    
48
_INFO_SEQ = None
49
_WARNING_SEQ = None
50
_ERROR_SEQ = None
51
_RESET_SEQ = None
52

    
53
_MULTIPLEXERS = {}
54

    
55
#: Unique ID per QA run
56
_RUN_UUID = utils.NewUUID()
57

    
58
#: Path to the QA query output log file
59
_QA_OUTPUT = pathutils.GetLogFilename("qa-output")
60

    
61

    
62
(INST_DOWN,
63
 INST_UP) = range(500, 502)
64

    
65
(FIRST_ARG,
66
 RETURN_VALUE) = range(1000, 1002)
67

    
68

    
69
def _SetupColours():
70
  """Initializes the colour constants.
71

72
  """
73
  # pylint: disable=W0603
74
  # due to global usage
75
  global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
76

    
77
  # Don't use colours if stdout isn't a terminal
78
  if not sys.stdout.isatty():
79
    return
80

    
81
  try:
82
    import curses
83
  except ImportError:
84
    # Don't use colours if curses module can't be imported
85
    return
86

    
87
  curses.setupterm()
88

    
89
  _RESET_SEQ = curses.tigetstr("op")
90

    
91
  setaf = curses.tigetstr("setaf")
92
  _INFO_SEQ = curses.tparm(setaf, curses.COLOR_GREEN)
93
  _WARNING_SEQ = curses.tparm(setaf, curses.COLOR_YELLOW)
94
  _ERROR_SEQ = curses.tparm(setaf, curses.COLOR_RED)
95

    
96

    
97
_SetupColours()
98

    
99

    
100
def AssertIn(item, sequence):
101
  """Raises an error when item is not in sequence.
102

103
  """
104
  if item not in sequence:
105
    raise qa_error.Error("%r not in %r" % (item, sequence))
106

    
107

    
108
def AssertNotIn(item, sequence):
109
  """Raises an error when item is in sequence.
110

111
  """
112
  if item in sequence:
113
    raise qa_error.Error("%r in %r" % (item, sequence))
114

    
115

    
116
def AssertEqual(first, second):
117
  """Raises an error when values aren't equal.
118

119
  """
120
  if not first == second:
121
    raise qa_error.Error("%r == %r" % (first, second))
122

    
123

    
124
def AssertMatch(string, pattern):
125
  """Raises an error when string doesn't match regexp pattern.
126

127
  """
128
  if not re.match(pattern, string):
129
    raise qa_error.Error("%r doesn't match /%r/" % (string, pattern))
130

    
131

    
132
def _GetName(entity, key):
133
  """Tries to get name of an entity.
134

135
  @type entity: string or dict
136
  @type key: string
137
  @param key: Dictionary key containing name
138

139
  """
140
  if isinstance(entity, basestring):
141
    result = entity
142
  elif isinstance(entity, dict):
143
    result = entity[key]
144
  else:
145
    raise qa_error.Error("Expected string or dictionary, got %s: %s" %
146
                         (type(entity), entity))
147

    
148
  if not ht.TNonEmptyString(result):
149
    raise Exception("Invalid name '%s'" % result)
150

    
151
  return result
152

    
153

    
154
def _AssertRetCode(rcode, fail, cmdstr, nodename):
155
  """Check the return value from a command and possibly raise an exception.
156

157
  """
158
  if fail and rcode == 0:
159
    raise qa_error.Error("Command '%s' on node %s was expected to fail but"
160
                         " didn't" % (cmdstr, nodename))
161
  elif not fail and rcode != 0:
162
    raise qa_error.Error("Command '%s' on node %s failed, exit code %s" %
163
                         (cmdstr, nodename, rcode))
164

    
165

    
166
def AssertCommand(cmd, fail=False, node=None, log_cmd=True):
167
  """Checks that a remote command succeeds.
168

169
  @param cmd: either a string (the command to execute) or a list (to
170
      be converted using L{utils.ShellQuoteArgs} into a string)
171
  @type fail: boolean
172
  @param fail: if the command is expected to fail instead of succeeding
173
  @param node: if passed, it should be the node on which the command
174
      should be executed, instead of the master node (can be either a
175
      dict or a string)
176
  @param log_cmd: if False, the command won't be logged (simply passed to
177
      StartSSH)
178
  @return: the return code of the command
179
  @raise qa_error.Error: if the command fails when it shouldn't or vice versa
180

181
  """
182
  if node is None:
183
    node = qa_config.GetMasterNode()
184

    
185
  nodename = _GetName(node, "primary")
186

    
187
  if isinstance(cmd, basestring):
188
    cmdstr = cmd
189
  else:
190
    cmdstr = utils.ShellQuoteArgs(cmd)
191

    
192
  rcode = StartSSH(nodename, cmdstr, log_cmd=log_cmd).wait()
193
  _AssertRetCode(rcode, fail, cmdstr, nodename)
194

    
195
  return rcode
196

    
197

    
198
def AssertRedirectedCommand(cmd, fail=False, node=None, log_cmd=True):
199
  """Executes a command with redirected output.
200

201
  The log will go to the qa-output log file in the ganeti log
202
  directory on the node where the command is executed. The fail and
203
  node parameters are passed unchanged to AssertCommand.
204

205
  @param cmd: the command to be executed, as a list; a string is not
206
      supported
207

208
  """
209
  if not isinstance(cmd, list):
210
    raise qa_error.Error("Non-list passed to AssertRedirectedCommand")
211
  ofile = utils.ShellQuote(_QA_OUTPUT)
212
  cmdstr = utils.ShellQuoteArgs(cmd)
213
  AssertCommand("echo ---- $(date) %s ---- >> %s" % (cmdstr, ofile),
214
                fail=False, node=node, log_cmd=False)
215
  return AssertCommand(cmdstr + " >> %s" % ofile,
216
                       fail=fail, node=node, log_cmd=log_cmd)
217

    
218

    
219
def GetSSHCommand(node, cmd, strict=True, opts=None, tty=None):
220
  """Builds SSH command to be executed.
221

222
  @type node: string
223
  @param node: node the command should run on
224
  @type cmd: string
225
  @param cmd: command to be executed in the node; if None or empty
226
      string, no command will be executed
227
  @type strict: boolean
228
  @param strict: whether to enable strict host key checking
229
  @type opts: list
230
  @param opts: list of additional options
231
  @type tty: boolean or None
232
  @param tty: if we should use tty; if None, will be auto-detected
233

234
  """
235
  args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-lroot"]
236

    
237
  if tty is None:
238
    tty = sys.stdout.isatty()
239

    
240
  if tty:
241
    args.append("-t")
242

    
243
  if strict:
244
    tmp = "yes"
245
  else:
246
    tmp = "no"
247
  args.append("-oStrictHostKeyChecking=%s" % tmp)
248
  args.append("-oClearAllForwardings=yes")
249
  args.append("-oForwardAgent=yes")
250
  if opts:
251
    args.extend(opts)
252
  if node in _MULTIPLEXERS:
253
    spath = _MULTIPLEXERS[node][0]
254
    args.append("-oControlPath=%s" % spath)
255
    args.append("-oControlMaster=no")
256
  args.append(node)
257
  if cmd:
258
    args.append(cmd)
259

    
260
  return args
261

    
262

    
263
def StartLocalCommand(cmd, _nolog_opts=False, log_cmd=True, **kwargs):
264
  """Starts a local command.
265

266
  """
267
  if log_cmd:
268
    if _nolog_opts:
269
      pcmd = [i for i in cmd if not i.startswith("-")]
270
    else:
271
      pcmd = cmd
272
    print "Command: %s" % utils.ShellQuoteArgs(pcmd)
273
  return subprocess.Popen(cmd, shell=False, **kwargs)
274

    
275

    
276
def StartSSH(node, cmd, strict=True, log_cmd=True):
277
  """Starts SSH.
278

279
  """
280
  return StartLocalCommand(GetSSHCommand(node, cmd, strict=strict),
281
                           _nolog_opts=True, log_cmd=log_cmd)
282

    
283

    
284
def StartMultiplexer(node):
285
  """Starts a multiplexer command.
286

287
  @param node: the node for which to open the multiplexer
288

289
  """
290
  if node in _MULTIPLEXERS:
291
    return
292

    
293
  # Note: yes, we only need mktemp, since we'll remove the file anyway
294
  sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
295
  utils.RemoveFile(sname)
296
  opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
297
  print "Created socket at %s" % sname
298
  child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
299
  _MULTIPLEXERS[node] = (sname, child)
300

    
301

    
302
def CloseMultiplexers():
303
  """Closes all current multiplexers and cleans up.
304

305
  """
306
  for node in _MULTIPLEXERS.keys():
307
    (sname, child) = _MULTIPLEXERS.pop(node)
308
    utils.KillProcess(child.pid, timeout=10, waitpid=True)
309
    utils.RemoveFile(sname)
310

    
311

    
312
def GetCommandOutput(node, cmd, tty=None, fail=False):
313
  """Returns the output of a command executed on the given node.
314

315
  @type node: string
316
  @param node: node the command should run on
317
  @type cmd: string
318
  @param cmd: command to be executed in the node (cannot be empty or None)
319
  @type tty: bool or None
320
  @param tty: if we should use tty; if None, it will be auto-detected
321
  @type fail: bool
322
  @param fail: whether the command is expected to fail
323
  """
324
  assert cmd
325
  p = StartLocalCommand(GetSSHCommand(node, cmd, tty=tty),
326
                        stdout=subprocess.PIPE)
327
  rcode = p.wait()
328
  _AssertRetCode(rcode, fail, node, cmd)
329
  return p.stdout.read()
330

    
331

    
332
def UploadFile(node, src):
333
  """Uploads a file to a node and returns the filename.
334

335
  Caller needs to remove the returned file on the node when it's not needed
336
  anymore.
337

338
  """
339
  # Make sure nobody else has access to it while preserving local permissions
340
  mode = os.stat(src).st_mode & 0700
341

    
342
  cmd = ('tmp=$(tempfile --mode %o --prefix gnt) && '
343
         '[[ -f "${tmp}" ]] && '
344
         'cat > "${tmp}" && '
345
         'echo "${tmp}"') % mode
346

    
347
  f = open(src, "r")
348
  try:
349
    p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
350
                         stdout=subprocess.PIPE)
351
    AssertEqual(p.wait(), 0)
352

    
353
    # Return temporary filename
354
    return p.stdout.read().strip()
355
  finally:
356
    f.close()
357

    
358

    
359
def UploadData(node, data, mode=0600, filename=None):
360
  """Uploads data to a node and returns the filename.
361

362
  Caller needs to remove the returned file on the node when it's not needed
363
  anymore.
364

365
  """
366
  if filename:
367
    tmp = "tmp=%s" % utils.ShellQuote(filename)
368
  else:
369
    tmp = "tmp=$(tempfile --mode %o --prefix gnt)" % mode
370
  cmd = ("%s && "
371
         "[[ -f \"${tmp}\" ]] && "
372
         "cat > \"${tmp}\" && "
373
         "echo \"${tmp}\"") % tmp
374

    
375
  p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False,
376
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE)
377
  p.stdin.write(data)
378
  p.stdin.close()
379
  AssertEqual(p.wait(), 0)
380

    
381
  # Return temporary filename
382
  return p.stdout.read().strip()
383

    
384

    
385
def BackupFile(node, path):
386
  """Creates a backup of a file on the node and returns the filename.
387

388
  Caller needs to remove the returned file on the node when it's not needed
389
  anymore.
390

391
  """
392
  cmd = ("tmp=$(tempfile --prefix .gnt --directory=$(dirname %s)) && "
393
         "[[ -f \"$tmp\" ]] && "
394
         "cp %s $tmp && "
395
         "echo $tmp") % (utils.ShellQuote(path), utils.ShellQuote(path))
396

    
397
  # Return temporary filename
398
  return GetCommandOutput(node, cmd).strip()
399

    
400

    
401
def _ResolveName(cmd, key):
402
  """Helper function.
403

404
  """
405
  master = qa_config.GetMasterNode()
406

    
407
  output = GetCommandOutput(master["primary"], utils.ShellQuoteArgs(cmd))
408
  for line in output.splitlines():
409
    (lkey, lvalue) = line.split(":", 1)
410
    if lkey == key:
411
      return lvalue.lstrip()
412
  raise KeyError("Key not found")
413

    
414

    
415
def ResolveInstanceName(instance):
416
  """Gets the full name of an instance.
417

418
  @type instance: string
419
  @param instance: Instance name
420

421
  """
422
  return _ResolveName(["gnt-instance", "info", instance],
423
                      "Instance name")
424

    
425

    
426
def ResolveNodeName(node):
427
  """Gets the full name of a node.
428

429
  """
430
  return _ResolveName(["gnt-node", "info", node["primary"]],
431
                      "Node name")
432

    
433

    
434
def GetNodeInstances(node, secondaries=False):
435
  """Gets a list of instances on a node.
436

437
  """
438
  master = qa_config.GetMasterNode()
439
  node_name = ResolveNodeName(node)
440

    
441
  # Get list of all instances
442
  cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
443
         "--output=name,pnode,snodes"]
444
  output = GetCommandOutput(master["primary"], utils.ShellQuoteArgs(cmd))
445

    
446
  instances = []
447
  for line in output.splitlines():
448
    (name, pnode, snodes) = line.split(":", 2)
449
    if ((not secondaries and pnode == node_name) or
450
        (secondaries and node_name in snodes.split(","))):
451
      instances.append(name)
452

    
453
  return instances
454

    
455

    
456
def _SelectQueryFields(rnd, fields):
457
  """Generates a list of fields for query tests.
458

459
  """
460
  # Create copy for shuffling
461
  fields = list(fields)
462
  rnd.shuffle(fields)
463

    
464
  # Check all fields
465
  yield fields
466
  yield sorted(fields)
467

    
468
  # Duplicate fields
469
  yield fields + fields
470

    
471
  # Check small groups of fields
472
  while fields:
473
    yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields]
474

    
475

    
476
def _List(listcmd, fields, names):
477
  """Runs a list command.
478

479
  """
480
  master = qa_config.GetMasterNode()
481

    
482
  cmd = [listcmd, "list", "--separator=|", "--no-headers",
483
         "--output", ",".join(fields)]
484

    
485
  if names:
486
    cmd.extend(names)
487

    
488
  return GetCommandOutput(master["primary"],
489
                          utils.ShellQuoteArgs(cmd)).splitlines()
490

    
491

    
492
def GenericQueryTest(cmd, fields, namefield="name", test_unknown=True):
493
  """Runs a number of tests on query commands.
494

495
  @param cmd: Command name
496
  @param fields: List of field names
497

498
  """
499
  rnd = random.Random(hash(cmd))
500

    
501
  fields = list(fields)
502
  rnd.shuffle(fields)
503

    
504
  # Test a number of field combinations
505
  for testfields in _SelectQueryFields(rnd, fields):
506
    AssertCommand([cmd, "list", "--output", ",".join(testfields)])
507

    
508
  if namefield is not None:
509
    namelist_fn = compat.partial(_List, cmd, [namefield])
510

    
511
    # When no names were requested, the list must be sorted
512
    names = namelist_fn(None)
513
    AssertEqual(names, utils.NiceSort(names))
514

    
515
    # When requesting specific names, the order must be kept
516
    revnames = list(reversed(names))
517
    AssertEqual(namelist_fn(revnames), revnames)
518

    
519
    randnames = list(names)
520
    rnd.shuffle(randnames)
521
    AssertEqual(namelist_fn(randnames), randnames)
522

    
523
  if test_unknown:
524
    # Listing unknown items must fail
525
    AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"],
526
                  fail=True)
527

    
528
  # Check exit code for listing unknown field
529
  AssertEqual(AssertCommand([cmd, "list", "--output=field/does/not/exist"],
530
                            fail=True),
531
              constants.EXIT_UNKNOWN_FIELD)
532

    
533

    
534
def GenericQueryFieldsTest(cmd, fields):
535
  master = qa_config.GetMasterNode()
536

    
537
  # Listing fields
538
  AssertCommand([cmd, "list-fields"])
539
  AssertCommand([cmd, "list-fields"] + fields)
540

    
541
  # Check listed fields (all, must be sorted)
542
  realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
543
  output = GetCommandOutput(master["primary"],
544
                            utils.ShellQuoteArgs(realcmd)).splitlines()
545
  AssertEqual([line.split("|", 1)[0] for line in output],
546
              utils.NiceSort(fields))
547

    
548
  # Check exit code for listing unknown field
549
  AssertEqual(AssertCommand([cmd, "list-fields", "field/does/not/exist"],
550
                            fail=True),
551
              constants.EXIT_UNKNOWN_FIELD)
552

    
553

    
554
def _FormatWithColor(text, seq):
555
  if not seq:
556
    return text
557
  return "%s%s%s" % (seq, text, _RESET_SEQ)
558

    
559

    
560
FormatWarning = lambda text: _FormatWithColor(text, _WARNING_SEQ)
561
FormatError = lambda text: _FormatWithColor(text, _ERROR_SEQ)
562
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
563

    
564

    
565
def AddToEtcHosts(hostnames):
566
  """Adds hostnames to /etc/hosts.
567

568
  @param hostnames: List of hostnames first used A records, all other CNAMEs
569

570
  """
571
  master = qa_config.GetMasterNode()
572
  tmp_hosts = UploadData(master["primary"], "", mode=0644)
573

    
574
  data = []
575
  for localhost in ("::1", "127.0.0.1"):
576
    data.append("%s %s" % (localhost, " ".join(hostnames)))
577

    
578
  try:
579
    AssertCommand("{ cat %s && echo -e '%s'; } > %s && mv %s %s" %
580
                  (utils.ShellQuote(pathutils.ETC_HOSTS),
581
                   "\\n".join(data),
582
                   utils.ShellQuote(tmp_hosts),
583
                   utils.ShellQuote(tmp_hosts),
584
                   utils.ShellQuote(pathutils.ETC_HOSTS)))
585
  except Exception:
586
    AssertCommand(["rm", "-f", tmp_hosts])
587
    raise
588

    
589

    
590
def RemoveFromEtcHosts(hostnames):
591
  """Remove hostnames from /etc/hosts.
592

593
  @param hostnames: List of hostnames first used A records, all other CNAMEs
594

595
  """
596
  master = qa_config.GetMasterNode()
597
  tmp_hosts = UploadData(master["primary"], "", mode=0644)
598
  quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
599

    
600
  sed_data = " ".join(hostnames)
601
  try:
602
    AssertCommand(("sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' %s > %s"
603
                   " && mv %s %s") %
604
                   (sed_data, utils.ShellQuote(pathutils.ETC_HOSTS),
605
                    quoted_tmp_hosts, quoted_tmp_hosts,
606
                    utils.ShellQuote(pathutils.ETC_HOSTS)))
607
  except Exception:
608
    AssertCommand(["rm", "-f", tmp_hosts])
609
    raise
610

    
611

    
612
def RunInstanceCheck(instance, running):
613
  """Check if instance is running or not.
614

615
  """
616
  instance_name = _GetName(instance, "name")
617

    
618
  script = qa_config.GetInstanceCheckScript()
619
  if not script:
620
    return
621

    
622
  master_node = qa_config.GetMasterNode()
623

    
624
  # Build command to connect to master node
625
  master_ssh = GetSSHCommand(master_node["primary"], "--")
626

    
627
  if running:
628
    running_shellval = "1"
629
    running_text = ""
630
  else:
631
    running_shellval = ""
632
    running_text = "not "
633

    
634
  print FormatInfo("Checking if instance '%s' is %srunning" %
635
                   (instance_name, running_text))
636

    
637
  args = [script, instance_name]
638
  env = {
639
    "PATH": constants.HOOKS_PATH,
640
    "RUN_UUID": _RUN_UUID,
641
    "MASTER_SSH": utils.ShellQuoteArgs(master_ssh),
642
    "INSTANCE_NAME": instance_name,
643
    "INSTANCE_RUNNING": running_shellval,
644
    }
645

    
646
  result = os.spawnve(os.P_WAIT, script, args, env)
647
  if result != 0:
648
    raise qa_error.Error("Instance check failed with result %s" % result)
649

    
650

    
651
def _InstanceCheckInner(expected, instarg, args, result):
652
  """Helper function used by L{InstanceCheck}.
653

654
  """
655
  if instarg == FIRST_ARG:
656
    instance = args[0]
657
  elif instarg == RETURN_VALUE:
658
    instance = result
659
  else:
660
    raise Exception("Invalid value '%s' for instance argument" % instarg)
661

    
662
  if expected in (INST_DOWN, INST_UP):
663
    RunInstanceCheck(instance, (expected == INST_UP))
664
  elif expected is not None:
665
    raise Exception("Invalid value '%s'" % expected)
666

    
667

    
668
def InstanceCheck(before, after, instarg):
669
  """Decorator to check instance status before and after test.
670

671
  @param before: L{INST_DOWN} if instance must be stopped before test,
672
    L{INST_UP} if instance must be running before test, L{None} to not check.
673
  @param after: L{INST_DOWN} if instance must be stopped after test,
674
    L{INST_UP} if instance must be running after test, L{None} to not check.
675
  @param instarg: L{FIRST_ARG} to use first argument to test as instance (a
676
    dictionary), L{RETURN_VALUE} to use return value (disallows pre-checks)
677

678
  """
679
  def decorator(fn):
680
    @functools.wraps(fn)
681
    def wrapper(*args, **kwargs):
682
      _InstanceCheckInner(before, instarg, args, NotImplemented)
683

    
684
      result = fn(*args, **kwargs)
685

    
686
      _InstanceCheckInner(after, instarg, args, result)
687

    
688
      return result
689
    return wrapper
690
  return decorator
691

    
692

    
693
def GetNonexistentGroups(count):
694
  """Gets group names which shouldn't exist on the cluster.
695

696
  @param count: Number of groups to get
697
  @rtype: list
698

699
  """
700
  groups = qa_config.get("groups", {})
701

    
702
  default = ["group1", "group2", "group3"]
703
  assert count <= len(default)
704

    
705
  candidates = groups.get("inexistent-groups", default)[:count]
706

    
707
  if len(candidates) < count:
708
    raise Exception("At least %s non-existent groups are needed" % count)
709

    
710
  return candidates