-# Copyright (C) 2007 Google Inc.
+#
+#
+
+# Copyright (C) 2007, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
"""
import os
+import re
import sys
import subprocess
+import random
+import tempfile
from ganeti import utils
+from ganeti import compat
+from ganeti import constants
import qa_config
import qa_error
_ERROR_SEQ = None
_RESET_SEQ = None
-
-# List of all hooks
-_hooks = []
+_MULTIPLEXERS = {}
def _SetupColours():
"""Initializes the colour constants.
"""
+ # pylint: disable=W0603
+ # due to global usage
global _INFO_SEQ, _WARNING_SEQ, _ERROR_SEQ, _RESET_SEQ
# Don't use colours if stdout isn't a terminal
_SetupColours()
+def AssertIn(item, sequence):
+ """Raises an error when item is not in sequence.
+
+ """
+ if item not in sequence:
+ raise qa_error.Error("%r not in %r" % (item, sequence))
+
+
+def AssertNotIn(item, sequence):
+ """Raises an error when item is in sequence.
+
+ """
+ if item in sequence:
+ raise qa_error.Error("%r in %r" % (item, sequence))
+
+
def AssertEqual(first, second):
"""Raises an error when values aren't equal.
"""
if not first == second:
- raise qa_error.Error('%r == %r' % (first, second))
+ raise qa_error.Error("%r == %r" % (first, second))
def AssertNotEqual(first, second):
"""
if not first != second:
- raise qa_error.Error('%r != %r' % (first, second))
+ raise qa_error.Error("%r != %r" % (first, second))
+
+
+def AssertMatch(string, pattern):
+ """Raises an error when string doesn't match regexp pattern.
+
+ """
+ if not re.match(pattern, string):
+ raise qa_error.Error("%r doesn't match /%r/" % (string, pattern))
+
+
+def AssertCommand(cmd, fail=False, node=None):
+ """Checks that a remote command succeeds.
+
+ @param cmd: either a string (the command to execute) or a list (to
+ be converted using L{utils.ShellQuoteArgs} into a string)
+ @type fail: boolean
+ @param fail: if the command is expected to fail instead of succeeding
+ @param node: if passed, it should be the node on which the command
+ should be executed, instead of the master node (can be either a
+ dict or a string)
+
+ """
+ if node is None:
+ node = qa_config.GetMasterNode()
+
+ if isinstance(node, basestring):
+ nodename = node
+ else:
+ nodename = node["primary"]
+
+ if isinstance(cmd, basestring):
+ cmdstr = cmd
+ else:
+ cmdstr = utils.ShellQuoteArgs(cmd)
+
+ rcode = StartSSH(nodename, cmdstr).wait()
+
+ if fail:
+ if rcode == 0:
+ raise qa_error.Error("Command '%s' on node %s was expected to fail but"
+ " didn't" % (cmdstr, nodename))
+ else:
+ if rcode != 0:
+ raise qa_error.Error("Command '%s' on node %s failed, exit code %s" %
+ (cmdstr, nodename, rcode))
+
+ return rcode
-def GetSSHCommand(node, cmd, strict=True):
+def GetSSHCommand(node, cmd, strict=True, opts=None, tty=True):
"""Builds SSH command to be executed.
+ @type node: string
+ @param node: node the command should run on
+ @type cmd: string
+ @param cmd: command to be executed in the node; if None or empty
+ string, no command will be executed
+ @type strict: boolean
+ @param strict: whether to enable strict host key checking
+ @type opts: list
+ @param opts: list of additional options
+ @type tty: Bool
+ @param tty: If we should use tty
+
"""
- args = [ 'ssh', '-oEscapeChar=none', '-oBatchMode=yes', '-l', 'root' ]
+ args = ["ssh", "-oEscapeChar=none", "-oBatchMode=yes", "-l", "root"]
+
+ if tty:
+ args.append("-t")
if strict:
- tmp = 'yes'
+ tmp = "yes"
else:
- tmp = 'no'
- args.append('-oStrictHostKeyChecking=%s' % tmp)
- args.append('-oClearAllForwardings=yes')
- args.append('-oForwardAgent=yes')
+ tmp = "no"
+ args.append("-oStrictHostKeyChecking=%s" % tmp)
+ args.append("-oClearAllForwardings=yes")
+ args.append("-oForwardAgent=yes")
+ if opts:
+ args.extend(opts)
+ if node in _MULTIPLEXERS:
+ spath = _MULTIPLEXERS[node][0]
+ args.append("-oControlPath=%s" % spath)
+ args.append("-oControlMaster=no")
args.append(node)
+ if cmd:
+ args.append(cmd)
- if qa_config.options.dry_run:
- prefix = 'exit 0; '
- else:
- prefix = ''
+ return args
- args.append(prefix + cmd)
- print 'SSH:', utils.ShellQuoteArgs(args)
+def StartLocalCommand(cmd, **kwargs):
+ """Starts a local command.
- return args
+ """
+ print "Command: %s" % utils.ShellQuoteArgs(cmd)
+ return subprocess.Popen(cmd, shell=False, **kwargs)
def StartSSH(node, cmd, strict=True):
"""Starts SSH.
"""
- return subprocess.Popen(GetSSHCommand(node, cmd, strict=strict),
- shell=False)
+ return StartLocalCommand(GetSSHCommand(node, cmd, strict=strict))
+
+
+def StartMultiplexer(node):
+ """Starts a multiplexer command.
+ @param node: the node for which to open the multiplexer
-def GetCommandOutput(node, cmd):
+ """
+ if node in _MULTIPLEXERS:
+ return
+
+ # Note: yes, we only need mktemp, since we'll remove the file anyway
+ sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
+ utils.RemoveFile(sname)
+ opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
+ print "Created socket at %s" % sname
+ child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
+ _MULTIPLEXERS[node] = (sname, child)
+
+
+def CloseMultiplexers():
+ """Closes all current multiplexers and cleans up.
+
+ """
+ for node in _MULTIPLEXERS.keys():
+ (sname, child) = _MULTIPLEXERS.pop(node)
+ utils.KillProcess(child.pid, timeout=10, waitpid=True)
+ utils.RemoveFile(sname)
+
+
+def GetCommandOutput(node, cmd, tty=True):
"""Returns the output of a command executed on the given node.
"""
- p = subprocess.Popen(GetSSHCommand(node, cmd),
- shell=False, stdout=subprocess.PIPE)
+ p = StartLocalCommand(GetSSHCommand(node, cmd, tty=tty),
+ stdout=subprocess.PIPE)
AssertEqual(p.wait(), 0)
return p.stdout.read()
Caller needs to remove the returned file on the node when it's not needed
anymore.
+
"""
# Make sure nobody else has access to it while preserving local permissions
mode = os.stat(src).st_mode & 0700
'cat > "${tmp}" && '
'echo "${tmp}"') % mode
- f = open(src, 'r')
+ f = open(src, "r")
try:
p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
stdout=subprocess.PIPE)
f.close()
+def UploadData(node, data, mode=0600, filename=None):
+ """Uploads data to a node and returns the filename.
+
+ Caller needs to remove the returned file on the node when it's not needed
+ anymore.
+
+ """
+ if filename:
+ tmp = "tmp=%s" % utils.ShellQuote(filename)
+ else:
+ tmp = "tmp=$(tempfile --mode %o --prefix gnt)" % mode
+ cmd = ("%s && "
+ "[[ -f \"${tmp}\" ]] && "
+ "cat > \"${tmp}\" && "
+ "echo \"${tmp}\"") % tmp
+
+ p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ p.stdin.write(data)
+ p.stdin.close()
+ AssertEqual(p.wait(), 0)
+
+ # Return temporary filename
+ return p.stdout.read().strip()
+
+
+def BackupFile(node, path):
+ """Creates a backup of a file on the node and returns the filename.
+
+ Caller needs to remove the returned file on the node when it's not needed
+ anymore.
+
+ """
+ cmd = ("tmp=$(tempfile --prefix .gnt --directory=$(dirname %s)) && "
+ "[[ -f \"$tmp\" ]] && "
+ "cp %s $tmp && "
+ "echo $tmp") % (utils.ShellQuote(path), utils.ShellQuote(path))
+
+ # Return temporary filename
+ return GetCommandOutput(node, cmd).strip()
+
+
def _ResolveName(cmd, key):
"""Helper function.
"""
master = qa_config.GetMasterNode()
- output = GetCommandOutput(master['primary'], utils.ShellQuoteArgs(cmd))
+ output = GetCommandOutput(master["primary"], utils.ShellQuoteArgs(cmd))
for line in output.splitlines():
- (lkey, lvalue) = line.split(':', 1)
+ (lkey, lvalue) = line.split(":", 1)
if lkey == key:
return lvalue.lstrip()
raise KeyError("Key not found")
def ResolveInstanceName(instance):
"""Gets the full name of an instance.
+ @type instance: string
+ @param instance: Instance name
+
"""
- return _ResolveName(['gnt-instance', 'info', instance['name']],
- 'Instance name')
+ return _ResolveName(["gnt-instance", "info", instance],
+ "Instance name")
def ResolveNodeName(node):
"""Gets the full name of a node.
"""
- return _ResolveName(['gnt-node', 'info', node['primary']],
- 'Node name')
+ return _ResolveName(["gnt-node", "info", node["primary"]],
+ "Node name")
def GetNodeInstances(node, secondaries=False):
"""
master = qa_config.GetMasterNode()
-
node_name = ResolveNodeName(node)
# Get list of all instances
- cmd = ['gnt-instance', 'list', '--separator=:', '--no-headers',
- '--output=name,pnode,snodes']
- output = GetCommandOutput(master['primary'], utils.ShellQuoteArgs(cmd))
+ cmd = ["gnt-instance", "list", "--separator=:", "--no-headers",
+ "--output=name,pnode,snodes"]
+ output = GetCommandOutput(master["primary"], utils.ShellQuoteArgs(cmd))
instances = []
for line in output.splitlines():
- (name, pnode, snodes) = line.split(':', 2)
+ (name, pnode, snodes) = line.split(":", 2)
if ((not secondaries and pnode == node_name) or
- (secondaries and node_name in snodes.split(','))):
+ (secondaries and node_name in snodes.split(","))):
instances.append(name)
return instances
+def _SelectQueryFields(rnd, fields):
+ """Generates a list of fields for query tests.
+
+ """
+ # Create copy for shuffling
+ fields = list(fields)
+ rnd.shuffle(fields)
+
+ # Check all fields
+ yield fields
+ yield sorted(fields)
+
+ # Duplicate fields
+ yield fields + fields
+
+ # Check small groups of fields
+ while fields:
+ yield [fields.pop() for _ in range(rnd.randint(2, 10)) if fields]
+
+
+def _List(listcmd, fields, names):
+ """Runs a list command.
+
+ """
+ master = qa_config.GetMasterNode()
+
+ cmd = [listcmd, "list", "--separator=|", "--no-header",
+ "--output", ",".join(fields)]
+
+ if names:
+ cmd.extend(names)
+
+ return GetCommandOutput(master["primary"],
+ utils.ShellQuoteArgs(cmd)).splitlines()
+
+
+def GenericQueryTest(cmd, fields):
+ """Runs a number of tests on query commands.
+
+ @param cmd: Command name
+ @param fields: List of field names
+
+ """
+ rnd = random.Random(hash(cmd))
+
+ fields = list(fields)
+ rnd.shuffle(fields)
+
+ # Test a number of field combinations
+ for testfields in _SelectQueryFields(rnd, fields):
+ AssertCommand([cmd, "list", "--output", ",".join(testfields)])
+
+ namelist_fn = compat.partial(_List, cmd, ["name"])
+
+ # When no names were requested, the list must be sorted
+ names = namelist_fn(None)
+ AssertEqual(names, utils.NiceSort(names))
+
+ # When requesting specific names, the order must be kept
+ revnames = list(reversed(names))
+ AssertEqual(namelist_fn(revnames), revnames)
+
+ randnames = list(names)
+ rnd.shuffle(randnames)
+ AssertEqual(namelist_fn(randnames), randnames)
+
+ # Listing unknown items must fail
+ AssertCommand([cmd, "list", "this.name.certainly.does.not.exist"], fail=True)
+
+ # Check exit code for listing unknown field
+ AssertEqual(AssertCommand([cmd, "list", "--output=field/does/not/exist"],
+ fail=True),
+ constants.EXIT_UNKNOWN_FIELD)
+
+
+def GenericQueryFieldsTest(cmd, fields):
+ master = qa_config.GetMasterNode()
+
+ # Listing fields
+ AssertCommand([cmd, "list-fields"])
+ AssertCommand([cmd, "list-fields"] + fields)
+
+ # Check listed fields (all, must be sorted)
+ realcmd = [cmd, "list-fields", "--separator=|", "--no-headers"]
+ output = GetCommandOutput(master["primary"],
+ utils.ShellQuoteArgs(realcmd)).splitlines()
+ AssertEqual([line.split("|", 1)[0] for line in output],
+ utils.NiceSort(fields))
+
+ # Check exit code for listing unknown field
+ AssertEqual(AssertCommand([cmd, "list-fields", "field/does/not/exist"],
+ fail=True),
+ constants.EXIT_UNKNOWN_FIELD)
+
+
def _FormatWithColor(text, seq):
if not seq:
return text
FormatInfo = lambda text: _FormatWithColor(text, _INFO_SEQ)
-def LoadHooks():
- """Load all QA hooks.
+def AddToEtcHosts(hostnames):
+ """Adds hostnames to /etc/hosts.
- """
- hooks_dir = qa_config.get('options', {}).get('hooks-dir', None)
- if not hooks_dir:
- return
- if hooks_dir not in sys.path:
- sys.path.insert(0, hooks_dir)
- for name in utils.ListVisibleFiles(hooks_dir):
- if name.endswith('.py'):
- # Load and instanciate hook
- _hooks.append(__import__(name[:-3], None, None, ['']).hook())
-
-
-class QaHookContext:
- name = None
- phase = None
- success = None
- args = None
- kwargs = None
-
-
-def _CallHooks(ctx):
- """Calls all hooks with the given context.
+ @param hostnames: List of hostnames first used A records, all other CNAMEs
"""
- if not _hooks:
- return
+ master = qa_config.GetMasterNode()
+ tmp_hosts = UploadData(master["primary"], "", mode=0644)
- name = "%s-%s" % (ctx.phase, ctx.name)
- if ctx.success is not None:
- msg = "%s (success=%s)" % (name, ctx.success)
- else:
- msg = name
- print FormatInfo("Begin %s" % msg)
- for hook in _hooks:
- hook.run(ctx)
- print FormatInfo("End %s" % name)
+ quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
+ data = []
+ for localhost in ("::1", "127.0.0.1"):
+ data.append("%s %s" % (localhost, " ".join(hostnames)))
+
+ try:
+ AssertCommand(("cat /etc/hosts > %s && echo -e '%s' >> %s && mv %s"
+ " /etc/hosts") % (quoted_tmp_hosts, "\\n".join(data),
+ quoted_tmp_hosts, quoted_tmp_hosts))
+ except qa_error.Error:
+ AssertCommand(["rm", tmp_hosts])
-def DefineHook(name):
- """Wraps a function with calls to hooks.
+def RemoveFromEtcHosts(hostnames):
+ """Remove hostnames from /etc/hosts.
- Usage: prefix function with @qa_utils.DefineHook(...)
+ @param hostnames: List of hostnames first used A records, all other CNAMEs
"""
- def wrapper(fn):
- def new_f(*args, **kwargs):
- # Create context
- ctx = QaHookContext()
- ctx.name = name
- ctx.phase = 'pre'
- ctx.args = args
- ctx.kwargs = kwargs
-
- _CallHooks(ctx)
- try:
- ctx.phase = 'post'
- ctx.success = True
- try:
- # Call real function
- return fn(*args, **kwargs)
- except:
- ctx.success = False
- raise
- finally:
- _CallHooks(ctx)
-
- # Override function metadata
- new_f.func_name = fn.func_name
- new_f.func_doc = fn.func_doc
-
- return new_f
-
- return wrapper
+ master = qa_config.GetMasterNode()
+ tmp_hosts = UploadData(master["primary"], "", mode=0644)
+ quoted_tmp_hosts = utils.ShellQuote(tmp_hosts)
+
+ sed_data = " ".join(hostnames)
+ try:
+ AssertCommand(("sed -e '/^\(::1\|127\.0\.0\.1\)\s\+%s/d' /etc/hosts > %s"
+ " && mv %s /etc/hosts") % (sed_data, quoted_tmp_hosts,
+ quoted_tmp_hosts))
+ except qa_error.Error:
+ AssertCommand(["rm", tmp_hosts])