commit cec9845cb50cd5a76b3f8421895ee52684ae4d5c
Author: Michael Hanselmann <hansmi@google.com>
Date:   Thu Sep 13 12:46:25 2007 +0000

    Split QA script into different modules.
    
    Reviewed-by: iustinp

diff --git a/qa/Makefile.am b/qa/Makefile.am
index 495e87c..bee5d32 100644
--- a/qa/Makefile.am
+++ b/qa/Makefile.am
@@ -1,2 +1,11 @@
-EXTRA_DIST = ganeti-qa.py qa-sample.yaml
+EXTRA_DIST = ganeti-qa.py qa-sample.yaml \
+	qa_cluster.py \
+	qa_config.py \
+	qa_daemon.py \
+	qa_env.py \
+	qa_error.py \
+	qa_instance.py \
+	qa_node.py
+	qa_other.py \
+	qa_utils.py
 CLEANFILES = *.py[co]
diff --git a/qa/ganeti-qa.py b/qa/ganeti-qa.py
index f202e9f..30cb7ae 100755
--- a/qa/ganeti-qa.py
+++ b/qa/ganeti-qa.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 #
 
-# Copyright (C) 2006, 2007 Google Inc.
+# Copyright (C) 2007 Google Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,7 +19,7 @@
 # 02110-1301, USA.
 
 
-"""Script for doing Q&A on Ganeti
+"""Script for doing QA on Ganeti.
 
 You can create the required known_hosts file using ssh-keyscan. It's mandatory
 to use the full name of a node (FQDN). For security reasons, verify the keys
@@ -27,63 +27,27 @@ before using them.
 Example: ssh-keyscan -t rsa node{1,2,3,4}.example.com > known_hosts
 """
 
-import os
-import re
 import sys
-import yaml
-import time
-import tempfile
 from datetime import datetime
 from optparse import OptionParser
 
-# I want more flexibility for testing over SSH, therefore I'm not using
-# Ganeti's ssh module.
-import subprocess
+import qa_cluster
+import qa_config
+import qa_daemon
+import qa_env
+import qa_instance
+import qa_node
+import qa_other
 
-from ganeti import utils
-from ganeti import constants
 
-# {{{ Global variables
-cfg = None
-options = None
-# }}}
-
-# {{{ Errors
-class Error(Exception):
-  """An error occurred during Q&A testing.
-
-  """
-  pass
-
-
-class OutOfNodesError(Error):
-  """Out of nodes.
-
-  """
-  pass
-
-
-class OutOfInstancesError(Error):
-  """Out of instances.
-
-  """
-  pass
-# }}}
-
-# {{{ Utilities
-def TestEnabled(test):
-  """Returns True if the given test is enabled."""
-  return cfg.get('tests', {}).get(test, False)
-
-
-def RunTest(callable, *args):
+def RunTest(fn, *args):
   """Runs a test after printing a header.
 
   """
-  if callable.__doc__:
-    desc = callable.__doc__.splitlines()[0].strip()
+  if fn.__doc__:
+    desc = fn.__doc__.splitlines()[0].strip()
   else:
-    desc = '%r' % callable
+    desc = '%r' % fn
 
   now = str(datetime.now())
 
@@ -92,577 +56,13 @@ def RunTest(callable, *args):
   print desc
   print '-' * 60
 
-  return callable(*args)
-
-
-def AssertEqual(first, second, msg=None):
-  """Raises an error when values aren't equal.
-
-  """
-  if not first == second:
-    raise Error(msg or '%r == %r' % (first, second))
-
-
-def GetSSHCommand(node, cmd, strict=True):
-  """Builds SSH command to be executed.
-
-  """
-  args = [ 'ssh', '-oEscapeChar=none', '-oBatchMode=yes', '-l', 'root' ]
-
-  if strict:
-    tmp = 'yes'
-  else:
-    tmp = 'no'
-  args.append('-oStrictHostKeyChecking=%s' % tmp)
-  args.append('-oClearAllForwardings=yes')
-  args.append('-oForwardAgent=yes')
-  args.append(node)
-
-  if options.dry_run:
-    prefix = 'exit 0; '
-  else:
-    prefix = ''
-
-  args.append(prefix + cmd)
-
-  print 'SSH:', utils.ShellQuoteArgs(args)
-
-  return args
-
-
-def StartSSH(node, cmd, strict=True):
-  """Starts SSH.
-
-  """
-  args = GetSSHCommand(node, cmd, strict=strict)
-  return subprocess.Popen(args, shell=False)
-
-
-def UploadFile(node, file):
-  """Uploads a file to a node and returns the filename.
-
-  Caller needs to remove the returned file on the node when it's not needed
-  anymore.
-  """
-  # Make sure nobody else has access to it while preserving local permissions
-  mode = os.stat(file).st_mode & 0700
-
-  cmd = ('tmp=$(tempfile --mode %o --prefix gnt) && '
-         '[[ -f "${tmp}" ]] && '
-         'cat > "${tmp}" && '
-         'echo "${tmp}"') % mode
-
-  f = open(file, 'r')
-  try:
-    p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
-                         stdout=subprocess.PIPE)
-    AssertEqual(p.wait(), 0)
-
-    # Return temporary filename
-    return p.stdout.read().strip()
-  finally:
-    f.close()
-# }}}
-
-# {{{ Config helpers
-def GetMasterNode():
-  return cfg['nodes'][0]
-
-
-def AcquireInstance():
-  """Returns an instance which isn't in use.
-
-  """
-  # Filter out unwanted instances
-  tmp_flt = lambda inst: not inst.get('_used', False)
-  instances = filter(tmp_flt, cfg['instances'])
-  del tmp_flt
-
-  if len(instances) == 0:
-    raise OutOfInstancesError("No instances left")
+  return fn(*args)
 
-  inst = instances[0]
-  inst['_used'] = True
-  return inst
 
-
-def ReleaseInstance(inst):
-  inst['_used'] = False
-
-
-def AcquireNode(exclude=None):
-  """Returns the least used node.
-
-  """
-  master = GetMasterNode()
-
-  # Filter out unwanted nodes
-  # TODO: Maybe combine filters
-  if exclude is None:
-    nodes = cfg['nodes'][:]
-  else:
-    nodes = filter(lambda node: node != exclude, cfg['nodes'])
-
-  tmp_flt = lambda node: node.get('_added', False) or node == master
-  nodes = filter(tmp_flt, nodes)
-  del tmp_flt
-
-  if len(nodes) == 0:
-    raise OutOfNodesError("No nodes left")
-
-  # Get node with least number of uses
-  def compare(a, b):
-    result = cmp(a.get('_count', 0), b.get('_count', 0))
-    if result == 0:
-      result = cmp(a['primary'], b['primary'])
-    return result
-
-  nodes.sort(cmp=compare)
-
-  node = nodes[0]
-  node['_count'] = node.get('_count', 0) + 1
-  return node
-
-
-def ReleaseNode(node):
-  node['_count'] = node.get('_count', 0) - 1
-# }}}
-
-# {{{ Environment tests
-def TestConfig():
-  """Test configuration for sanity.
-
-  """
-  if len(cfg['nodes']) < 1:
-    raise Error("Need at least one node")
-  if len(cfg['instances']) < 1:
-    raise Error("Need at least one instance")
-  # TODO: Add more checks
-
-
-def TestSshConnection():
-  """Test SSH connection.
-
-  """
-  for node in cfg['nodes']:
-    AssertEqual(StartSSH(node['primary'], 'exit').wait(), 0)
-
-
-def TestGanetiCommands():
-  """Test availibility of Ganeti commands.
+def main():
+  """Main program.
 
   """
-  cmds = ( ['gnt-cluster', '--version'],
-           ['gnt-os', '--version'],
-           ['gnt-node', '--version'],
-           ['gnt-instance', '--version'],
-           ['gnt-backup', '--version'],
-           ['ganeti-noded', '--version'],
-           ['ganeti-watcher', '--version'] )
-
-  cmd = ' && '.join([utils.ShellQuoteArgs(i) for i in cmds])
-
-  for node in cfg['nodes']:
-    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
-
-
-def TestIcmpPing():
-  """ICMP ping each node.
-
-  """
-  for node in cfg['nodes']:
-    check = []
-    for i in cfg['nodes']:
-      check.append(i['primary'])
-      if i.has_key('secondary'):
-        check.append(i['secondary'])
-
-    ping = lambda ip: utils.ShellQuoteArgs(['ping', '-w', '3', '-c', '1', ip])
-    cmd = ' && '.join([ping(i) for i in check])
-
-    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
-# }}}
-
-# {{{ Cluster tests
-def TestClusterInit():
-  """gnt-cluster init"""
-  master = GetMasterNode()
-
-  cmd = ['gnt-cluster', 'init']
-  if master.get('secondary', None):
-    cmd.append('--secondary-ip=%s' % master['secondary'])
-  if cfg.get('bridge', None):
-    cmd.append('--bridge=%s' % cfg['bridge'])
-    cmd.append('--master-netdev=%s' % cfg['bridge'])
-  cmd.append(cfg['name'])
-
-  AssertEqual(StartSSH(master['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestClusterVerify():
-  """gnt-cluster verify"""
-  cmd = ['gnt-cluster', 'verify']
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestClusterInfo():
-  """gnt-cluster info"""
-  cmd = ['gnt-cluster', 'info']
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestClusterBurnin():
-  """Burnin"""
-  master = GetMasterNode()
-
-  # Get as many instances as we need
-  instances = []
-  try:
-    for _ in xrange(0, cfg.get('options', {}).get('burnin-instances', 1)):
-      instances.append(AcquireInstance())
-  except OutOfInstancesError:
-    print "Not enough instances, continuing anyway."
-
-  if len(instances) < 1:
-    raise Error("Burnin needs at least one instance")
-
-  # Run burnin
-  try:
-    script = UploadFile(master['primary'], '../tools/burnin')
-    try:
-      cmd = [script,
-             '--os=%s' % cfg['os'],
-             '--os-size=%s' % cfg['os-size'],
-             '--swap-size=%s' % cfg['swap-size']]
-      cmd += [inst['name'] for inst in instances]
-      AssertEqual(StartSSH(master['primary'],
-                           utils.ShellQuoteArgs(cmd)).wait(), 0)
-    finally:
-      cmd = ['rm', '-f', script]
-      AssertEqual(StartSSH(master['primary'],
-                           utils.ShellQuoteArgs(cmd)).wait(), 0)
-  finally:
-    for inst in instances:
-      ReleaseInstance(inst)
-
-
-def TestClusterMasterFailover():
-  """gnt-cluster masterfailover"""
-  master = GetMasterNode()
-
-  failovermaster = AcquireNode(exclude=master)
-  try:
-    cmd = ['gnt-cluster', 'masterfailover']
-    AssertEqual(StartSSH(failovermaster['primary'],
-                         utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-    cmd = ['gnt-cluster', 'masterfailover']
-    AssertEqual(StartSSH(master['primary'],
-                         utils.ShellQuoteArgs(cmd)).wait(), 0)
-  finally:
-    ReleaseNode(failovermaster)
-
-
-def TestClusterCopyfile():
-  """gnt-cluster copyfile"""
-  master = GetMasterNode()
-
-  # Create temporary file
-  f = tempfile.NamedTemporaryFile()
-  f.write("I'm a testfile.\n")
-  f.flush()
-  f.seek(0)
-
-  # Upload file to master node
-  testname = UploadFile(master['primary'], f.name)
-  try:
-    # Copy file to all nodes
-    cmd = ['gnt-cluster', 'copyfile', testname]
-    AssertEqual(StartSSH(master['primary'],
-                         utils.ShellQuoteArgs(cmd)).wait(), 0)
-  finally:
-    # Remove file from all nodes
-    for node in cfg['nodes']:
-      cmd = ['rm', '-f', testname]
-      AssertEqual(StartSSH(node['primary'],
-                           utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestClusterDestroy():
-  """gnt-cluster destroy"""
-  cmd = ['gnt-cluster', 'destroy', '--yes-do-it']
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-# }}}
-
-# {{{ Node tests
-def _NodeAdd(node):
-  if node.get('_added', False):
-    raise Error("Node %s already in cluster" % node['primary'])
-
-  cmd = ['gnt-node', 'add']
-  if node.get('secondary', None):
-    cmd.append('--secondary-ip=%s' % node['secondary'])
-  cmd.append(node['primary'])
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-  node['_added'] = True
-
-
-def TestNodeAddAll():
-  """Adding all nodes to cluster."""
-  master = GetMasterNode()
-  for node in cfg['nodes']:
-    if node != master:
-      _NodeAdd(node)
-
-
-def _NodeRemove(node):
-  cmd = ['gnt-node', 'remove', node['primary']]
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-  node['_added'] = False
-
-
-def TestNodeRemoveAll():
-  """Removing all nodes from cluster."""
-  master = GetMasterNode()
-  for node in cfg['nodes']:
-    if node != master:
-      _NodeRemove(node)
-
-
-def TestNodeInfo():
-  """gnt-node info"""
-  cmd = ['gnt-node', 'info']
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestNodeVolumes():
-  """gnt-node volumes"""
-  cmd = ['gnt-node', 'volumes']
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-# }}}
-
-# {{{ Instance tests
-def _DiskTest(node, instance, args):
-  cmd = ['gnt-instance', 'add',
-         '--os-type=%s' % cfg['os'],
-         '--os-size=%s' % cfg['os-size'],
-         '--swap-size=%s' % cfg['swap-size'],
-         '--memory=%s' % cfg['mem'],
-         '--node=%s' % node['primary']]
-  if args:
-    cmd += args
-  cmd.append(instance['name'])
-
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-  return instance
-
-
-def TestInstanceAddWithPlainDisk(node):
-  """gnt-instance add -t plain"""
-  return _DiskTest(node, AcquireInstance(), ['--disk-template=plain'])
-
-
-def TestInstanceAddWithLocalMirrorDisk(node):
-  """gnt-instance add -t local_raid1"""
-  return _DiskTest(node, AcquireInstance(), ['--disk-template=local_raid1'])
-
-
-def TestInstanceAddWithRemoteRaidDisk(node, node2):
-  """gnt-instance add -t remote_raid1"""
-  return _DiskTest(node, AcquireInstance(),
-                   ['--disk-template=remote_raid1',
-                    '--secondary-node=%s' % node2['primary']])
-
-
-def TestInstanceRemove(instance):
-  """gnt-instance remove"""
-  cmd = ['gnt-instance', 'remove', '-f', instance['name']]
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-  ReleaseInstance(instance)
-
-
-def TestInstanceStartup(instance):
-  """gnt-instance startup"""
-  cmd = ['gnt-instance', 'startup', instance['name']]
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestInstanceShutdown(instance):
-  """gnt-instance shutdown"""
-  cmd = ['gnt-instance', 'shutdown', instance['name']]
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestInstanceFailover(instance):
-  """gnt-instance failover"""
-  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestInstanceInfo(instance):
-  """gnt-instance info"""
-  cmd = ['gnt-instance', 'info', instance['name']]
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-# }}}
-
-# {{{ Daemon tests
-def _ResolveInstanceName(instance):
-  """Gets the full Xen name of an instance.
-
-  """
-  master = GetMasterNode()
-
-  info_cmd = utils.ShellQuoteArgs(['gnt-instance', 'info', instance['name']])
-  sed_cmd = utils.ShellQuoteArgs(['sed', '-n', '-e', 's/^Instance name: *//p'])
-
-  cmd = '%s | %s' % (info_cmd, sed_cmd)
-  p = subprocess.Popen(GetSSHCommand(master['primary'], cmd), shell=False,
-                       stdout=subprocess.PIPE)
-  AssertEqual(p.wait(), 0)
-
-  return p.stdout.read().strip()
-
-
-def _InstanceRunning(node, name):
-  """Checks whether an instance is running.
-
-  Args:
-    node: Node the instance runs on
-    name: Full name of Xen instance
-  """
-  cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
-  ret = StartSSH(node['primary'], cmd).wait()
-  return ret == 0
-
-
-def _XmShutdownInstance(node, name):
-  """Shuts down instance using "xm" and waits for completion.
-
-  Args:
-    node: Node the instance runs on
-    name: Full name of Xen instance
-  """
-  cmd = ['xm', 'shutdown', name]
-  AssertEqual(StartSSH(GetMasterNode()['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-  # Wait up to a minute
-  end = time.time() + 60
-  while time.time() <= end:
-    if not _InstanceRunning(node, name):
-      break
-    time.sleep(5)
-  else:
-    raise Error("xm shutdown failed")
-
-
-def _ResetWatcherDaemon(node):
-  """Removes the watcher daemon's state file.
-
-  Args:
-    node: Node to be reset
-  """
-  cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
-  AssertEqual(StartSSH(node['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestInstanceAutomaticRestart(node, instance):
-  """Test automatic restart of instance by ganeti-watcher.
-
-  Note: takes up to 6 minutes to complete.
-  """
-  master = GetMasterNode()
-  inst_name = _ResolveInstanceName(instance)
-
-  _ResetWatcherDaemon(node)
-  _XmShutdownInstance(node, inst_name)
-
-  # Give it a bit more than five minutes to start again
-  restart_at = time.time() + 330
-
-  # Wait until it's running again
-  while time.time() <= restart_at:
-    if _InstanceRunning(node, inst_name):
-      break
-    time.sleep(15)
-  else:
-    raise Error("Daemon didn't restart instance in time")
-
-  cmd = ['gnt-instance', 'info', inst_name]
-  AssertEqual(StartSSH(master['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-
-
-def TestInstanceConsecutiveFailures(node, instance):
-  """Test five consecutive instance failures.
-
-  Note: takes at least 35 minutes to complete.
-  """
-  master = GetMasterNode()
-  inst_name = _ResolveInstanceName(instance)
-
-  _ResetWatcherDaemon(node)
-  _XmShutdownInstance(node, inst_name)
-
-  # Do shutdowns for 30 minutes
-  finished_at = time.time() + (35 * 60)
-
-  while time.time() <= finished_at:
-    if _InstanceRunning(node, inst_name):
-      _XmShutdownInstance(node, inst_name)
-    time.sleep(30)
-
-  # Check for some time whether the instance doesn't start again
-  check_until = time.time() + 330
-  while time.time() <= check_until:
-    if _InstanceRunning(node, inst_name):
-      raise Error("Instance started when it shouldn't")
-    time.sleep(30)
-
-  cmd = ['gnt-instance', 'info', inst_name]
-  AssertEqual(StartSSH(master['primary'],
-                       utils.ShellQuoteArgs(cmd)).wait(), 0)
-# }}}
-
-# {{{ Other tests
-def TestUploadKnownHostsFile(localpath):
-  """Uploading known_hosts file.
-
-  """
-  master = GetMasterNode()
-
-  tmpfile = UploadFile(master['primary'], localpath)
-  try:
-    cmd = ['mv', tmpfile, constants.SSH_KNOWN_HOSTS_FILE]
-    AssertEqual(StartSSH(master['primary'],
-                         utils.ShellQuoteArgs(cmd)).wait(), 0)
-  except:
-    cmd = ['rm', '-f', tmpfile]
-    AssertEqual(StartSSH(master['primary'],
-                         utils.ShellQuoteArgs(cmd)).wait(), 0)
-    raise
-# }}}
-
-# {{{ Main program
-if __name__ == '__main__':
-  # {{{ Option parsing
   parser = OptionParser(usage="%prog [options] <config-file> "
                               "<known-hosts-file>")
   parser.add_option('--dry-run', dest='dry_run',
@@ -671,121 +71,116 @@ if __name__ == '__main__':
   parser.add_option('--yes-do-it', dest='yes_do_it',
       action="store_true",
       help="Really execute the tests")
-  (options, args) = parser.parse_args()
-  # }}}
+  (qa_config.options, args) = parser.parse_args()
 
   if len(args) == 2:
     (config_file, known_hosts_file) = args
   else:
     parser.error("Not enough arguments.")
 
-  if not options.yes_do_it:
+  if not qa_config.options.yes_do_it:
     print ("Executing this script irreversibly destroys any Ganeti\n"
            "configuration on all nodes involved. If you really want\n"
            "to start testing, supply the --yes-do-it option.")
     sys.exit(1)
 
-  f = open(config_file, 'r')
-  try:
-    cfg = yaml.load(f.read())
-  finally:
-    f.close()
-
-  RunTest(TestConfig)
+  qa_config.Load(config_file)
 
-  RunTest(TestUploadKnownHostsFile, known_hosts_file)
+  RunTest(qa_other.TestUploadKnownHostsFile, known_hosts_file)
 
-  if TestEnabled('env'):
-    RunTest(TestSshConnection)
-    RunTest(TestIcmpPing)
-    RunTest(TestGanetiCommands)
+  if qa_config.TestEnabled('env'):
+    RunTest(qa_env.TestSshConnection)
+    RunTest(qa_env.TestIcmpPing)
+    RunTest(qa_env.TestGanetiCommands)
 
-  RunTest(TestClusterInit)
+  RunTest(qa_cluster.TestClusterInit)
 
-  RunTest(TestNodeAddAll)
+  RunTest(qa_node.TestNodeAddAll)
 
-  if TestEnabled('cluster-verify'):
-    RunTest(TestClusterVerify)
+  if qa_config.TestEnabled('cluster-verify'):
+    RunTest(qa_cluster.TestClusterVerify)
 
-  if TestEnabled('cluster-info'):
-    RunTest(TestClusterInfo)
+  if qa_config.TestEnabled('cluster-info'):
+    RunTest(qa_cluster.TestClusterInfo)
 
-  if TestEnabled('cluster-copyfile'):
-    RunTest(TestClusterCopyfile)
+  if qa_config.TestEnabled('cluster-copyfile'):
+    RunTest(qa_cluster.TestClusterCopyfile)
 
-  if TestEnabled('node-info'):
-    RunTest(TestNodeInfo)
+  if qa_config.TestEnabled('node-info'):
+    RunTest(qa_node.TestNodeInfo)
 
-  if TestEnabled('cluster-burnin'):
-    RunTest(TestClusterBurnin)
+  if qa_config.TestEnabled('cluster-burnin'):
+    RunTest(qa_cluster.TestClusterBurnin)
 
-  if TestEnabled('cluster-master-failover'):
-    RunTest(TestClusterMasterFailover)
+  if qa_config.TestEnabled('cluster-master-failover'):
+    RunTest(qa_cluster.TestClusterMasterFailover)
 
-  node = AcquireNode()
+  node = qa_config.AcquireNode()
   try:
-    if TestEnabled('instance-add-plain-disk'):
-      instance = RunTest(TestInstanceAddWithPlainDisk, node)
-      RunTest(TestInstanceShutdown, instance)
-      RunTest(TestInstanceStartup, instance)
+    if qa_config.TestEnabled('instance-add-plain-disk'):
+      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, node)
+      RunTest(qa_instance.TestInstanceShutdown, instance)
+      RunTest(qa_instance.TestInstanceStartup, instance)
 
-      if TestEnabled('instance-info'):
-        RunTest(TestInstanceInfo, instance)
+      if qa_config.TestEnabled('instance-info'):
+        RunTest(qa_instance.TestInstanceInfo, instance)
 
-      if TestEnabled('instance-automatic-restart'):
-        RunTest(TestInstanceAutomaticRestart, node, instance)
+      if qa_config.TestEnabled('instance-automatic-restart'):
+        RunTest(qa_daemon.TestInstanceAutomaticRestart, node, instance)
 
-      if TestEnabled('instance-consecutive-failures'):
-        RunTest(TestInstanceConsecutiveFailures, node, instance)
+      if qa_config.TestEnabled('instance-consecutive-failures'):
+        RunTest(qa_daemon.TestInstanceConsecutiveFailures, node, instance)
 
-      if TestEnabled('node-volumes'):
-        RunTest(TestNodeVolumes)
+      if qa_config.TestEnabled('node-volumes'):
+        RunTest(qa_node.TestNodeVolumes)
 
-      RunTest(TestInstanceRemove, instance)
+      RunTest(qa_instance.TestInstanceRemove, instance)
       del instance
 
-    if TestEnabled('instance-add-local-mirror-disk'):
-      instance = RunTest(TestInstanceAddWithLocalMirrorDisk, node)
-      RunTest(TestInstanceShutdown, instance)
-      RunTest(TestInstanceStartup, instance)
+    if qa_config.TestEnabled('instance-add-local-mirror-disk'):
+      instance = RunTest(qa_instance.TestInstanceAddWithLocalMirrorDisk, node)
+      RunTest(qa_instance.TestInstanceShutdown, instance)
+      RunTest(qa_instance.TestInstanceStartup, instance)
 
-      if TestEnabled('instance-info'):
-        RunTest(TestInstanceInfo, instance)
+      if qa_config.TestEnabled('instance-info'):
+        RunTest(qa_instance.TestInstanceInfo, instance)
 
-      if TestEnabled('node-volumes'):
-        RunTest(TestNodeVolumes)
+      if qa_config.TestEnabled('node-volumes'):
+        RunTest(qa_node.TestNodeVolumes)
 
-      RunTest(TestInstanceRemove, instance)
+      RunTest(qa_instance.TestInstanceRemove, instance)
       del instance
 
-    if TestEnabled('instance-add-remote-raid-disk'):
-      node2 = AcquireNode(exclude=node)
+    if qa_config.TestEnabled('instance-add-remote-raid-disk'):
+      node2 = qa_config.AcquireNode(exclude=node)
       try:
-        instance = RunTest(TestInstanceAddWithRemoteRaidDisk, node, node2)
-        RunTest(TestInstanceShutdown, instance)
-        RunTest(TestInstanceStartup, instance)
+        instance = RunTest(qa_instance.TestInstanceAddWithRemoteRaidDisk,
+                           node, node2)
+        RunTest(qa_instance.TestInstanceShutdown, instance)
+        RunTest(qa_instance.TestInstanceStartup, instance)
 
-        if TestEnabled('instance-info'):
-          RunTest(TestInstanceInfo, instance)
+        if qa_config.TestEnabled('instance-info'):
+          RunTest(qa_instance.TestInstanceInfo, instance)
 
-        if TestEnabled('instance-failover'):
-          RunTest(TestInstanceFailover, instance)
+        if qa_config.TestEnabled('instance-failover'):
+          RunTest(qa_instance.TestInstanceFailover, instance)
 
-        if TestEnabled('node-volumes'):
-          RunTest(TestNodeVolumes)
+        if qa_config.TestEnabled('node-volumes'):
+          RunTest(qa_node.TestNodeVolumes)
 
-        RunTest(TestInstanceRemove, instance)
+        RunTest(qa_instance.TestInstanceRemove, instance)
         del instance
       finally:
-        ReleaseNode(node2)
+        qa_config.ReleaseNode(node2)
 
   finally:
-    ReleaseNode(node)
+    qa_config.ReleaseNode(node)
 
-  RunTest(TestNodeRemoveAll)
+  RunTest(qa_node.TestNodeRemoveAll)
 
-  if TestEnabled('cluster-destroy'):
-    RunTest(TestClusterDestroy)
-# }}}
+  if qa_config.TestEnabled('cluster-destroy'):
+    RunTest(qa_cluster.TestClusterDestroy)
 
-# vim: foldmethod=marker :
+
+if __name__ == '__main__':
+  main()
diff --git a/qa/qa_cluster.py b/qa/qa_cluster.py
new file mode 100644
index 0000000..d9ba6e2
--- /dev/null
+++ b/qa/qa_cluster.py
@@ -0,0 +1,156 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Cluster related QA tests.
+
+"""
+
+import tempfile
+
+from ganeti import utils
+
+import qa_config
+import qa_utils
+import qa_error
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def TestClusterInit():
+  """gnt-cluster init"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-cluster', 'init']
+
+  if master.get('secondary', None):
+    cmd.append('--secondary-ip=%s' % master['secondary'])
+
+  bridge = qa_config.get('bridge', None)
+  if bridge:
+    cmd.append('--bridge=%s' % bridge)
+    cmd.append('--master-netdev=%s' % bridge)
+
+  cmd.append(qa_config.get('name'))
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterVerify():
+  """gnt-cluster verify"""
+  cmd = ['gnt-cluster', 'verify']
+  master = qa_config.GetMasterNode()
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterInfo():
+  """gnt-cluster info"""
+  cmd = ['gnt-cluster', 'info']
+  master = qa_config.GetMasterNode()
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterBurnin():
+  """Burnin"""
+  master = qa_config.GetMasterNode()
+
+  # Get as many instances as we need
+  instances = []
+  try:
+    num = qa_config.get('options', {}).get('burnin-instances', 1)
+    for _ in xrange(0, num):
+      instances.append(qa_config.AcquireInstance())
+  except qa_error.OutOfInstancesError:
+    print "Not enough instances, continuing anyway."
+
+  if len(instances) < 1:
+    raise qa_error.Error("Burnin needs at least one instance")
+
+  # Run burnin
+  try:
+    script = qa_utils.UploadFile(master['primary'], '../tools/burnin')
+    try:
+      cmd = [script,
+             '--os=%s' % qa_config.get('os'),
+             '--os-size=%s' % qa_config.get('os-size'),
+             '--swap-size=%s' % qa_config.get('swap-size')]
+      cmd += [inst['name'] for inst in instances]
+      AssertEqual(StartSSH(master['primary'],
+                           utils.ShellQuoteArgs(cmd)).wait(), 0)
+    finally:
+      cmd = ['rm', '-f', script]
+      AssertEqual(StartSSH(master['primary'],
+                           utils.ShellQuoteArgs(cmd)).wait(), 0)
+  finally:
+    for inst in instances:
+      qa_config.ReleaseInstance(inst)
+
+
+def TestClusterMasterFailover():
+  """gnt-cluster masterfailover"""
+  master = qa_config.GetMasterNode()
+
+  failovermaster = qa_config.AcquireNode(exclude=master)
+  try:
+    cmd = ['gnt-cluster', 'masterfailover']
+    AssertEqual(StartSSH(failovermaster['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+    cmd = ['gnt-cluster', 'masterfailover']
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+  finally:
+    qa_config.ReleaseNode(failovermaster)
+
+
+def TestClusterCopyfile():
+  """gnt-cluster copyfile"""
+  master = qa_config.GetMasterNode()
+
+  # Create temporary file
+  f = tempfile.NamedTemporaryFile()
+  f.write("I'm a testfile.\n")
+  f.flush()
+  f.seek(0)
+
+  # Upload file to master node
+  testname = qa_utils.UploadFile(master['primary'], f.name)
+  try:
+    # Copy file to all nodes
+    cmd = ['gnt-cluster', 'copyfile', testname]
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+  finally:
+    # Remove file from all nodes
+    for node in qa_config.get('nodes'):
+      cmd = ['rm', '-f', testname]
+      AssertEqual(StartSSH(node['primary'],
+                           utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestClusterDestroy():
+  """gnt-cluster destroy"""
+  cmd = ['gnt-cluster', 'destroy', '--yes-do-it']
+  master = qa_config.GetMasterNode()
+
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
diff --git a/qa/qa_config.py b/qa/qa_config.py
new file mode 100644
index 0000000..bf176ba
--- /dev/null
+++ b/qa/qa_config.py
@@ -0,0 +1,124 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""QA configuration.
+
+"""
+
+
+import yaml
+
+import qa_error
+
+
+cfg = None
+options = None
+
+
+def Load(path):
+  """Loads the passed configuration file.
+
+  """
+  global cfg
+
+  f = open(path, 'r')
+  try:
+    cfg = yaml.load(f.read())
+  finally:
+    f.close()
+
+  Validate()
+
+
+def Validate():
+  if len(cfg['nodes']) < 1:
+    raise qa_error.Error("Need at least one node")
+  if len(cfg['instances']) < 1:
+    raise qa_error.Error("Need at least one instance")
+
+
+def get(name, default=None):
+  return cfg.get(name, default)
+
+
+def TestEnabled(test):
+  """Returns True if the given test is enabled."""
+  return cfg.get('tests', {}).get(test, False)
+
+
+def GetMasterNode():
+  return cfg['nodes'][0]
+
+
+def AcquireInstance():
+  """Returns an instance which isn't in use.
+
+  """
+  # Filter out unwanted instances
+  tmp_flt = lambda inst: not inst.get('_used', False)
+  instances = filter(tmp_flt, cfg['instances'])
+  del tmp_flt
+
+  if len(instances) == 0:
+    raise qa_error.OutOfInstancesError("No instances left")
+
+  inst = instances[0]
+  inst['_used'] = True
+  return inst
+
+
+def ReleaseInstance(inst):
+  inst['_used'] = False
+
+
+def AcquireNode(exclude=None):
+  """Returns the least used node.
+
+  """
+  master = GetMasterNode()
+
+  # Filter out unwanted nodes
+  # TODO: Maybe combine filters
+  if exclude is None:
+    nodes = cfg['nodes'][:]
+  else:
+    nodes = filter(lambda node: node != exclude, cfg['nodes'])
+
+  tmp_flt = lambda node: node.get('_added', False) or node == master
+  nodes = filter(tmp_flt, nodes)
+  del tmp_flt
+
+  if len(nodes) == 0:
+    raise qa_error.OutOfNodesError("No nodes left")
+
+  # Get node with least number of uses
+  def compare(a, b):
+    result = cmp(a.get('_count', 0), b.get('_count', 0))
+    if result == 0:
+      result = cmp(a['primary'], b['primary'])
+    return result
+
+  nodes.sort(cmp=compare)
+
+  node = nodes[0]
+  node['_count'] = node.get('_count', 0) + 1
+  return node
+
+
+def ReleaseNode(node):
+  node['_count'] = node.get('_count', 0) - 1
diff --git a/qa/qa_daemon.py b/qa/qa_daemon.py
new file mode 100644
index 0000000..1fb3951
--- /dev/null
+++ b/qa/qa_daemon.py
@@ -0,0 +1,154 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Daemon related QA tests.
+
+"""
+
+import time
+import subprocess
+
+from ganeti import utils
+from ganeti import constants
+
+import qa_config
+import qa_utils
+import qa_error
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def _ResolveInstanceName(instance):
+  """Gets the full Xen name of an instance.
+
+  """
+  master = qa_config.GetMasterNode()
+
+  info_cmd = utils.ShellQuoteArgs(['gnt-instance', 'info', instance['name']])
+  sed_cmd = utils.ShellQuoteArgs(['sed', '-n', '-e', 's/^Instance name: *//p'])
+
+  cmd = '%s | %s' % (info_cmd, sed_cmd)
+  ssh_cmd = qa_utils.GetSSHCommand(master['primary'], cmd)
+  p = subprocess.Popen(ssh_cmd, shell=False, stdout=subprocess.PIPE)
+  AssertEqual(p.wait(), 0)
+
+  return p.stdout.read().strip()
+
+
+def _InstanceRunning(node, name):
+  """Checks whether an instance is running.
+
+  Args:
+    node: Node the instance runs on
+    name: Full name of Xen instance
+  """
+  cmd = utils.ShellQuoteArgs(['xm', 'list', name]) + ' >/dev/null'
+  ret = StartSSH(node['primary'], cmd).wait()
+  return ret == 0
+
+
+def _XmShutdownInstance(node, name):
+  """Shuts down instance using "xm" and waits for completion.
+
+  Args:
+    node: Node the instance runs on
+    name: Full name of Xen instance
+  """
+  master = qa_config.GetMasterNode()
+
+  cmd = ['xm', 'shutdown', name]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+  # Wait up to a minute
+  end = time.time() + 60
+  while time.time() <= end:
+    if not _InstanceRunning(node, name):
+      break
+    time.sleep(5)
+  else:
+    raise qa_error.Error("xm shutdown failed")
+
+
+def _ResetWatcherDaemon(node):
+  """Removes the watcher daemon's state file.
+
+  Args:
+    node: Node to be reset
+  """
+  cmd = ['rm', '-f', constants.WATCHER_STATEFILE]
+  AssertEqual(StartSSH(node['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceAutomaticRestart(node, instance):
+  """Test automatic restart of instance by ganeti-watcher.
+
+  Note: takes up to 6 minutes to complete.
+  """
+  master = qa_config.GetMasterNode()
+  inst_name = _ResolveInstanceName(instance)
+
+  _ResetWatcherDaemon(node)
+  _XmShutdownInstance(node, inst_name)
+
+  # Give it a bit more than five minutes to start again
+  restart_at = time.time() + 330
+
+  # Wait until it's running again
+  while time.time() <= restart_at:
+    if _InstanceRunning(node, inst_name):
+      break
+    time.sleep(15)
+  else:
+    raise qa_error.Error("Daemon didn't restart instance in time")
+
+  cmd = ['gnt-instance', 'info', inst_name]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceConsecutiveFailures(node, instance):
+  """Test five consecutive instance failures.
+
+  Note: takes at least 35 minutes to complete.
+  """
+  master = qa_config.GetMasterNode()
+  inst_name = _ResolveInstanceName(instance)
+
+  _ResetWatcherDaemon(node)
+  _XmShutdownInstance(node, inst_name)
+
+  # Do shutdowns for 30 minutes
+  finished_at = time.time() + (35 * 60)
+
+  while time.time() <= finished_at:
+    if _InstanceRunning(node, inst_name):
+      _XmShutdownInstance(node, inst_name)
+    time.sleep(30)
+
+  # Check for some time whether the instance doesn't start again
+  check_until = time.time() + 330
+  while time.time() <= check_until:
+    if _InstanceRunning(node, inst_name):
+      raise qa_error.Error("Instance started when it shouldn't")
+    time.sleep(30)
+
+  cmd = ['gnt-instance', 'info', inst_name]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
diff --git a/qa/qa_env.py b/qa/qa_env.py
new file mode 100644
index 0000000..266fb11
--- /dev/null
+++ b/qa/qa_env.py
@@ -0,0 +1,72 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Cluster environment related QA tests.
+
+"""
+
+from ganeti import utils
+
+import qa_config
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def TestSshConnection():
+  """Test SSH connection.
+
+  """
+  for node in qa_config.get('nodes'):
+    AssertEqual(StartSSH(node['primary'], 'exit').wait(), 0)
+
+
+def TestGanetiCommands():
+  """Test availibility of Ganeti commands.
+
+  """
+  cmds = ( ['gnt-cluster', '--version'],
+           ['gnt-os', '--version'],
+           ['gnt-node', '--version'],
+           ['gnt-instance', '--version'],
+           ['gnt-backup', '--version'],
+           ['ganeti-noded', '--version'],
+           ['ganeti-watcher', '--version'] )
+
+  cmd = ' && '.join([utils.ShellQuoteArgs(i) for i in cmds])
+
+  for node in qa_config.get('nodes'):
+    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
+
+
+def TestIcmpPing():
+  """ICMP ping each node.
+
+  """
+  nodes = qa_config.get('nodes')
+
+  for node in nodes:
+    check = []
+    for i in nodes:
+      check.append(i['primary'])
+      if i.has_key('secondary'):
+        check.append(i['secondary'])
+
+    ping = lambda ip: utils.ShellQuoteArgs(['ping', '-w', '3', '-c', '1', ip])
+    cmd = ' && '.join([ping(i) for i in check])
+
+    AssertEqual(StartSSH(node['primary'], cmd).wait(), 0)
diff --git a/qa/qa_error.py b/qa/qa_error.py
new file mode 100644
index 0000000..60c1a46
--- /dev/null
+++ b/qa/qa_error.py
@@ -0,0 +1,37 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+class Error(Exception):
+  """An error occurred during Q&A testing.
+
+  """
+  pass
+
+
+class OutOfNodesError(Error):
+  """Out of nodes.
+
+  """
+  pass
+
+
+class OutOfInstancesError(Error):
+  """Out of instances.
+
+  """
+  pass
diff --git a/qa/qa_instance.py b/qa/qa_instance.py
new file mode 100644
index 0000000..ef2346b
--- /dev/null
+++ b/qa/qa_instance.py
@@ -0,0 +1,114 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Instance related QA tests.
+
+"""
+
+from ganeti import utils
+
+import qa_config
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def _DiskTest(node, args):
+  master = qa_config.GetMasterNode()
+
+  instance = qa_config.AcquireInstance()
+  try:
+    cmd = ['gnt-instance', 'add',
+           '--os-type=%s' % qa_config.get('os'),
+           '--os-size=%s' % qa_config.get('os-size'),
+           '--swap-size=%s' % qa_config.get('swap-size'),
+           '--memory=%s' % qa_config.get('mem'),
+           '--node=%s' % node['primary']]
+    if args:
+      cmd += args
+    cmd.append(instance['name'])
+
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+    return instance
+  except:
+    qa_config.ReleaseInstance(instance)
+    raise
+
+
+def TestInstanceAddWithPlainDisk(node):
+  """gnt-instance add -t plain"""
+  return _DiskTest(node, ['--disk-template=plain'])
+
+
+def TestInstanceAddWithLocalMirrorDisk(node):
+  """gnt-instance add -t local_raid1"""
+  return _DiskTest(node, ['--disk-template=local_raid1'])
+
+
+def TestInstanceAddWithRemoteRaidDisk(node, node2):
+  """gnt-instance add -t remote_raid1"""
+  return _DiskTest(node,
+                   ['--disk-template=remote_raid1',
+                    '--secondary-node=%s' % node2['primary']])
+
+
+def TestInstanceRemove(instance):
+  """gnt-instance remove"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'remove', '-f', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+  qa_config.ReleaseInstance(instance)
+
+
+def TestInstanceStartup(instance):
+  """gnt-instance startup"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'startup', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceShutdown(instance):
+  """gnt-instance shutdown"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'shutdown', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceFailover(instance):
+  """gnt-instance failover"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestInstanceInfo(instance):
+  """gnt-instance info"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-instance', 'info', instance['name']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
diff --git a/qa/qa_node.py b/qa/qa_node.py
new file mode 100644
index 0000000..44f89b1
--- /dev/null
+++ b/qa/qa_node.py
@@ -0,0 +1,83 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+from ganeti import utils
+
+import qa_config
+import qa_error
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def _NodeAdd(node):
+  master = qa_config.GetMasterNode()
+
+  if node.get('_added', False):
+    raise qa_error.Error("Node %s already in cluster" % node['primary'])
+
+  cmd = ['gnt-node', 'add']
+  if node.get('secondary', None):
+    cmd.append('--secondary-ip=%s' % node['secondary'])
+  cmd.append(node['primary'])
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+  node['_added'] = True
+
+
+def _NodeRemove(node):
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-node', 'remove', node['primary']]
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+  node['_added'] = False
+
+
+def TestNodeAddAll():
+  """Adding all nodes to cluster."""
+  master = qa_config.GetMasterNode()
+  for node in qa_config.get('nodes'):
+    if node != master:
+      _NodeAdd(node)
+
+
+def TestNodeRemoveAll():
+  """Removing all nodes from cluster."""
+  master = qa_config.GetMasterNode()
+  for node in qa_config.get('nodes'):
+    if node != master:
+      _NodeRemove(node)
+
+
+def TestNodeInfo():
+  """gnt-node info"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-node', 'info']
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
+
+
+def TestNodeVolumes():
+  """gnt-node volumes"""
+  master = qa_config.GetMasterNode()
+
+  cmd = ['gnt-node', 'volumes']
+  AssertEqual(StartSSH(master['primary'],
+                       utils.ShellQuoteArgs(cmd)).wait(), 0)
diff --git a/qa/qa_other.py b/qa/qa_other.py
new file mode 100644
index 0000000..6882254
--- /dev/null
+++ b/qa/qa_other.py
@@ -0,0 +1,43 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+from ganeti import utils
+from ganeti import constants
+
+import qa_config
+import qa_utils
+
+from qa_utils import AssertEqual, StartSSH
+
+
+def TestUploadKnownHostsFile(localpath):
+  """Uploading known_hosts file.
+
+  """
+  master = qa_config.GetMasterNode()
+
+  tmpfile = qa_utils.UploadFile(master['primary'], localpath)
+  try:
+    cmd = ['mv', tmpfile, constants.SSH_KNOWN_HOSTS_FILE]
+    AssertEqual(StartSSH(master['primary'],
+                         utils.ShellQuoteArgs(cmd)).wait(), 0)
+  except:
+    cmd = ['rm', '-f', tmpfile]
+    AssertEqual(StartSSH(master['primary'],
+                utils.ShellQuoteArgs(cmd)).wait(), 0)
+    raise
diff --git a/qa/qa_utils.py b/qa/qa_utils.py
new file mode 100644
index 0000000..3889352
--- /dev/null
+++ b/qa/qa_utils.py
@@ -0,0 +1,99 @@
+# Copyright (C) 2007 Google Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+
+"""Utilities for QA tests.
+
+"""
+
+import os
+import subprocess
+
+from ganeti import utils
+
+import qa_config
+import qa_error
+
+
+def AssertEqual(first, second, msg=None):
+  """Raises an error when values aren't equal.
+
+  """
+  if not first == second:
+    raise qa_error.Error(msg or '%r == %r' % (first, second))
+
+
+def GetSSHCommand(node, cmd, strict=True):
+  """Builds SSH command to be executed.
+
+  """
+  args = [ 'ssh', '-oEscapeChar=none', '-oBatchMode=yes', '-l', 'root' ]
+
+  if strict:
+    tmp = 'yes'
+  else:
+    tmp = 'no'
+  args.append('-oStrictHostKeyChecking=%s' % tmp)
+  args.append('-oClearAllForwardings=yes')
+  args.append('-oForwardAgent=yes')
+  args.append(node)
+
+  if qa_config.options.dry_run:
+    prefix = 'exit 0; '
+  else:
+    prefix = ''
+
+  args.append(prefix + cmd)
+
+  print 'SSH:', utils.ShellQuoteArgs(args)
+
+  return args
+
+
+def StartSSH(node, cmd, strict=True):
+  """Starts SSH.
+
+  """
+  args = GetSSHCommand(node, cmd, strict=strict)
+  return subprocess.Popen(args, shell=False)
+
+
+def UploadFile(node, src):
+  """Uploads a file to a node and returns the filename.
+
+  Caller needs to remove the returned file on the node when it's not needed
+  anymore.
+  """
+  # Make sure nobody else has access to it while preserving local permissions
+  mode = os.stat(src).st_mode & 0700
+
+  cmd = ('tmp=$(tempfile --mode %o --prefix gnt) && '
+         '[[ -f "${tmp}" ]] && '
+         'cat > "${tmp}" && '
+         'echo "${tmp}"') % mode
+
+  f = open(src, 'r')
+  try:
+    p = subprocess.Popen(GetSSHCommand(node, cmd), shell=False, stdin=f,
+                         stdout=subprocess.PIPE)
+    AssertEqual(p.wait(), 0)
+
+    # Return temporary filename
+    return p.stdout.read().strip()
+  finally:
+    f.close()
+# }}}
