QA: use a persistent SSH connection to the master

The recent additions to QA (many more tests) make QA slow if the
machine on which the QA runs is not very close to the tested nodes —
or in general, when the SSH handhaske is costly.

We discussed before about using a persistent connection, and here is
the patch that implements it. On a very small QA (very very small), it
cuts down a lot of time (almost half), so it should be useful even for
a full QA.

I've also thought about changing from external ssh to paramiko, but I
estimated that it would be more work to correctly interleave the IO
from the remote process than just running a background SSH.

Also note that yes, the global dict is ugly, but I don't know of
another simple way to implement this.

Signed-off-by: Iustin Pop <iustin@google.com>
Reviewed-by: Michael Hanselmann <hansmi@google.com>

diff --git a/qa/ganeti-qa.py b/qa/ganeti-qa.py
index 3d61634..6dc909d 100755 (executable)
--- a/qa/ganeti-qa.py
+++ b/qa/ganeti-qa.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python -u
 #
 
-# Copyright (C) 2007, 2008, 2009, 2010 Google Inc.
+# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -354,30 +354,10 @@ def RunHardwareFailureTests(instance, pnode, snode):
             pnode, snode)
 
 
-@rapi.client.UsesRapiClient
-def main():
-  """Main program.
+def RunQa():
+  """Main QA body.
 
   """
-  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
-  parser.add_option('--yes-do-it', dest='yes_do_it',
-      action="store_true",
-      help="Really execute the tests")
-  (qa_config.options, args) = parser.parse_args()
-
-  if len(args) == 1:
-    (config_file, ) = args
-  else:
-    parser.error("Wrong number of arguments.")
-
-  if not qa_config.options.yes_do_it:
-    print ("Executing this script irreversibly destroys any Ganeti\n"
-           "configuration on all nodes involved. If you really want\n"
-           "to start testing, supply the --yes-do-it option.")
-    sys.exit(1)
-
-  qa_config.Load(config_file)
-
   rapi_user = "ganeti-qa"
   rapi_secret = utils.GenerateSecret()
 
@@ -476,5 +456,35 @@ def main():
   RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
 
 
+@rapi.client.UsesRapiClient
+def main():
+  """Main program.
+
+  """
+  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
+  parser.add_option('--yes-do-it', dest='yes_do_it',
+      action="store_true",
+      help="Really execute the tests")
+  (qa_config.options, args) = parser.parse_args()
+
+  if len(args) == 1:
+    (config_file, ) = args
+  else:
+    parser.error("Wrong number of arguments.")
+
+  if not qa_config.options.yes_do_it:
+    print ("Executing this script irreversibly destroys any Ganeti\n"
+           "configuration on all nodes involved. If you really want\n"
+           "to start testing, supply the --yes-do-it option.")
+    sys.exit(1)
+
+  qa_config.Load(config_file)
+
+  qa_utils.StartMultiplexer(qa_config.GetMasterNode()["primary"])
+  try:
+    RunQa()
+  finally:
+    qa_utils.CloseMultiplexers()
+
 if __name__ == '__main__':
   main()

diff --git a/qa/qa_utils.py b/qa/qa_utils.py
index 110f4a3..20856f9 100644 (file)
--- a/qa/qa_utils.py
+++ b/qa/qa_utils.py
@@ -1,7 +1,7 @@
 #
 #
 
-# Copyright (C) 2007 Google Inc.
+# Copyright (C) 2007, 2011 Google Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,6 +28,7 @@ import re
 import sys
 import subprocess
 import random
+import tempfile
 
 from ganeti import utils
 from ganeti import compat
@@ -42,6 +43,8 @@ _WARNING_SEQ = None
 _ERROR_SEQ = None
 _RESET_SEQ = None
 
+_MULTIPLEXERS = {}
+
 
 def _SetupColours():
   """Initializes the colour constants.
@@ -143,15 +146,18 @@ def AssertCommand(cmd, fail=False, node=None):
   return rcode
 
 
-def GetSSHCommand(node, cmd, strict=True):
+def GetSSHCommand(node, cmd, strict=True, opts=None):
   """Builds SSH command to be executed.
 
   @type node: string
   @param node: node the command should run on
   @type cmd: string
-  @param cmd: command to be executed in the node
+  @param cmd: command to be executed in the node; if None or empty
+      string, no command will be executed
   @type strict: boolean
   @param strict: whether to enable strict host key checking
+  @type opts: list
+  @param opts: list of additional options
 
   """
   args = [ 'ssh', '-oEscapeChar=none', '-oBatchMode=yes', '-l', 'root', '-t' ]
@@ -163,8 +169,15 @@ def GetSSHCommand(node, cmd, strict=True):
   args.append('-oStrictHostKeyChecking=%s' % tmp)
   args.append('-oClearAllForwardings=yes')
   args.append('-oForwardAgent=yes')
+  if opts:
+    args.extend(opts)
+  if node in _MULTIPLEXERS:
+    spath = _MULTIPLEXERS[node][0]
+    args.append('-oControlPath=%s' % spath)
+    args.append('-oControlMaster=no')
   args.append(node)
-  args.append(cmd)
+  if cmd:
+    args.append(cmd)
 
   return args
 
@@ -184,6 +197,34 @@ def StartSSH(node, cmd, strict=True):
   return StartLocalCommand(GetSSHCommand(node, cmd, strict=strict))
 
 
+def StartMultiplexer(node):
+  """Starts a multiplexer command.
+
+  @param node: the node for which to open the multiplexer
+
+  """
+  if node in _MULTIPLEXERS:
+    return
+
+  # Note: yes, we only need mktemp, since we'll remove the file anyway
+  sname = tempfile.mktemp(prefix="ganeti-qa-multiplexer.")
+  utils.RemoveFile(sname)
+  opts = ["-N", "-oControlPath=%s" % sname, "-oControlMaster=yes"]
+  print "Created socket at %s" % sname
+  child = StartLocalCommand(GetSSHCommand(node, None, opts=opts))
+  _MULTIPLEXERS[node] = (sname, child)
+
+
+def CloseMultiplexers():
+  """Closes all current multiplexers and cleans up.
+
+  """
+  for node in _MULTIPLEXERS.keys():
+    (sname, child) = _MULTIPLEXERS.pop(node)
+    utils.KillProcess(child.pid, timeout=10, waitpid=True)
+    utils.RemoveFile(sname)
+
+
 def GetCommandOutput(node, cmd):
   """Returns the output of a command executed on the given node.