Rename masterfailover to master-failover
[ganeti-local] / lib / bootstrap.py
index 4f3fa98..5a2c5ba 100644 (file)
@@ -1,7 +1,7 @@
 #
 #
 
-# Copyright (C) 2006, 2007, 2008 Google Inc.
+# Copyright (C) 2006, 2007, 2008, 2010 Google Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,9 +25,9 @@
 
 import os
 import os.path
-import sha
 import re
 import logging
+import time
 
 from ganeti import rpc
 from ganeti import ssh
@@ -37,6 +37,11 @@ from ganeti import config
 from ganeti import constants
 from ganeti import objects
 from ganeti import ssconf
+from ganeti import serializer
+from ganeti import hypervisor
+from ganeti import bdev
+from ganeti import netutils
+
 
 def _InitSSHSetup():
   """Setup the SSH configuration for the cluster.
@@ -59,43 +64,165 @@ def _InitSSHSetup():
     raise errors.OpExecError("Could not generate ssh keypair, error %s" %
                              result.output)
 
-  f = open(pub_key, 'r')
-  try:
-    utils.AddAuthorizedKey(auth_keys, f.read(8192))
-  finally:
-    f.close()
+  utils.AddAuthorizedKey(auth_keys, utils.ReadFile(pub_key))
 
 
-def _InitGanetiServerSetup():
-  """Setup the necessary configuration for the initial node daemon.
+def GenerateHmacKey(file_name):
+  """Writes a new HMAC key.
 
-  This creates the nodepass file containing the shared password for
-  the cluster and also generates the SSL certificate.
+  @type file_name: str
+  @param file_name: Path to output file
 
   """
-  result = utils.RunCmd(["openssl", "req", "-new", "-newkey", "rsa:1024",
-                         "-days", str(365*5), "-nodes", "-x509",
-                         "-keyout", constants.SSL_CERT_FILE,
-                         "-out", constants.SSL_CERT_FILE, "-batch"])
-  if result.failed:
-    raise errors.OpExecError("could not generate server ssl cert, command"
-                             " %s had exitcode %s and error message %s" %
-                             (result.cmd, result.exit_code, result.output))
+  utils.WriteFile(file_name, data="%s\n" % utils.GenerateSecret(), mode=0400,
+                  backup=True)
+
+
+def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_confd_hmac_key,
+                          new_cds, rapi_cert_pem=None, cds=None,
+                          nodecert_file=constants.NODED_CERT_FILE,
+                          rapicert_file=constants.RAPI_CERT_FILE,
+                          hmackey_file=constants.CONFD_HMAC_KEY,
+                          cds_file=constants.CLUSTER_DOMAIN_SECRET_FILE):
+  """Updates the cluster certificates, keys and secrets.
+
+  @type new_cluster_cert: bool
+  @param new_cluster_cert: Whether to generate a new cluster certificate
+  @type new_rapi_cert: bool
+  @param new_rapi_cert: Whether to generate a new RAPI certificate
+  @type new_confd_hmac_key: bool
+  @param new_confd_hmac_key: Whether to generate a new HMAC key
+  @type new_cds: bool
+  @param new_cds: Whether to generate a new cluster domain secret
+  @type rapi_cert_pem: string
+  @param rapi_cert_pem: New RAPI certificate in PEM format
+  @type cds: string
+  @param cds: New cluster domain secret
+  @type nodecert_file: string
+  @param nodecert_file: optional override of the node cert file path
+  @type rapicert_file: string
+  @param rapicert_file: optional override of the rapi cert file path
+  @type hmackey_file: string
+  @param hmackey_file: optional override of the hmac key file path
+
+  """
+  # noded SSL certificate
+  cluster_cert_exists = os.path.exists(nodecert_file)
+  if new_cluster_cert or not cluster_cert_exists:
+    if cluster_cert_exists:
+      utils.CreateBackup(nodecert_file)
+
+    logging.debug("Generating new cluster certificate at %s", nodecert_file)
+    utils.GenerateSelfSignedSslCert(nodecert_file)
+
+  # confd HMAC key
+  if new_confd_hmac_key or not os.path.exists(hmackey_file):
+    logging.debug("Writing new confd HMAC key to %s", hmackey_file)
+    GenerateHmacKey(hmackey_file)
+
+  # RAPI
+  rapi_cert_exists = os.path.exists(rapicert_file)
 
-  os.chmod(constants.SSL_CERT_FILE, 0400)
+  if rapi_cert_pem:
+    # Assume rapi_pem contains a valid PEM-formatted certificate and key
+    logging.debug("Writing RAPI certificate at %s", rapicert_file)
+    utils.WriteFile(rapicert_file, data=rapi_cert_pem, backup=True)
 
-  result = utils.RunCmd([constants.NODE_INITD_SCRIPT, "restart"])
+  elif new_rapi_cert or not rapi_cert_exists:
+    if rapi_cert_exists:
+      utils.CreateBackup(rapicert_file)
+
+    logging.debug("Generating new RAPI certificate at %s", rapicert_file)
+    utils.GenerateSelfSignedSslCert(rapicert_file)
+
+  # Cluster domain secret
+  if cds:
+    logging.debug("Writing cluster domain secret to %s", cds_file)
+    utils.WriteFile(cds_file, data=cds, backup=True)
+
+  elif new_cds or not os.path.exists(cds_file):
+    logging.debug("Generating new cluster domain secret at %s", cds_file)
+    GenerateHmacKey(cds_file)
+
+
+def _InitGanetiServerSetup(master_name):
+  """Setup the necessary configuration for the initial node daemon.
+
+  This creates the nodepass file containing the shared password for
+  the cluster, generates the SSL certificate and starts the node daemon.
+
+  @type master_name: str
+  @param master_name: Name of the master node
+
+  """
+  # Generate cluster secrets
+  GenerateClusterCrypto(True, False, False, False)
 
+  result = utils.RunCmd([constants.DAEMON_UTIL, "start", constants.NODED])
   if result.failed:
     raise errors.OpExecError("Could not start the node daemon, command %s"
                              " had exitcode %s and error %s" %
                              (result.cmd, result.exit_code, result.output))
 
+  _WaitForNodeDaemon(master_name)
+
+
+def _WaitForNodeDaemon(node_name):
+  """Wait for node daemon to become responsive.
+
+  """
+  def _CheckNodeDaemon():
+    result = rpc.RpcRunner.call_version([node_name])[node_name]
+    if result.fail_msg:
+      raise utils.RetryAgain()
+
+  try:
+    utils.Retry(_CheckNodeDaemon, 1.0, 10.0)
+  except utils.RetryTimeout:
+    raise errors.OpExecError("Node daemon on %s didn't answer queries within"
+                             " 10 seconds" % node_name)
+
+
+def _InitFileStorage(file_storage_dir):
+  """Initialize if needed the file storage.
+
+  @param file_storage_dir: the user-supplied value
+  @return: either empty string (if file storage was disabled at build
+      time) or the normalized path to the storage directory
+
+  """
+  if not constants.ENABLE_FILE_STORAGE:
+    return ""
+
+  file_storage_dir = os.path.normpath(file_storage_dir)
+
+  if not os.path.isabs(file_storage_dir):
+    raise errors.OpPrereqError("The file storage directory you passed is"
+                               " not an absolute path.", errors.ECODE_INVAL)
+
+  if not os.path.exists(file_storage_dir):
+    try:
+      os.makedirs(file_storage_dir, 0750)
+    except OSError, err:
+      raise errors.OpPrereqError("Cannot create file storage directory"
+                                 " '%s': %s" % (file_storage_dir, err),
+                                 errors.ECODE_ENVIRON)
+
+  if not os.path.isdir(file_storage_dir):
+    raise errors.OpPrereqError("The file storage directory '%s' is not"
+                               " a directory." % file_storage_dir,
+                               errors.ECODE_ENVIRON)
+  return file_storage_dir
+
 
-def InitCluster(cluster_name, mac_prefix, def_bridge,
+#pylint: disable-msg=R0913
+def InitCluster(cluster_name, mac_prefix,
                 master_netdev, file_storage_dir, candidate_pool_size,
-                secondary_ip=None, vg_name=None, beparams=None, hvparams=None,
-                enabled_hypervisors=None, default_hypervisor=None):
+                secondary_ip=None, vg_name=None, beparams=None,
+                nicparams=None, hvparams=None, enabled_hypervisors=None,
+                modify_etc_hosts=True, modify_ssh_setup=True,
+                maintain_node_health=False, drbd_helper=None,
+                uid_pool=None, default_iallocator=None):
   """Initialise the cluster.
 
   @type candidate_pool_size: int
@@ -104,35 +231,49 @@ def InitCluster(cluster_name, mac_prefix, def_bridge,
   """
   # TODO: complete the docstring
   if config.ConfigWriter.IsCluster():
-    raise errors.OpPrereqError("Cluster is already initialised")
+    raise errors.OpPrereqError("Cluster is already initialised",
+                               errors.ECODE_STATE)
+
+  if not enabled_hypervisors:
+    raise errors.OpPrereqError("Enabled hypervisors list must contain at"
+                               " least one member", errors.ECODE_INVAL)
+  invalid_hvs = set(enabled_hypervisors) - constants.HYPER_TYPES
+  if invalid_hvs:
+    raise errors.OpPrereqError("Enabled hypervisors contains invalid"
+                               " entries: %s" % invalid_hvs,
+                               errors.ECODE_INVAL)
 
-  hostname = utils.HostInfo()
+  hostname = netutils.GetHostInfo()
 
   if hostname.ip.startswith("127."):
     raise errors.OpPrereqError("This host's IP resolves to the private"
                                " range (%s). Please fix DNS or %s." %
-                               (hostname.ip, constants.ETC_HOSTS))
+                               (hostname.ip, constants.ETC_HOSTS),
+                               errors.ECODE_ENVIRON)
 
-  if not utils.OwnIpAddress(hostname.ip):
+  if not netutils.OwnIpAddress(hostname.ip):
     raise errors.OpPrereqError("Inconsistency: this host's name resolves"
                                " to %s,\nbut this ip address does not"
-                               " belong to this host."
-                               " Aborting." % hostname.ip)
+                               " belong to this host. Aborting." %
+                               hostname.ip, errors.ECODE_ENVIRON)
 
-  clustername = utils.HostInfo(cluster_name)
+  clustername = \
+    netutils.GetHostInfo(netutils.HostInfo.NormalizeName(cluster_name))
 
-  if utils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT,
+  if netutils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT,
                    timeout=5):
-    raise errors.OpPrereqError("Cluster IP already active. Aborting.")
+    raise errors.OpPrereqError("Cluster IP already active. Aborting.",
+                               errors.ECODE_NOTUNIQUE)
 
   if secondary_ip:
-    if not utils.IsValidIP(secondary_ip):
-      raise errors.OpPrereqError("Invalid secondary ip given")
+    if not netutils.IsValidIP4(secondary_ip):
+      raise errors.OpPrereqError("Invalid secondary ip given",
+                                 errors.ECODE_INVAL)
     if (secondary_ip != hostname.ip and
-        not utils.OwnIpAddress(secondary_ip)):
+        not netutils.OwnIpAddress(secondary_ip)):
       raise errors.OpPrereqError("You gave %s as secondary IP,"
                                  " but it does not belong to this host." %
-                                 secondary_ip)
+                                 secondary_ip, errors.ECODE_ENVIRON)
   else:
     secondary_ip = hostname.ip
 
@@ -142,55 +283,71 @@ def InitCluster(cluster_name, mac_prefix, def_bridge,
                                           constants.MIN_VG_SIZE)
     if vgstatus:
       raise errors.OpPrereqError("Error: %s\nspecify --no-lvm-storage if"
-                                 " you are not using lvm" % vgstatus)
-
-  file_storage_dir = os.path.normpath(file_storage_dir)
-
-  if not os.path.isabs(file_storage_dir):
-    raise errors.OpPrereqError("The file storage directory you passed is"
-                               " not an absolute path.")
+                                 " you are not using lvm" % vgstatus,
+                                 errors.ECODE_INVAL)
 
-  if not os.path.exists(file_storage_dir):
+  if drbd_helper is not None:
     try:
-      os.makedirs(file_storage_dir, 0750)
-    except OSError, err:
-      raise errors.OpPrereqError("Cannot create file storage directory"
-                                 " '%s': %s" %
-                                 (file_storage_dir, err))
-
-  if not os.path.isdir(file_storage_dir):
-    raise errors.OpPrereqError("The file storage directory '%s' is not"
-                               " a directory." % file_storage_dir)
+      curr_helper = bdev.BaseDRBD.GetUsermodeHelper()
+    except errors.BlockDeviceError, err:
+      raise errors.OpPrereqError("Error while checking drbd helper"
+                                 " (specify --no-drbd-storage if you are not"
+                                 " using drbd): %s" % str(err),
+                                 errors.ECODE_ENVIRON)
+    if drbd_helper != curr_helper:
+      raise errors.OpPrereqError("Error: requiring %s as drbd helper but %s"
+                                 " is the current helper" % (drbd_helper,
+                                                             curr_helper),
+                                 errors.ECODE_INVAL)
+
+  file_storage_dir = _InitFileStorage(file_storage_dir)
 
   if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
-    raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix)
+    raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix,
+                               errors.ECODE_INVAL)
 
   result = utils.RunCmd(["ip", "link", "show", "dev", master_netdev])
   if result.failed:
     raise errors.OpPrereqError("Invalid master netdev given (%s): '%s'" %
                                (master_netdev,
-                                result.output.strip()))
+                                result.output.strip()), errors.ECODE_INVAL)
+
+  dirs = [(constants.RUN_GANETI_DIR, constants.RUN_DIRS_MODE)]
+  utils.EnsureDirs(dirs)
 
-  if not (os.path.isfile(constants.NODE_INITD_SCRIPT) and
-          os.access(constants.NODE_INITD_SCRIPT, os.X_OK)):
-    raise errors.OpPrereqError("Init.d script '%s' missing or not"
-                               " executable." % constants.NODE_INITD_SCRIPT)
+  utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
+  utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
+  objects.NIC.CheckParameterSyntax(nicparams)
 
-  utils.CheckBEParams(beparams)
+  # hvparams is a mapping of hypervisor->hvparams dict
+  for hv_name, hv_params in hvparams.iteritems():
+    utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
+    hv_class = hypervisor.GetHypervisor(hv_name)
+    hv_class.CheckParameterSyntax(hv_params)
 
-  # set up the inter-node password and certificate
-  _InitGanetiServerSetup()
+  # set up the inter-node password and certificate, start noded
+  _InitGanetiServerSetup(hostname.name)
 
   # set up ssh config and /etc/hosts
-  f = open(constants.SSH_HOST_RSA_PUB, 'r')
-  try:
-    sshline = f.read()
-  finally:
-    f.close()
+  sshline = utils.ReadFile(constants.SSH_HOST_RSA_PUB)
   sshkey = sshline.split(" ")[1]
 
-  utils.AddHostToEtcHosts(hostname.name)
-  _InitSSHSetup()
+  if modify_etc_hosts:
+    utils.AddHostToEtcHosts(hostname.name)
+
+  if modify_ssh_setup:
+    _InitSSHSetup()
+
+  if default_iallocator is not None:
+    alloc_script = utils.FindFile(default_iallocator,
+                                  constants.IALLOCATOR_SEARCH_PATH,
+                                  os.path.isfile)
+    if alloc_script is None:
+      raise errors.OpPrereqError("Invalid default iallocator script '%s'"
+                                 " specified" % default_iallocator,
+                                 errors.ECODE_INVAL)
+
+  now = time.time()
 
   # init of cluster config file
   cluster_config = objects.Cluster(
@@ -199,7 +356,6 @@ def InitCluster(cluster_name, mac_prefix, def_bridge,
     highest_used_port=(constants.FIRST_DRBD_PORT - 1),
     mac_prefix=mac_prefix,
     volume_group_name=vg_name,
-    default_bridge=def_bridge,
     tcpudp_port_pool=set(),
     master_node=hostname.name,
     master_ip=clustername.ip,
@@ -207,28 +363,36 @@ def InitCluster(cluster_name, mac_prefix, def_bridge,
     cluster_name=clustername.name,
     file_storage_dir=file_storage_dir,
     enabled_hypervisors=enabled_hypervisors,
-    default_hypervisor=default_hypervisor,
-    beparams={constants.BEGR_DEFAULT: beparams},
+    beparams={constants.PP_DEFAULT: beparams},
+    nicparams={constants.PP_DEFAULT: nicparams},
     hvparams=hvparams,
     candidate_pool_size=candidate_pool_size,
+    modify_etc_hosts=modify_etc_hosts,
+    modify_ssh_setup=modify_ssh_setup,
+    uid_pool=uid_pool,
+    ctime=now,
+    mtime=now,
+    uuid=utils.NewUUID(),
+    maintain_node_health=maintain_node_health,
+    drbd_usermode_helper=drbd_helper,
+    default_iallocator=default_iallocator,
     )
   master_node_config = objects.Node(name=hostname.name,
                                     primary_ip=hostname.ip,
                                     secondary_ip=secondary_ip,
                                     serial_no=1,
                                     master_candidate=True,
-                                    offline=False,
+                                    offline=False, drained=False,
                                     )
-
-  sscfg = InitConfig(constants.CONFIG_VERSION,
-                     cluster_config, master_node_config)
-  ssh.WriteKnownHostsFile(sscfg, constants.SSH_KNOWN_HOSTS_FILE)
+  InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
   cfg = config.ConfigWriter()
-  cfg.Update(cfg.GetClusterInfo())
+  ssh.WriteKnownHostsFile(cfg, constants.SSH_KNOWN_HOSTS_FILE)
+  cfg.Update(cfg.GetClusterInfo(), logging.error)
 
   # start the master ip
   # TODO: Review rpc call from bootstrap
-  rpc.RpcRunner.call_node_start_master(hostname.name, True)
+  # TODO: Warn on failed start master
+  rpc.RpcRunner.call_node_start_master(hostname.name, True, False)
 
 
 def InitConfig(version, cluster_config, master_node_config,
@@ -247,23 +411,21 @@ def InitConfig(version, cluster_config, master_node_config,
   @type cfg_file: string
   @param cfg_file: configuration file path
 
-  @rtype: L{ssconf.SimpleConfigWriter}
-  @returns: initialized config instance
-
   """
   nodes = {
     master_node_config.name: master_node_config,
     }
 
+  now = time.time()
   config_data = objects.ConfigData(version=version,
                                    cluster=cluster_config,
                                    nodes=nodes,
                                    instances={},
-                                   serial_no=1)
-  cfg = ssconf.SimpleConfigWriter.FromDict(config_data.ToDict(), cfg_file)
-  cfg.Save()
-
-  return cfg
+                                   serial_no=1,
+                                   ctime=now, mtime=now)
+  utils.WriteFile(cfg_file,
+                  data=serializer.Dump(config_data.ToDict()),
+                  mode=0600)
 
 
 def FinalizeClusterDestroy(master):
@@ -273,12 +435,17 @@ def FinalizeClusterDestroy(master):
   begun in cmdlib.LUDestroyOpcode.
 
   """
+  cfg = config.ConfigWriter()
+  modify_ssh_setup = cfg.GetClusterInfo().modify_ssh_setup
   result = rpc.RpcRunner.call_node_stop_master(master, True)
-  if result.failed or not result.data:
-    logging.warning("Could not disable the master role")
-  result = rpc.RpcRunner.call_node_leave_cluster(master)
-  if result.failed or not result.data:
-    logging.warning("Could not shutdown the node daemon and cleanup the node")
+  msg = result.fail_msg
+  if msg:
+    logging.warning("Could not disable the master role: %s", msg)
+  result = rpc.RpcRunner.call_node_leave_cluster(master, modify_ssh_setup)
+  msg = result.fail_msg
+  if msg:
+    logging.warning("Could not shutdown the node daemon and cleanup"
+                    " the node: %s", msg)
 
 
 def SetupNodeDaemon(cluster_name, node, ssh_key_check):
@@ -294,25 +461,47 @@ def SetupNodeDaemon(cluster_name, node, ssh_key_check):
 
   """
   sshrunner = ssh.SshRunner(cluster_name)
-  gntpem = utils.ReadFile(constants.SSL_CERT_FILE)
+
+  noded_cert = utils.ReadFile(constants.NODED_CERT_FILE)
+  rapi_cert = utils.ReadFile(constants.RAPI_CERT_FILE)
+  confd_hmac_key = utils.ReadFile(constants.CONFD_HMAC_KEY)
+
   # in the base64 pem encoding, neither '!' nor '.' are valid chars,
   # so we use this to detect an invalid certificate; as long as the
   # cert doesn't contain this, the here-document will be correctly
-  # parsed by the shell sequence below
-  if re.search('^!EOF\.', gntpem, re.MULTILINE):
-    raise errors.OpExecError("invalid PEM encoding in the SSL certificate")
-  if not gntpem.endswith("\n"):
-    raise errors.OpExecError("PEM must end with newline")
+  # parsed by the shell sequence below. HMAC keys are hexadecimal strings,
+  # so the same restrictions apply.
+  for content in (noded_cert, rapi_cert, confd_hmac_key):
+    if re.search('^!EOF\.', content, re.MULTILINE):
+      raise errors.OpExecError("invalid SSL certificate or HMAC key")
+
+  if not noded_cert.endswith("\n"):
+    noded_cert += "\n"
+  if not rapi_cert.endswith("\n"):
+    rapi_cert += "\n"
+  if not confd_hmac_key.endswith("\n"):
+    confd_hmac_key += "\n"
 
   # set up inter-node password and certificate and restarts the node daemon
   # and then connect with ssh to set password and start ganeti-noded
   # note that all the below variables are sanitized at this point,
   # either by being constants or by the checks above
+  # TODO: Could this command exceed a shell's maximum command length?
   mycommand = ("umask 077 && "
                "cat > '%s' << '!EOF.' && \n"
-               "%s!EOF.\n%s restart" %
-               (constants.SSL_CERT_FILE, gntpem,
-                constants.NODE_INITD_SCRIPT))
+               "%s!EOF.\n"
+               "cat > '%s' << '!EOF.' && \n"
+               "%s!EOF.\n"
+               "cat > '%s' << '!EOF.' && \n"
+               "%s!EOF.\n"
+               "chmod 0400 %s %s %s && "
+               "%s start %s" %
+               (constants.NODED_CERT_FILE, noded_cert,
+                constants.RAPI_CERT_FILE, rapi_cert,
+                constants.CONFD_HMAC_KEY, confd_hmac_key,
+                constants.NODED_CERT_FILE, constants.RAPI_CERT_FILE,
+                constants.CONFD_HMAC_KEY,
+                constants.DAEMON_UTIL, constants.NODED))
 
   result = sshrunner.Run(node, 'root', mycommand, batch=False,
                          ask_key=ssh_key_check,
@@ -323,14 +512,20 @@ def SetupNodeDaemon(cluster_name, node, ssh_key_check):
                              " output: %s" %
                              (node, result.fail_reason, result.output))
 
+  _WaitForNodeDaemon(node)
 
-def MasterFailover():
+
+def MasterFailover(no_voting=False):
   """Failover the master node.
 
   This checks that we are not already the master, and will cause the
   current master to cease being master, and the non-master to become
   new master.
 
+  @type no_voting: boolean
+  @param no_voting: force the operation without remote nodes agreement
+                      (dangerous)
+
   """
   sstore = ssconf.SimpleStore()
 
@@ -342,7 +537,7 @@ def MasterFailover():
     raise errors.OpPrereqError("This commands must be run on the node"
                                " where you want the new master to be."
                                " %s is already the master" %
-                               old_master)
+                               old_master, errors.ECODE_INVAL)
 
   if new_master not in mc_list:
     mc_no_master = [name for name in mc_list if name != old_master]
@@ -350,20 +545,24 @@ def MasterFailover():
                                " as master candidates. Only these nodes"
                                " can become masters. Current list of"
                                " master candidates is:\n"
-                               "%s" % ('\n'.join(mc_no_master)))
-
-  vote_list = GatherMasterVotes(node_list)
-
-  if vote_list:
-    voted_master = vote_list[0][0]
-    if voted_master is None:
-      raise errors.OpPrereqError("Cluster is inconsistent, most nodes did not"
-                                 " respond.")
-    elif voted_master != old_master:
-      raise errors.OpPrereqError("I have wrong configuration, I believe the"
-                                 " master is %s but the other nodes voted for"
-                                 " %s. Please resync the configuration of"
-                                 " this node." % (old_master, voted_master))
+                               "%s" % ('\n'.join(mc_no_master)),
+                               errors.ECODE_STATE)
+
+  if not no_voting:
+    vote_list = GatherMasterVotes(node_list)
+
+    if vote_list:
+      voted_master = vote_list[0][0]
+      if voted_master is None:
+        raise errors.OpPrereqError("Cluster is inconsistent, most nodes did"
+                                   " not respond.", errors.ECODE_ENVIRON)
+      elif voted_master != old_master:
+        raise errors.OpPrereqError("I have a wrong configuration, I believe"
+                                   " the master is %s but the other nodes"
+                                   " voted %s. Please resync the configuration"
+                                   " of this node." %
+                                   (old_master, voted_master),
+                                   errors.ECODE_STATE)
   # end checks
 
   rcode = 0
@@ -371,11 +570,24 @@ def MasterFailover():
   logging.info("Setting master to %s, old master: %s", new_master, old_master)
 
   result = rpc.RpcRunner.call_node_stop_master(old_master, True)
-  if result.failed or not result.data:
+  msg = result.fail_msg
+  if msg:
     logging.error("Could not disable the master role on the old master"
-                 " %s, please disable manually", old_master)
+                 " %s, please disable manually: %s", old_master, msg)
 
+  master_ip = sstore.GetMasterIP()
+  total_timeout = 30
   # Here we have a phase where no master should be running
+  def _check_ip():
+    if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):
+      raise utils.RetryAgain()
+
+  try:
+    utils.Retry(_check_ip, (1, 1.5, 5), total_timeout)
+  except utils.RetryTimeout:
+    logging.warning("The master IP is still reachable after %s seconds,"
+                    " continuing but activating the master on the current"
+                    " node will probably fail", total_timeout)
 
   # instantiate a real config writer, as we now know we have the
   # configuration data
@@ -385,17 +597,34 @@ def MasterFailover():
   cluster_info.master_node = new_master
   # this will also regenerate the ssconf files, since we updated the
   # cluster info
-  cfg.Update(cluster_info)
+  cfg.Update(cluster_info, logging.error)
 
-  result = rpc.RpcRunner.call_node_start_master(new_master, True)
-  if result.failed or not result.data:
+  result = rpc.RpcRunner.call_node_start_master(new_master, True, no_voting)
+  msg = result.fail_msg
+  if msg:
     logging.error("Could not start the master role on the new master"
-                  " %s, please check", new_master)
+                  " %s, please check: %s", new_master, msg)
     rcode = 1
 
   return rcode
 
 
+def GetMaster():
+  """Returns the current master node.
+
+  This is a separate function in bootstrap since it's needed by
+  gnt-cluster, and instead of importing directly ssconf, it's better
+  to abstract it in bootstrap, where we do use ssconf in other
+  functions too.
+
+  """
+  sstore = ssconf.SimpleStore()
+
+  old_master, _ = ssconf.GetMasterAndMyself(sstore)
+
+  return old_master
+
+
 def GatherMasterVotes(node_list):
   """Check the agreement on who is the master.
 
@@ -412,12 +641,12 @@ def GatherMasterVotes(node_list):
 
   @type node_list: list
   @param node_list: the list of nodes to query for master info; the current
-      node wil be removed if it is in the list
+      node will be removed if it is in the list
   @rtype: list
   @return: list of (node, votes)
 
   """
-  myself = utils.HostInfo().name
+  myself = netutils.HostInfo().name
   try:
     node_list.remove(myself)
   except ValueError:
@@ -433,9 +662,16 @@ def GatherMasterVotes(node_list):
   votes = {}
   for node in results:
     nres = results[node]
-    data = nres.data
-    if nres.failed or not isinstance(data, (tuple, list)) or len(data) < 3:
-      # here the rpc layer should have already logged errors
+    data = nres.payload
+    msg = nres.fail_msg
+    fail = False
+    if msg:
+      logging.warning("Error contacting node %s: %s", node, msg)
+      fail = True
+    elif not isinstance(data, (tuple, list)) or len(data) < 3:
+      logging.warning("Invalid data received from node %s: %s", node, data)
+      fail = True
+    if fail:
       if None not in votes:
         votes[None] = 0
       votes[None] += 1