Migration: warn the user about hv version mismatch

[ganeti-local] / lib / bootstrap.py

diff --git a/lib/bootstrap.py b/lib/bootstrap.py
index 24f9a0a..85c2251 100644 (file)
--- a/lib/bootstrap.py
+++ b/lib/bootstrap.py
@@ -1,7 +1,7 @@
 #
 #
 
 #
 #
 

-# Copyright (C) 2006, 2007, 2008, 2010 Google Inc.
+# Copyright (C) 2006, 2007, 2008, 2010, 2011 Google Inc.

 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

@@ -42,10 +42,16 @@ from ganeti import hypervisor
 from ganeti import bdev
 from ganeti import netutils
 from ganeti import backend
 from ganeti import bdev
 from ganeti import netutils
 from ganeti import backend

+from ganeti import luxi
+from ganeti import jstore
+

 
 # ec_id for InitConfig's temporary reservation manager
 _INITCONF_ECID = "initconfig-ecid"
 
 
 # ec_id for InitConfig's temporary reservation manager
 _INITCONF_ECID = "initconfig-ecid"
 

+#: After how many seconds daemon must be responsive
+_DAEMON_READY_TIMEOUT = 10.0
+

 
 def _InitSSHSetup():
   """Setup the SSH configuration for the cluster.
 
 def _InitSSHSetup():
   """Setup the SSH configuration for the cluster.

@@ -82,10 +88,14 @@ def GenerateHmacKey(file_name):
                   backup=True)
 
 
                   backup=True)
 
 

-def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_confd_hmac_key,
-                          new_cds, rapi_cert_pem=None, cds=None,
+def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_spice_cert,
+                          new_confd_hmac_key, new_cds,
+                          rapi_cert_pem=None, spice_cert_pem=None,
+                          spice_cacert_pem=None, cds=None,

                           nodecert_file=constants.NODED_CERT_FILE,
                           rapicert_file=constants.RAPI_CERT_FILE,
                           nodecert_file=constants.NODED_CERT_FILE,
                           rapicert_file=constants.RAPI_CERT_FILE,

+                          spicecert_file=constants.SPICE_CERT_FILE,
+                          spicecacert_file=constants.SPICE_CACERT_FILE,

                           hmackey_file=constants.CONFD_HMAC_KEY,
                           cds_file=constants.CLUSTER_DOMAIN_SECRET_FILE):
   """Updates the cluster certificates, keys and secrets.
                           hmackey_file=constants.CONFD_HMAC_KEY,
                           cds_file=constants.CLUSTER_DOMAIN_SECRET_FILE):
   """Updates the cluster certificates, keys and secrets.

@@ -94,18 +104,29 @@ def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_confd_hmac_key,
   @param new_cluster_cert: Whether to generate a new cluster certificate
   @type new_rapi_cert: bool
   @param new_rapi_cert: Whether to generate a new RAPI certificate
   @param new_cluster_cert: Whether to generate a new cluster certificate
   @type new_rapi_cert: bool
   @param new_rapi_cert: Whether to generate a new RAPI certificate

+  @type new_spice_cert: bool
+  @param new_spice_cert: Whether to generate a new SPICE certificate

   @type new_confd_hmac_key: bool
   @param new_confd_hmac_key: Whether to generate a new HMAC key
   @type new_cds: bool
   @param new_cds: Whether to generate a new cluster domain secret
   @type rapi_cert_pem: string
   @param rapi_cert_pem: New RAPI certificate in PEM format
   @type new_confd_hmac_key: bool
   @param new_confd_hmac_key: Whether to generate a new HMAC key
   @type new_cds: bool
   @param new_cds: Whether to generate a new cluster domain secret
   @type rapi_cert_pem: string
   @param rapi_cert_pem: New RAPI certificate in PEM format

+  @type spice_cert_pem: string
+  @param spice_cert_pem: New SPICE certificate in PEM format
+  @type spice_cacert_pem: string
+  @param spice_cacert_pem: Certificate of the CA that signed the SPICE
+                           certificate, in PEM format

   @type cds: string
   @param cds: New cluster domain secret
   @type nodecert_file: string
   @param nodecert_file: optional override of the node cert file path
   @type rapicert_file: string
   @param rapicert_file: optional override of the rapi cert file path
   @type cds: string
   @param cds: New cluster domain secret
   @type nodecert_file: string
   @param nodecert_file: optional override of the node cert file path
   @type rapicert_file: string
   @param rapicert_file: optional override of the rapi cert file path

+  @type spicecert_file: string
+  @param spicecert_file: optional override of the spice cert file path
+  @type spicecacert_file: string
+  @param spicecacert_file: optional override of the spice CA cert file path

   @type hmackey_file: string
   @param hmackey_file: optional override of the hmac key file path
 
   @type hmackey_file: string
   @param hmackey_file: optional override of the hmac key file path
 

@@ -139,6 +160,31 @@ def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_confd_hmac_key,
     logging.debug("Generating new RAPI certificate at %s", rapicert_file)
     utils.GenerateSelfSignedSslCert(rapicert_file)
 
     logging.debug("Generating new RAPI certificate at %s", rapicert_file)
     utils.GenerateSelfSignedSslCert(rapicert_file)
 

+  # SPICE
+  spice_cert_exists = os.path.exists(spicecert_file)
+  spice_cacert_exists = os.path.exists(spicecacert_file)
+  if spice_cert_pem:
+    # spice_cert_pem implies also spice_cacert_pem
+    logging.debug("Writing SPICE certificate at %s", spicecert_file)
+    utils.WriteFile(spicecert_file, data=spice_cert_pem, backup=True)
+    logging.debug("Writing SPICE CA certificate at %s", spicecacert_file)
+    utils.WriteFile(spicecacert_file, data=spice_cacert_pem, backup=True)
+  elif new_spice_cert or not spice_cert_exists:
+    if spice_cert_exists:
+      utils.CreateBackup(spicecert_file)
+    if spice_cacert_exists:
+      utils.CreateBackup(spicecacert_file)
+
+    logging.debug("Generating new self-signed SPICE certificate at %s",
+                  spicecert_file)
+    (_, cert_pem) = utils.GenerateSelfSignedSslCert(spicecert_file)
+
+    # Self-signed certificate -> the public certificate is also the CA public
+    # certificate
+    logging.debug("Writing the public certificate to %s",
+                  spicecert_file)
+    utils.io.WriteFile(spicecacert_file, mode=0400, data=cert_pem)
+

   # Cluster domain secret
   if cds:
     logging.debug("Writing cluster domain secret to %s", cds_file)
   # Cluster domain secret
   if cds:
     logging.debug("Writing cluster domain secret to %s", cds_file)

@@ -160,7 +206,7 @@ def _InitGanetiServerSetup(master_name):
 
   """
   # Generate cluster secrets
 
   """
   # Generate cluster secrets

-  GenerateClusterCrypto(True, False, False, False)
+  GenerateClusterCrypto(True, False, False, False, False)

 
   result = utils.RunCmd([constants.DAEMON_UTIL, "start", constants.NODED])
   if result.failed:
 
   result = utils.RunCmd([constants.DAEMON_UTIL, "start", constants.NODED])
   if result.failed:

@@ -181,10 +227,30 @@ def _WaitForNodeDaemon(node_name):
       raise utils.RetryAgain()
 
   try:
       raise utils.RetryAgain()
 
   try:

-    utils.Retry(_CheckNodeDaemon, 1.0, 10.0)
+    utils.Retry(_CheckNodeDaemon, 1.0, _DAEMON_READY_TIMEOUT)

   except utils.RetryTimeout:
     raise errors.OpExecError("Node daemon on %s didn't answer queries within"
   except utils.RetryTimeout:
     raise errors.OpExecError("Node daemon on %s didn't answer queries within"

-                             " 10 seconds" % node_name)
+                             " %s seconds" % (node_name, _DAEMON_READY_TIMEOUT))
+
+
+def _WaitForMasterDaemon():
+  """Wait for master daemon to become responsive.
+
+  """
+  def _CheckMasterDaemon():
+    try:
+      cl = luxi.Client()
+      (cluster_name, ) = cl.QueryConfigValues(["cluster_name"])
+    except Exception:
+      raise utils.RetryAgain()
+
+    logging.debug("Received cluster name %s from master", cluster_name)
+
+  try:
+    utils.Retry(_CheckMasterDaemon, 1.0, _DAEMON_READY_TIMEOUT)
+  except utils.RetryTimeout:
+    raise errors.OpExecError("Master daemon didn't answer queries within"
+                             " %s seconds" % _DAEMON_READY_TIMEOUT)

 
 
 def _InitFileStorage(file_storage_dir):
 
 
 def _InitFileStorage(file_storage_dir):

@@ -195,14 +261,11 @@ def _InitFileStorage(file_storage_dir):
       time) or the normalized path to the storage directory
 
   """
       time) or the normalized path to the storage directory
 
   """

-  if not constants.ENABLE_FILE_STORAGE:
-    return ""
-

   file_storage_dir = os.path.normpath(file_storage_dir)
 
   if not os.path.isabs(file_storage_dir):
   file_storage_dir = os.path.normpath(file_storage_dir)
 
   if not os.path.isabs(file_storage_dir):

-    raise errors.OpPrereqError("The file storage directory you passed is"
-                               " not an absolute path.", errors.ECODE_INVAL)
+    raise errors.OpPrereqError("File storage directory '%s' is not an absolute"
+                               " path" % file_storage_dir, errors.ECODE_INVAL)

 
   if not os.path.exists(file_storage_dir):
     try:
 
   if not os.path.exists(file_storage_dir):
     try:

@@ -219,14 +282,13 @@ def _InitFileStorage(file_storage_dir):
   return file_storage_dir
 
 
   return file_storage_dir
 
 

-#pylint: disable-msg=R0913
-def InitCluster(cluster_name, mac_prefix,
-                master_netdev, file_storage_dir, candidate_pool_size,
-                secondary_ip=None, vg_name=None, beparams=None,
-                nicparams=None, hvparams=None, enabled_hypervisors=None,
-                modify_etc_hosts=True, modify_ssh_setup=True,
-                maintain_node_health=False, drbd_helper=None,
-                uid_pool=None, default_iallocator=None,
+def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913
+                master_netdev, file_storage_dir, shared_file_storage_dir,
+                candidate_pool_size, secondary_ip=None, vg_name=None,
+                beparams=None, nicparams=None, ndparams=None, hvparams=None,
+                enabled_hypervisors=None, modify_etc_hosts=True,
+                modify_ssh_setup=True, maintain_node_health=False,
+                drbd_helper=None, uid_pool=None, default_iallocator=None,

                 primary_ip_version=None, prealloc_wipe_disks=False):
   """Initialise the cluster.
 
                 primary_ip_version=None, prealloc_wipe_disks=False):
   """Initialise the cluster.
 

@@ -248,7 +310,6 @@ def InitCluster(cluster_name, mac_prefix,
                                " entries: %s" % invalid_hvs,
                                errors.ECODE_INVAL)
 
                                " entries: %s" % invalid_hvs,
                                errors.ECODE_INVAL)
 

-

   ipcls = None
   if primary_ip_version == constants.IP4_VERSION:
     ipcls = netutils.IP4Address
   ipcls = None
   if primary_ip_version == constants.IP4_VERSION:
     ipcls = netutils.IP4Address

@@ -321,7 +382,15 @@ def InitCluster(cluster_name, mac_prefix,
                                                              curr_helper),
                                  errors.ECODE_INVAL)
 
                                                              curr_helper),
                                  errors.ECODE_INVAL)
 

-  file_storage_dir = _InitFileStorage(file_storage_dir)
+  if constants.ENABLE_FILE_STORAGE:
+    file_storage_dir = _InitFileStorage(file_storage_dir)
+  else:
+    file_storage_dir = ""
+
+  if constants.ENABLE_SHARED_FILE_STORAGE:
+    shared_file_storage_dir = _InitFileStorage(shared_file_storage_dir)
+  else:
+    shared_file_storage_dir = ""

 
   if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
     raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix,
 
   if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
     raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix,

@@ -340,6 +409,11 @@ def InitCluster(cluster_name, mac_prefix,
   utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
   objects.NIC.CheckParameterSyntax(nicparams)
 
   utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
   objects.NIC.CheckParameterSyntax(nicparams)
 

+  if ndparams is not None:
+    utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
+  else:
+    ndparams = dict(constants.NDC_DEFAULTS)
+

   # hvparams is a mapping of hypervisor->hvparams dict
   for hv_name, hv_params in hvparams.iteritems():
     utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
   # hvparams is a mapping of hypervisor->hvparams dict
   for hv_name, hv_params in hvparams.iteritems():
     utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)

@@ -364,6 +438,12 @@ def InitCluster(cluster_name, mac_prefix,
       raise errors.OpPrereqError("Invalid default iallocator script '%s'"
                                  " specified" % default_iallocator,
                                  errors.ECODE_INVAL)
       raise errors.OpPrereqError("Invalid default iallocator script '%s'"
                                  " specified" % default_iallocator,
                                  errors.ECODE_INVAL)

+  elif constants.HTOOLS:
+    # htools was enabled at build-time, we default to it
+    if utils.FindFile(constants.IALLOC_HAIL,
+                      constants.IALLOCATOR_SEARCH_PATH,
+                      os.path.isfile):
+      default_iallocator = constants.IALLOC_HAIL

 
   now = time.time()
 
 
   now = time.time()
 

@@ -380,9 +460,11 @@ def InitCluster(cluster_name, mac_prefix,
     master_netdev=master_netdev,
     cluster_name=clustername.name,
     file_storage_dir=file_storage_dir,
     master_netdev=master_netdev,
     cluster_name=clustername.name,
     file_storage_dir=file_storage_dir,

+    shared_file_storage_dir=shared_file_storage_dir,

     enabled_hypervisors=enabled_hypervisors,
     beparams={constants.PP_DEFAULT: beparams},
     nicparams={constants.PP_DEFAULT: nicparams},
     enabled_hypervisors=enabled_hypervisors,
     beparams={constants.PP_DEFAULT: beparams},
     nicparams={constants.PP_DEFAULT: nicparams},

+    ndparams=ndparams,

     hvparams=hvparams,
     candidate_pool_size=candidate_pool_size,
     modify_etc_hosts=modify_etc_hosts,
     hvparams=hvparams,
     candidate_pool_size=candidate_pool_size,
     modify_etc_hosts=modify_etc_hosts,

@@ -402,6 +484,7 @@ def InitCluster(cluster_name, mac_prefix,
                                     serial_no=1,
                                     master_candidate=True,
                                     offline=False, drained=False,
                                     serial_no=1,
                                     master_candidate=True,
                                     offline=False, drained=False,

+                                    ctime=now, mtime=now,

                                     )
   InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
   cfg = config.ConfigWriter(offline=True)
                                     )
   InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
   cfg = config.ConfigWriter(offline=True)

@@ -412,10 +495,14 @@ def InitCluster(cluster_name, mac_prefix,
   # set up the inter-node password and certificate
   _InitGanetiServerSetup(hostname.name)
 
   # set up the inter-node password and certificate
   _InitGanetiServerSetup(hostname.name)
 

-  # start the master ip
-  # TODO: Review rpc call from bootstrap
-  # TODO: Warn on failed start master
-  rpc.RpcRunner.call_node_start_master(hostname.name, True, False)
+  logging.debug("Starting daemons")
+  result = utils.RunCmd([constants.DAEMON_UTIL, "start-all"])
+  if result.failed:
+    raise errors.OpExecError("Could not start daemons, command %s"
+                             " had exitcode %s and error %s" %
+                             (result.cmd, result.exit_code, result.output))
+
+  _WaitForMasterDaemon()

 
 
 def InitConfig(version, cluster_config, master_node_config,
 
 
 def InitConfig(version, cluster_config, master_node_config,

@@ -445,7 +532,7 @@ def InitConfig(version, cluster_config, master_node_config,
     }
   default_nodegroup = objects.NodeGroup(
     uuid=uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID),
     }
   default_nodegroup = objects.NodeGroup(
     uuid=uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID),

-    name="default",
+    name=constants.INITIAL_NODE_GROUP_NAME,

     members=[master_node_config.name],
     )
   nodegroups = {
     members=[master_node_config.name],
     )
   nodegroups = {

@@ -498,27 +585,7 @@ def SetupNodeDaemon(cluster_name, node, ssh_key_check):
   """
   family = ssconf.SimpleStore().GetPrimaryIPFamily()
   sshrunner = ssh.SshRunner(cluster_name,
   """
   family = ssconf.SimpleStore().GetPrimaryIPFamily()
   sshrunner = ssh.SshRunner(cluster_name,

-                            ipv6=family==netutils.IP6Address.family)
-
-  noded_cert = utils.ReadFile(constants.NODED_CERT_FILE)
-  rapi_cert = utils.ReadFile(constants.RAPI_CERT_FILE)
-  confd_hmac_key = utils.ReadFile(constants.CONFD_HMAC_KEY)
-
-  # in the base64 pem encoding, neither '!' nor '.' are valid chars,
-  # so we use this to detect an invalid certificate; as long as the
-  # cert doesn't contain this, the here-document will be correctly
-  # parsed by the shell sequence below. HMAC keys are hexadecimal strings,
-  # so the same restrictions apply.
-  for content in (noded_cert, rapi_cert, confd_hmac_key):
-    if re.search('^!EOF\.', content, re.MULTILINE):
-      raise errors.OpExecError("invalid SSL certificate or HMAC key")
-
-  if not noded_cert.endswith("\n"):
-    noded_cert += "\n"
-  if not rapi_cert.endswith("\n"):
-    rapi_cert += "\n"
-  if not confd_hmac_key.endswith("\n"):
-    confd_hmac_key += "\n"
+                            ipv6=(family == netutils.IP6Address.family))

 
   bind_address = constants.IP4_ADDRESS_ANY
   if family == netutils.IP6Address.family:
 
   bind_address = constants.IP4_ADDRESS_ANY
   if family == netutils.IP6Address.family:

@@ -530,11 +597,12 @@ def SetupNodeDaemon(cluster_name, node, ssh_key_check):
   # either by being constants or by the checks above
   sshrunner.CopyFileToNode(node, constants.NODED_CERT_FILE)
   sshrunner.CopyFileToNode(node, constants.RAPI_CERT_FILE)
   # either by being constants or by the checks above
   sshrunner.CopyFileToNode(node, constants.NODED_CERT_FILE)
   sshrunner.CopyFileToNode(node, constants.RAPI_CERT_FILE)

+  sshrunner.CopyFileToNode(node, constants.SPICE_CERT_FILE)
+  sshrunner.CopyFileToNode(node, constants.SPICE_CACERT_FILE)

   sshrunner.CopyFileToNode(node, constants.CONFD_HMAC_KEY)
   sshrunner.CopyFileToNode(node, constants.CONFD_HMAC_KEY)

-  mycommand = ("%s stop-all; %s start %s -b '%s'" % (constants.DAEMON_UTIL,
-                                                     constants.DAEMON_UTIL,
-                                                     constants.NODED,
-                                                     bind_address))
+  mycommand = ("%s stop-all; %s start %s -b %s" %
+               (constants.DAEMON_UTIL, constants.DAEMON_UTIL, constants.NODED,
+                utils.ShellQuote(bind_address)))

 
   result = sshrunner.Run(node, 'root', mycommand, batch=False,
                          ask_key=ssh_key_check,
 
   result = sshrunner.Run(node, 'root', mycommand, batch=False,
                          ask_key=ssh_key_check,

@@ -634,6 +702,7 @@ def MasterFailover(no_voting=False):
 
   master_ip = sstore.GetMasterIP()
   total_timeout = 30
 
   master_ip = sstore.GetMasterIP()
   total_timeout = 30

+

   # Here we have a phase where no master should be running
   def _check_ip():
     if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):
   # Here we have a phase where no master should be running
   def _check_ip():
     if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):

@@ -646,6 +715,10 @@ def MasterFailover(no_voting=False):
                     " continuing but activating the master on the current"
                     " node will probably fail", total_timeout)
 
                     " continuing but activating the master on the current"
                     " node will probably fail", total_timeout)
 

+  if jstore.CheckDrainFlag():
+    logging.info("Undraining job queue")
+    jstore.SetDrainFlag(False)
+

   logging.info("Starting the master daemons on the new master")
 
   result = rpc.RpcRunner.call_node_start_master(new_master, True, no_voting)
   logging.info("Starting the master daemons on the new master")
 
   result = rpc.RpcRunner.call_node_start_master(new_master, True, no_voting)