4 # Copyright (C) 2006, 2007, 2008, 2010, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Functions to bootstrap a new cluster.
33 from ganeti.cmdlib import cluster
34 from ganeti import rpc
35 from ganeti import ssh
36 from ganeti import utils
37 from ganeti import errors
38 from ganeti import config
39 from ganeti import constants
40 from ganeti import objects
41 from ganeti import ssconf
42 from ganeti import serializer
43 from ganeti import hypervisor
44 from ganeti.storage import drbd
45 from ganeti.storage import filestorage
46 from ganeti import netutils
47 from ganeti import luxi
48 from ganeti import jstore
49 from ganeti import pathutils
52 # ec_id for InitConfig's temporary reservation manager
53 _INITCONF_ECID = "initconfig-ecid"
55 #: After how many seconds daemon must be responsive
56 _DAEMON_READY_TIMEOUT = 10.0
60 """Setup the SSH configuration for the cluster.
62 This generates a dsa keypair for root, adds the pub key to the
63 permitted hosts and adds the hostkey to its own known hosts.
66 priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.SSH_LOGIN_USER)
68 for name in priv_key, pub_key:
69 if os.path.exists(name):
70 utils.CreateBackup(name)
71 utils.RemoveFile(name)
73 result = utils.RunCmd(["ssh-keygen", "-t", "dsa",
77 raise errors.OpExecError("Could not generate ssh keypair, error %s" %
80 utils.AddAuthorizedKey(auth_keys, utils.ReadFile(pub_key))
83 def GenerateHmacKey(file_name):
84 """Writes a new HMAC key.
87 @param file_name: Path to output file
90 utils.WriteFile(file_name, data="%s\n" % utils.GenerateSecret(), mode=0400,
94 def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_spice_cert,
95 new_confd_hmac_key, new_cds,
96 rapi_cert_pem=None, spice_cert_pem=None,
97 spice_cacert_pem=None, cds=None,
98 nodecert_file=pathutils.NODED_CERT_FILE,
99 rapicert_file=pathutils.RAPI_CERT_FILE,
100 spicecert_file=pathutils.SPICE_CERT_FILE,
101 spicecacert_file=pathutils.SPICE_CACERT_FILE,
102 hmackey_file=pathutils.CONFD_HMAC_KEY,
103 cds_file=pathutils.CLUSTER_DOMAIN_SECRET_FILE):
104 """Updates the cluster certificates, keys and secrets.
106 @type new_cluster_cert: bool
107 @param new_cluster_cert: Whether to generate a new cluster certificate
108 @type new_rapi_cert: bool
109 @param new_rapi_cert: Whether to generate a new RAPI certificate
110 @type new_spice_cert: bool
111 @param new_spice_cert: Whether to generate a new SPICE certificate
112 @type new_confd_hmac_key: bool
113 @param new_confd_hmac_key: Whether to generate a new HMAC key
115 @param new_cds: Whether to generate a new cluster domain secret
116 @type rapi_cert_pem: string
117 @param rapi_cert_pem: New RAPI certificate in PEM format
118 @type spice_cert_pem: string
119 @param spice_cert_pem: New SPICE certificate in PEM format
120 @type spice_cacert_pem: string
121 @param spice_cacert_pem: Certificate of the CA that signed the SPICE
122 certificate, in PEM format
124 @param cds: New cluster domain secret
125 @type nodecert_file: string
126 @param nodecert_file: optional override of the node cert file path
127 @type rapicert_file: string
128 @param rapicert_file: optional override of the rapi cert file path
129 @type spicecert_file: string
130 @param spicecert_file: optional override of the spice cert file path
131 @type spicecacert_file: string
132 @param spicecacert_file: optional override of the spice CA cert file path
133 @type hmackey_file: string
134 @param hmackey_file: optional override of the hmac key file path
137 # noded SSL certificate
138 cluster_cert_exists = os.path.exists(nodecert_file)
139 if new_cluster_cert or not cluster_cert_exists:
140 if cluster_cert_exists:
141 utils.CreateBackup(nodecert_file)
143 logging.debug("Generating new cluster certificate at %s", nodecert_file)
144 utils.GenerateSelfSignedSslCert(nodecert_file)
147 if new_confd_hmac_key or not os.path.exists(hmackey_file):
148 logging.debug("Writing new confd HMAC key to %s", hmackey_file)
149 GenerateHmacKey(hmackey_file)
152 rapi_cert_exists = os.path.exists(rapicert_file)
155 # Assume rapi_pem contains a valid PEM-formatted certificate and key
156 logging.debug("Writing RAPI certificate at %s", rapicert_file)
157 utils.WriteFile(rapicert_file, data=rapi_cert_pem, backup=True)
159 elif new_rapi_cert or not rapi_cert_exists:
161 utils.CreateBackup(rapicert_file)
163 logging.debug("Generating new RAPI certificate at %s", rapicert_file)
164 utils.GenerateSelfSignedSslCert(rapicert_file)
167 spice_cert_exists = os.path.exists(spicecert_file)
168 spice_cacert_exists = os.path.exists(spicecacert_file)
170 # spice_cert_pem implies also spice_cacert_pem
171 logging.debug("Writing SPICE certificate at %s", spicecert_file)
172 utils.WriteFile(spicecert_file, data=spice_cert_pem, backup=True)
173 logging.debug("Writing SPICE CA certificate at %s", spicecacert_file)
174 utils.WriteFile(spicecacert_file, data=spice_cacert_pem, backup=True)
175 elif new_spice_cert or not spice_cert_exists:
176 if spice_cert_exists:
177 utils.CreateBackup(spicecert_file)
178 if spice_cacert_exists:
179 utils.CreateBackup(spicecacert_file)
181 logging.debug("Generating new self-signed SPICE certificate at %s",
183 (_, cert_pem) = utils.GenerateSelfSignedSslCert(spicecert_file)
185 # Self-signed certificate -> the public certificate is also the CA public
187 logging.debug("Writing the public certificate to %s",
189 utils.io.WriteFile(spicecacert_file, mode=0400, data=cert_pem)
191 # Cluster domain secret
193 logging.debug("Writing cluster domain secret to %s", cds_file)
194 utils.WriteFile(cds_file, data=cds, backup=True)
196 elif new_cds or not os.path.exists(cds_file):
197 logging.debug("Generating new cluster domain secret at %s", cds_file)
198 GenerateHmacKey(cds_file)
201 def _InitGanetiServerSetup(master_name):
202 """Setup the necessary configuration for the initial node daemon.
204 This creates the nodepass file containing the shared password for
205 the cluster, generates the SSL certificate and starts the node daemon.
207 @type master_name: str
208 @param master_name: Name of the master node
211 # Generate cluster secrets
212 GenerateClusterCrypto(True, False, False, False, False)
214 result = utils.RunCmd([pathutils.DAEMON_UTIL, "start", constants.NODED])
216 raise errors.OpExecError("Could not start the node daemon, command %s"
217 " had exitcode %s and error %s" %
218 (result.cmd, result.exit_code, result.output))
220 _WaitForNodeDaemon(master_name)
223 def _WaitForNodeDaemon(node_name):
224 """Wait for node daemon to become responsive.
227 def _CheckNodeDaemon():
228 # Pylint bug <http://www.logilab.org/ticket/35642>
229 # pylint: disable=E1101
230 result = rpc.BootstrapRunner().call_version([node_name])[node_name]
232 raise utils.RetryAgain()
235 utils.Retry(_CheckNodeDaemon, 1.0, _DAEMON_READY_TIMEOUT)
236 except utils.RetryTimeout:
237 raise errors.OpExecError("Node daemon on %s didn't answer queries within"
238 " %s seconds" % (node_name, _DAEMON_READY_TIMEOUT))
241 def _WaitForMasterDaemon():
242 """Wait for master daemon to become responsive.
245 def _CheckMasterDaemon():
248 (cluster_name, ) = cl.QueryConfigValues(["cluster_name"])
250 raise utils.RetryAgain()
252 logging.debug("Received cluster name %s from master", cluster_name)
255 utils.Retry(_CheckMasterDaemon, 1.0, _DAEMON_READY_TIMEOUT)
256 except utils.RetryTimeout:
257 raise errors.OpExecError("Master daemon didn't answer queries within"
258 " %s seconds" % _DAEMON_READY_TIMEOUT)
261 def _WaitForSshDaemon(hostname, port, family):
262 """Wait for SSH daemon to become responsive.
265 hostip = netutils.GetHostname(name=hostname, family=family).ip
267 def _CheckSshDaemon():
268 if netutils.TcpPing(hostip, port, timeout=1.0, live_port_needed=True):
269 logging.debug("SSH daemon on %s:%s (IP address %s) has become"
270 " responsive", hostname, port, hostip)
272 raise utils.RetryAgain()
275 utils.Retry(_CheckSshDaemon, 1.0, _DAEMON_READY_TIMEOUT)
276 except utils.RetryTimeout:
277 raise errors.OpExecError("SSH daemon on %s:%s (IP address %s) didn't"
278 " become responsive within %s seconds" %
279 (hostname, port, hostip, _DAEMON_READY_TIMEOUT))
282 def RunNodeSetupCmd(cluster_name, node, basecmd, debug, verbose,
283 use_cluster_key, ask_key, strict_host_check, data):
284 """Runs a command to configure something on a remote machine.
286 @type cluster_name: string
287 @param cluster_name: Cluster name
289 @param node: Node name
290 @type basecmd: string
291 @param basecmd: Base command (path on the remote machine)
293 @param debug: Enable debug output
295 @param verbose: Enable verbose output
296 @type use_cluster_key: bool
297 @param use_cluster_key: See L{ssh.SshRunner.BuildCmd}
299 @param ask_key: See L{ssh.SshRunner.BuildCmd}
300 @type strict_host_check: bool
301 @param strict_host_check: See L{ssh.SshRunner.BuildCmd}
302 @param data: JSON-serializable input data for script (passed to stdin)
307 # Pass --debug/--verbose to the external script if set on our invocation
309 cmd.append("--debug")
312 cmd.append("--verbose")
314 family = ssconf.SimpleStore().GetPrimaryIPFamily()
315 srun = ssh.SshRunner(cluster_name,
316 ipv6=(family == netutils.IP6Address.family))
317 scmd = srun.BuildCmd(node, constants.SSH_LOGIN_USER,
318 utils.ShellQuoteArgs(cmd),
319 batch=False, ask_key=ask_key, quiet=False,
320 strict_host_check=strict_host_check,
321 use_cluster_key=use_cluster_key)
323 tempfh = tempfile.TemporaryFile()
325 tempfh.write(serializer.DumpJson(data))
328 result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh)
333 raise errors.OpExecError("Command '%s' failed: %s" %
334 (result.cmd, result.fail_reason))
336 _WaitForSshDaemon(node, netutils.GetDaemonPort(constants.SSH), family)
339 def _InitFileStorageDir(file_storage_dir):
340 """Initialize if needed the file storage.
342 @param file_storage_dir: the user-supplied value
343 @return: either empty string (if file storage was disabled at build
344 time) or the normalized path to the storage directory
347 file_storage_dir = os.path.normpath(file_storage_dir)
349 if not os.path.isabs(file_storage_dir):
350 raise errors.OpPrereqError("File storage directory '%s' is not an absolute"
351 " path" % file_storage_dir, errors.ECODE_INVAL)
353 if not os.path.exists(file_storage_dir):
355 os.makedirs(file_storage_dir, 0750)
357 raise errors.OpPrereqError("Cannot create file storage directory"
358 " '%s': %s" % (file_storage_dir, err),
359 errors.ECODE_ENVIRON)
361 if not os.path.isdir(file_storage_dir):
362 raise errors.OpPrereqError("The file storage directory '%s' is not"
363 " a directory." % file_storage_dir,
364 errors.ECODE_ENVIRON)
366 return file_storage_dir
369 def _PrepareFileBasedStorage(
370 enabled_disk_templates, file_storage_dir,
371 default_dir, file_disk_template,
372 init_fn=_InitFileStorageDir, acceptance_fn=None):
373 """Checks if a file-base storage type is enabled and inits the dir.
375 @type enabled_disk_templates: list of string
376 @param enabled_disk_templates: list of enabled disk templates
377 @type file_storage_dir: string
378 @param file_storage_dir: the file storage directory
379 @type default_dir: string
380 @param default_dir: default file storage directory when C{file_storage_dir}
382 @type file_disk_template: string
383 @param file_disk_template: a disk template whose storage type is 'ST_FILE'
385 @returns: the name of the actual file storage directory
388 assert (file_disk_template in
389 utils.storage.GetDiskTemplatesOfStorageType(constants.ST_FILE))
390 if file_storage_dir is None:
391 file_storage_dir = default_dir
392 if not acceptance_fn:
394 lambda path: filestorage.CheckFileStoragePathAcceptance(
395 path, exact_match_ok=True)
397 cluster.CheckFileStoragePathVsEnabledDiskTemplates(
398 logging.warning, file_storage_dir, enabled_disk_templates)
400 file_storage_enabled = file_disk_template in enabled_disk_templates
401 if file_storage_enabled:
403 acceptance_fn(file_storage_dir)
404 except errors.FileStoragePathError as e:
405 raise errors.OpPrereqError(str(e))
406 result_file_storage_dir = init_fn(file_storage_dir)
408 result_file_storage_dir = file_storage_dir
409 return result_file_storage_dir
412 def _PrepareFileStorage(
413 enabled_disk_templates, file_storage_dir, init_fn=_InitFileStorageDir,
415 """Checks if file storage is enabled and inits the dir.
417 @see: C{_PrepareFileBasedStorage}
420 return _PrepareFileBasedStorage(
421 enabled_disk_templates, file_storage_dir,
422 pathutils.DEFAULT_FILE_STORAGE_DIR, constants.DT_FILE,
423 init_fn=init_fn, acceptance_fn=acceptance_fn)
426 def _PrepareSharedFileStorage(
427 enabled_disk_templates, file_storage_dir, init_fn=_InitFileStorageDir,
429 """Checks if shared file storage is enabled and inits the dir.
431 @see: C{_PrepareFileBasedStorage}
434 return _PrepareFileBasedStorage(
435 enabled_disk_templates, file_storage_dir,
436 pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR, constants.DT_SHARED_FILE,
437 init_fn=init_fn, acceptance_fn=acceptance_fn)
440 def _InitCheckEnabledDiskTemplates(enabled_disk_templates):
441 """Checks the sanity of the enabled disk templates.
444 if not enabled_disk_templates:
445 raise errors.OpPrereqError("Enabled disk templates list must contain at"
446 " least one member", errors.ECODE_INVAL)
447 invalid_disk_templates = \
448 set(enabled_disk_templates) - constants.DISK_TEMPLATES
449 if invalid_disk_templates:
450 raise errors.OpPrereqError("Enabled disk templates list contains invalid"
451 " entries: %s" % invalid_disk_templates,
455 def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
456 master_netmask, master_netdev, file_storage_dir,
457 shared_file_storage_dir, candidate_pool_size, secondary_ip=None,
458 vg_name=None, beparams=None, nicparams=None, ndparams=None,
459 hvparams=None, diskparams=None, enabled_hypervisors=None,
460 modify_etc_hosts=True, modify_ssh_setup=True,
461 maintain_node_health=False, drbd_helper=None, uid_pool=None,
462 default_iallocator=None, primary_ip_version=None, ipolicy=None,
463 prealloc_wipe_disks=False, use_external_mip_script=False,
464 hv_state=None, disk_state=None, enabled_disk_templates=None):
465 """Initialise the cluster.
467 @type candidate_pool_size: int
468 @param candidate_pool_size: master candidate pool size
469 @type enabled_disk_templates: list of string
470 @param enabled_disk_templates: list of disk_templates to be used in this
474 # TODO: complete the docstring
475 if config.ConfigWriter.IsCluster():
476 raise errors.OpPrereqError("Cluster is already initialised",
479 if not enabled_hypervisors:
480 raise errors.OpPrereqError("Enabled hypervisors list must contain at"
481 " least one member", errors.ECODE_INVAL)
482 invalid_hvs = set(enabled_hypervisors) - constants.HYPER_TYPES
484 raise errors.OpPrereqError("Enabled hypervisors contains invalid"
485 " entries: %s" % invalid_hvs,
488 _InitCheckEnabledDiskTemplates(enabled_disk_templates)
491 ipcls = netutils.IPAddress.GetClassFromIpVersion(primary_ip_version)
492 except errors.ProgrammerError:
493 raise errors.OpPrereqError("Invalid primary ip version: %d." %
494 primary_ip_version, errors.ECODE_INVAL)
496 hostname = netutils.GetHostname(family=ipcls.family)
497 if not ipcls.IsValid(hostname.ip):
498 raise errors.OpPrereqError("This host's IP (%s) is not a valid IPv%d"
499 " address." % (hostname.ip, primary_ip_version),
502 if ipcls.IsLoopback(hostname.ip):
503 raise errors.OpPrereqError("This host's IP (%s) resolves to a loopback"
504 " address. Please fix DNS or %s." %
505 (hostname.ip, pathutils.ETC_HOSTS),
506 errors.ECODE_ENVIRON)
508 if not ipcls.Own(hostname.ip):
509 raise errors.OpPrereqError("Inconsistency: this host's name resolves"
510 " to %s,\nbut this ip address does not"
511 " belong to this host" %
512 hostname.ip, errors.ECODE_ENVIRON)
514 clustername = netutils.GetHostname(name=cluster_name, family=ipcls.family)
516 if netutils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT, timeout=5):
517 raise errors.OpPrereqError("Cluster IP already active",
518 errors.ECODE_NOTUNIQUE)
521 if primary_ip_version == constants.IP6_VERSION:
522 raise errors.OpPrereqError("When using a IPv6 primary address, a valid"
523 " IPv4 address must be given as secondary",
525 secondary_ip = hostname.ip
527 if not netutils.IP4Address.IsValid(secondary_ip):
528 raise errors.OpPrereqError("Secondary IP address (%s) has to be a valid"
529 " IPv4 address." % secondary_ip,
532 if not netutils.IP4Address.Own(secondary_ip):
533 raise errors.OpPrereqError("You gave %s as secondary IP,"
534 " but it does not belong to this host." %
535 secondary_ip, errors.ECODE_ENVIRON)
537 if master_netmask is not None:
538 if not ipcls.ValidateNetmask(master_netmask):
539 raise errors.OpPrereqError("CIDR netmask (%s) not valid for IPv%s " %
540 (master_netmask, primary_ip_version),
543 master_netmask = ipcls.iplen
546 # Check if volume group is valid
547 vgstatus = utils.CheckVolumeGroupSize(utils.ListVolumeGroups(), vg_name,
548 constants.MIN_VG_SIZE)
550 raise errors.OpPrereqError("Error: %s" % vgstatus, errors.ECODE_INVAL)
552 if drbd_helper is not None:
554 curr_helper = drbd.DRBD8.GetUsermodeHelper()
555 except errors.BlockDeviceError, err:
556 raise errors.OpPrereqError("Error while checking drbd helper"
557 " (specify --no-drbd-storage if you are not"
558 " using drbd): %s" % str(err),
559 errors.ECODE_ENVIRON)
560 if drbd_helper != curr_helper:
561 raise errors.OpPrereqError("Error: requiring %s as drbd helper but %s"
562 " is the current helper" % (drbd_helper,
566 logging.debug("Stopping daemons (if any are running)")
567 result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-all"])
569 raise errors.OpExecError("Could not stop daemons, command %s"
570 " had exitcode %s and error '%s'" %
571 (result.cmd, result.exit_code, result.output))
573 file_storage_dir = _PrepareFileStorage(enabled_disk_templates,
575 shared_file_storage_dir = _PrepareSharedFileStorage(enabled_disk_templates,
576 shared_file_storage_dir)
578 if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
579 raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix,
582 result = utils.RunCmd(["ip", "link", "show", "dev", master_netdev])
584 raise errors.OpPrereqError("Invalid master netdev given (%s): '%s'" %
586 result.output.strip()), errors.ECODE_INVAL)
588 dirs = [(pathutils.RUN_DIR, constants.RUN_DIRS_MODE)]
589 utils.EnsureDirs(dirs)
591 objects.UpgradeBeParams(beparams)
592 utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
593 utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
595 objects.NIC.CheckParameterSyntax(nicparams)
597 full_ipolicy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, ipolicy)
599 if ndparams is not None:
600 utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
602 ndparams = dict(constants.NDC_DEFAULTS)
604 # This is ugly, as we modify the dict itself
605 # FIXME: Make utils.ForceDictType pure functional or write a wrapper
608 for hvname, hvs_data in hv_state.items():
609 utils.ForceDictType(hvs_data, constants.HVSTS_PARAMETER_TYPES)
610 hv_state[hvname] = objects.Cluster.SimpleFillHvState(hvs_data)
612 hv_state = dict((hvname, constants.HVST_DEFAULTS)
613 for hvname in enabled_hypervisors)
615 # FIXME: disk_state has no default values yet
617 for storage, ds_data in disk_state.items():
618 if storage not in constants.DS_VALID_TYPES:
619 raise errors.OpPrereqError("Invalid storage type in disk state: %s" %
620 storage, errors.ECODE_INVAL)
621 for ds_name, state in ds_data.items():
622 utils.ForceDictType(state, constants.DSS_PARAMETER_TYPES)
623 ds_data[ds_name] = objects.Cluster.SimpleFillDiskState(state)
625 # hvparams is a mapping of hypervisor->hvparams dict
626 for hv_name, hv_params in hvparams.iteritems():
627 utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
628 hv_class = hypervisor.GetHypervisor(hv_name)
629 hv_class.CheckParameterSyntax(hv_params)
631 # diskparams is a mapping of disk-template->diskparams dict
632 for template, dt_params in diskparams.items():
633 param_keys = set(dt_params.keys())
634 default_param_keys = set(constants.DISK_DT_DEFAULTS[template].keys())
635 if not (param_keys <= default_param_keys):
636 unknown_params = param_keys - default_param_keys
637 raise errors.OpPrereqError("Invalid parameters for disk template %s:"
639 utils.CommaJoin(unknown_params)),
641 utils.ForceDictType(dt_params, constants.DISK_DT_TYPES)
642 if template == constants.DT_DRBD8 and vg_name is not None:
643 # The default METAVG value is equal to the VG name set at init time,
645 dt_params[constants.DRBD_DEFAULT_METAVG] = vg_name
648 utils.VerifyDictOptions(diskparams, constants.DISK_DT_DEFAULTS)
649 except errors.OpPrereqError, err:
650 raise errors.OpPrereqError("While verify diskparam options: %s" % err,
653 # set up ssh config and /etc/hosts
656 if os.path.isfile(pathutils.SSH_HOST_RSA_PUB):
657 sshline = utils.ReadFile(pathutils.SSH_HOST_RSA_PUB)
658 rsa_sshkey = sshline.split(" ")[1]
659 if os.path.isfile(pathutils.SSH_HOST_DSA_PUB):
660 sshline = utils.ReadFile(pathutils.SSH_HOST_DSA_PUB)
661 dsa_sshkey = sshline.split(" ")[1]
662 if not rsa_sshkey and not dsa_sshkey:
663 raise errors.OpPrereqError("Failed to find SSH public keys",
664 errors.ECODE_ENVIRON)
667 utils.AddHostToEtcHosts(hostname.name, hostname.ip)
672 if default_iallocator is not None:
673 alloc_script = utils.FindFile(default_iallocator,
674 constants.IALLOCATOR_SEARCH_PATH,
676 if alloc_script is None:
677 raise errors.OpPrereqError("Invalid default iallocator script '%s'"
678 " specified" % default_iallocator,
680 elif constants.HTOOLS:
681 # htools was enabled at build-time, we default to it
682 if utils.FindFile(constants.IALLOC_HAIL,
683 constants.IALLOCATOR_SEARCH_PATH,
685 default_iallocator = constants.IALLOC_HAIL
689 # init of cluster config file
690 cluster_config = objects.Cluster(
692 rsahostkeypub=rsa_sshkey,
693 dsahostkeypub=dsa_sshkey,
694 highest_used_port=(constants.FIRST_DRBD_PORT - 1),
695 mac_prefix=mac_prefix,
696 volume_group_name=vg_name,
697 tcpudp_port_pool=set(),
698 master_ip=clustername.ip,
699 master_netmask=master_netmask,
700 master_netdev=master_netdev,
701 cluster_name=clustername.name,
702 file_storage_dir=file_storage_dir,
703 shared_file_storage_dir=shared_file_storage_dir,
704 enabled_hypervisors=enabled_hypervisors,
705 beparams={constants.PP_DEFAULT: beparams},
706 nicparams={constants.PP_DEFAULT: nicparams},
709 diskparams=diskparams,
710 candidate_pool_size=candidate_pool_size,
711 modify_etc_hosts=modify_etc_hosts,
712 modify_ssh_setup=modify_ssh_setup,
716 maintain_node_health=maintain_node_health,
717 drbd_usermode_helper=drbd_helper,
718 default_iallocator=default_iallocator,
719 primary_ip_family=ipcls.family,
720 prealloc_wipe_disks=prealloc_wipe_disks,
721 use_external_mip_script=use_external_mip_script,
722 ipolicy=full_ipolicy,
723 hv_state_static=hv_state,
724 disk_state_static=disk_state,
725 enabled_disk_templates=enabled_disk_templates,
727 master_node_config = objects.Node(name=hostname.name,
728 primary_ip=hostname.ip,
729 secondary_ip=secondary_ip,
731 master_candidate=True,
732 offline=False, drained=False,
733 ctime=now, mtime=now,
735 InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
736 cfg = config.ConfigWriter(offline=True)
737 ssh.WriteKnownHostsFile(cfg, pathutils.SSH_KNOWN_HOSTS_FILE)
738 cfg.Update(cfg.GetClusterInfo(), logging.error)
739 ssconf.WriteSsconfFiles(cfg.GetSsconfValues())
741 # set up the inter-node password and certificate
742 _InitGanetiServerSetup(hostname.name)
744 logging.debug("Starting daemons")
745 result = utils.RunCmd([pathutils.DAEMON_UTIL, "start-all"])
747 raise errors.OpExecError("Could not start daemons, command %s"
748 " had exitcode %s and error %s" %
749 (result.cmd, result.exit_code, result.output))
751 _WaitForMasterDaemon()
754 def InitConfig(version, cluster_config, master_node_config,
755 cfg_file=pathutils.CLUSTER_CONF_FILE):
756 """Create the initial cluster configuration.
758 It will contain the current node, which will also be the master
759 node, and no instances.
762 @param version: configuration version
763 @type cluster_config: L{objects.Cluster}
764 @param cluster_config: cluster configuration
765 @type master_node_config: L{objects.Node}
766 @param master_node_config: master node configuration
767 @type cfg_file: string
768 @param cfg_file: configuration file path
771 uuid_generator = config.TemporaryReservationManager()
772 cluster_config.uuid = uuid_generator.Generate([], utils.NewUUID,
774 master_node_config.uuid = uuid_generator.Generate([], utils.NewUUID,
776 cluster_config.master_node = master_node_config.uuid
778 master_node_config.uuid: master_node_config,
780 default_nodegroup = objects.NodeGroup(
781 uuid=uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID),
782 name=constants.INITIAL_NODE_GROUP_NAME,
783 members=[master_node_config.uuid],
787 default_nodegroup.uuid: default_nodegroup,
790 config_data = objects.ConfigData(version=version,
791 cluster=cluster_config,
792 nodegroups=nodegroups,
797 ctime=now, mtime=now)
798 utils.WriteFile(cfg_file,
799 data=serializer.Dump(config_data.ToDict()),
803 def FinalizeClusterDestroy(master_uuid):
804 """Execute the last steps of cluster destroy
806 This function shuts down all the daemons, completing the destroy
807 begun in cmdlib.LUDestroyOpcode.
810 cfg = config.ConfigWriter()
811 modify_ssh_setup = cfg.GetClusterInfo().modify_ssh_setup
812 runner = rpc.BootstrapRunner()
814 master_name = cfg.GetNodeName(master_uuid)
816 master_params = cfg.GetMasterNetworkParameters()
817 master_params.uuid = master_uuid
818 ems = cfg.GetUseExternalMipScript()
819 result = runner.call_node_deactivate_master_ip(master_name, master_params,
822 msg = result.fail_msg
824 logging.warning("Could not disable the master IP: %s", msg)
826 result = runner.call_node_stop_master(master_name)
827 msg = result.fail_msg
829 logging.warning("Could not disable the master role: %s", msg)
831 result = runner.call_node_leave_cluster(master_name, modify_ssh_setup)
832 msg = result.fail_msg
834 logging.warning("Could not shutdown the node daemon and cleanup"
835 " the node: %s", msg)
838 def SetupNodeDaemon(opts, cluster_name, node):
839 """Add a node to the cluster.
841 This function must be called before the actual opcode, and will ssh
842 to the remote node, copy the needed files, and start ganeti-noded,
843 allowing the master to do the rest via normal rpc calls.
845 @param cluster_name: the cluster name
846 @param node: the name of the new node
850 constants.NDS_CLUSTER_NAME: cluster_name,
851 constants.NDS_NODE_DAEMON_CERTIFICATE:
852 utils.ReadFile(pathutils.NODED_CERT_FILE),
853 constants.NDS_SSCONF: ssconf.SimpleStore().ReadAll(),
854 constants.NDS_START_NODE_DAEMON: True,
857 RunNodeSetupCmd(cluster_name, node, pathutils.NODE_DAEMON_SETUP,
858 opts.debug, opts.verbose,
859 True, opts.ssh_key_check, opts.ssh_key_check, data)
861 _WaitForNodeDaemon(node)
864 def MasterFailover(no_voting=False):
865 """Failover the master node.
867 This checks that we are not already the master, and will cause the
868 current master to cease being master, and the non-master to become
871 @type no_voting: boolean
872 @param no_voting: force the operation without remote nodes agreement
876 sstore = ssconf.SimpleStore()
878 old_master, new_master = ssconf.GetMasterAndMyself(sstore)
879 node_names = sstore.GetNodeList()
880 mc_list = sstore.GetMasterCandidates()
882 if old_master == new_master:
883 raise errors.OpPrereqError("This commands must be run on the node"
884 " where you want the new master to be."
885 " %s is already the master" %
886 old_master, errors.ECODE_INVAL)
888 if new_master not in mc_list:
889 mc_no_master = [name for name in mc_list if name != old_master]
890 raise errors.OpPrereqError("This node is not among the nodes marked"
891 " as master candidates. Only these nodes"
892 " can become masters. Current list of"
893 " master candidates is:\n"
894 "%s" % ("\n".join(mc_no_master)),
898 vote_list = GatherMasterVotes(node_names)
901 voted_master = vote_list[0][0]
902 if voted_master is None:
903 raise errors.OpPrereqError("Cluster is inconsistent, most nodes did"
904 " not respond.", errors.ECODE_ENVIRON)
905 elif voted_master != old_master:
906 raise errors.OpPrereqError("I have a wrong configuration, I believe"
907 " the master is %s but the other nodes"
908 " voted %s. Please resync the configuration"
910 (old_master, voted_master),
916 logging.info("Setting master to %s, old master: %s", new_master, old_master)
919 # instantiate a real config writer, as we now know we have the
921 cfg = config.ConfigWriter(accept_foreign=True)
923 old_master_node = cfg.GetNodeInfoByName(old_master)
924 if old_master_node is None:
925 raise errors.OpPrereqError("Could not find old master node '%s' in"
926 " cluster configuration." % old_master,
929 cluster_info = cfg.GetClusterInfo()
930 new_master_node = cfg.GetNodeInfoByName(new_master)
931 if new_master_node is None:
932 raise errors.OpPrereqError("Could not find new master node '%s' in"
933 " cluster configuration." % new_master,
936 cluster_info.master_node = new_master_node.uuid
937 # this will also regenerate the ssconf files, since we updated the
939 cfg.Update(cluster_info, logging.error)
940 except errors.ConfigurationError, err:
941 logging.error("Error while trying to set the new master: %s",
945 # if cfg.Update worked, then it means the old master daemon won't be
946 # able now to write its own config file (we rely on locking in both
947 # backend.UploadFile() and ConfigWriter._Write(); hence the next
948 # step is to kill the old master
950 logging.info("Stopping the master daemon on node %s", old_master)
952 runner = rpc.BootstrapRunner()
953 master_params = cfg.GetMasterNetworkParameters()
954 master_params.uuid = old_master_node.uuid
955 ems = cfg.GetUseExternalMipScript()
956 result = runner.call_node_deactivate_master_ip(old_master,
959 msg = result.fail_msg
961 logging.warning("Could not disable the master IP: %s", msg)
963 result = runner.call_node_stop_master(old_master)
964 msg = result.fail_msg
966 logging.error("Could not disable the master role on the old master"
967 " %s, please disable manually: %s", old_master, msg)
969 logging.info("Checking master IP non-reachability...")
971 master_ip = sstore.GetMasterIP()
974 # Here we have a phase where no master should be running
976 if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):
977 raise utils.RetryAgain()
980 utils.Retry(_check_ip, (1, 1.5, 5), total_timeout)
981 except utils.RetryTimeout:
982 logging.warning("The master IP is still reachable after %s seconds,"
983 " continuing but activating the master on the current"
984 " node will probably fail", total_timeout)
986 if jstore.CheckDrainFlag():
987 logging.info("Undraining job queue")
988 jstore.SetDrainFlag(False)
990 logging.info("Starting the master daemons on the new master")
992 result = rpc.BootstrapRunner().call_node_start_master_daemons(new_master,
994 msg = result.fail_msg
996 logging.error("Could not start the master role on the new master"
997 " %s, please check: %s", new_master, msg)
1000 logging.info("Master failed over from %s to %s", old_master, new_master)
1005 """Returns the current master node.
1007 This is a separate function in bootstrap since it's needed by
1008 gnt-cluster, and instead of importing directly ssconf, it's better
1009 to abstract it in bootstrap, where we do use ssconf in other
1013 sstore = ssconf.SimpleStore()
1015 old_master, _ = ssconf.GetMasterAndMyself(sstore)
1020 def GatherMasterVotes(node_names):
1021 """Check the agreement on who is the master.
1023 This function will return a list of (node, number of votes), ordered
1024 by the number of votes. Errors will be denoted by the key 'None'.
1026 Note that the sum of votes is the number of nodes this machine
1027 knows, whereas the number of entries in the list could be different
1028 (if some nodes vote for another master).
1030 We remove ourselves from the list since we know that (bugs aside)
1031 since we use the same source for configuration information for both
1032 backend and boostrap, we'll always vote for ourselves.
1034 @type node_names: list
1035 @param node_names: the list of nodes to query for master info; the current
1036 node will be removed if it is in the list
1038 @return: list of (node, votes)
1041 myself = netutils.Hostname.GetSysName()
1043 node_names.remove(myself)
1047 # no nodes left (eventually after removing myself)
1049 results = rpc.BootstrapRunner().call_master_info(node_names)
1050 if not isinstance(results, dict):
1051 # this should not happen (unless internal error in rpc)
1052 logging.critical("Can't complete rpc call, aborting master startup")
1053 return [(None, len(node_names))]
1055 for node_name in results:
1056 nres = results[node_name]
1061 logging.warning("Error contacting node %s: %s", node_name, msg)
1063 # for now we accept both length 3, 4 and 5 (data[3] is primary ip version
1064 # and data[4] is the master netmask)
1065 elif not isinstance(data, (tuple, list)) or len(data) < 3:
1066 logging.warning("Invalid data received from node %s: %s",
1070 if None not in votes:
1074 master_node = data[2]
1075 if master_node not in votes:
1076 votes[master_node] = 0
1077 votes[master_node] += 1
1079 vote_list = [v for v in votes.items()]
1080 # sort first on number of votes then on name, since we want None
1081 # sorted later if we have the half of the nodes not responding, and
1082 # half voting all for the same master
1083 vote_list.sort(key=lambda x: (x[1], x[0]), reverse=True)