4 # Copyright (C) 2006, 2007, 2008, 2010, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Functions to bootstrap a new cluster.
33 from ganeti.cmdlib import cluster
34 from ganeti import rpc
35 from ganeti import ssh
36 from ganeti import utils
37 from ganeti import errors
38 from ganeti import config
39 from ganeti import constants
40 from ganeti import objects
41 from ganeti import ssconf
42 from ganeti import serializer
43 from ganeti import hypervisor
44 from ganeti.storage import drbd
45 from ganeti.storage import filestorage
46 from ganeti import netutils
47 from ganeti import luxi
48 from ganeti import jstore
49 from ganeti import pathutils
52 # ec_id for InitConfig's temporary reservation manager
53 _INITCONF_ECID = "initconfig-ecid"
55 #: After how many seconds daemon must be responsive
56 _DAEMON_READY_TIMEOUT = 10.0
60 """Setup the SSH configuration for the cluster.
62 This generates a dsa keypair for root, adds the pub key to the
63 permitted hosts and adds the hostkey to its own known hosts.
66 priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.SSH_LOGIN_USER)
68 for name in priv_key, pub_key:
69 if os.path.exists(name):
70 utils.CreateBackup(name)
71 utils.RemoveFile(name)
73 result = utils.RunCmd(["ssh-keygen", "-t", "dsa",
77 raise errors.OpExecError("Could not generate ssh keypair, error %s" %
80 utils.AddAuthorizedKey(auth_keys, utils.ReadFile(pub_key))
83 def GenerateHmacKey(file_name):
84 """Writes a new HMAC key.
87 @param file_name: Path to output file
90 utils.WriteFile(file_name, data="%s\n" % utils.GenerateSecret(), mode=0400,
94 def GenerateClusterCrypto(new_cluster_cert, new_rapi_cert, new_spice_cert,
95 new_confd_hmac_key, new_cds,
96 rapi_cert_pem=None, spice_cert_pem=None,
97 spice_cacert_pem=None, cds=None,
98 nodecert_file=pathutils.NODED_CERT_FILE,
99 rapicert_file=pathutils.RAPI_CERT_FILE,
100 spicecert_file=pathutils.SPICE_CERT_FILE,
101 spicecacert_file=pathutils.SPICE_CACERT_FILE,
102 hmackey_file=pathutils.CONFD_HMAC_KEY,
103 cds_file=pathutils.CLUSTER_DOMAIN_SECRET_FILE):
104 """Updates the cluster certificates, keys and secrets.
106 @type new_cluster_cert: bool
107 @param new_cluster_cert: Whether to generate a new cluster certificate
108 @type new_rapi_cert: bool
109 @param new_rapi_cert: Whether to generate a new RAPI certificate
110 @type new_spice_cert: bool
111 @param new_spice_cert: Whether to generate a new SPICE certificate
112 @type new_confd_hmac_key: bool
113 @param new_confd_hmac_key: Whether to generate a new HMAC key
115 @param new_cds: Whether to generate a new cluster domain secret
116 @type rapi_cert_pem: string
117 @param rapi_cert_pem: New RAPI certificate in PEM format
118 @type spice_cert_pem: string
119 @param spice_cert_pem: New SPICE certificate in PEM format
120 @type spice_cacert_pem: string
121 @param spice_cacert_pem: Certificate of the CA that signed the SPICE
122 certificate, in PEM format
124 @param cds: New cluster domain secret
125 @type nodecert_file: string
126 @param nodecert_file: optional override of the node cert file path
127 @type rapicert_file: string
128 @param rapicert_file: optional override of the rapi cert file path
129 @type spicecert_file: string
130 @param spicecert_file: optional override of the spice cert file path
131 @type spicecacert_file: string
132 @param spicecacert_file: optional override of the spice CA cert file path
133 @type hmackey_file: string
134 @param hmackey_file: optional override of the hmac key file path
137 # noded SSL certificate
138 cluster_cert_exists = os.path.exists(nodecert_file)
139 if new_cluster_cert or not cluster_cert_exists:
140 if cluster_cert_exists:
141 utils.CreateBackup(nodecert_file)
143 logging.debug("Generating new cluster certificate at %s", nodecert_file)
144 utils.GenerateSelfSignedSslCert(nodecert_file)
147 if new_confd_hmac_key or not os.path.exists(hmackey_file):
148 logging.debug("Writing new confd HMAC key to %s", hmackey_file)
149 GenerateHmacKey(hmackey_file)
152 rapi_cert_exists = os.path.exists(rapicert_file)
155 # Assume rapi_pem contains a valid PEM-formatted certificate and key
156 logging.debug("Writing RAPI certificate at %s", rapicert_file)
157 utils.WriteFile(rapicert_file, data=rapi_cert_pem, backup=True)
159 elif new_rapi_cert or not rapi_cert_exists:
161 utils.CreateBackup(rapicert_file)
163 logging.debug("Generating new RAPI certificate at %s", rapicert_file)
164 utils.GenerateSelfSignedSslCert(rapicert_file)
167 spice_cert_exists = os.path.exists(spicecert_file)
168 spice_cacert_exists = os.path.exists(spicecacert_file)
170 # spice_cert_pem implies also spice_cacert_pem
171 logging.debug("Writing SPICE certificate at %s", spicecert_file)
172 utils.WriteFile(spicecert_file, data=spice_cert_pem, backup=True)
173 logging.debug("Writing SPICE CA certificate at %s", spicecacert_file)
174 utils.WriteFile(spicecacert_file, data=spice_cacert_pem, backup=True)
175 elif new_spice_cert or not spice_cert_exists:
176 if spice_cert_exists:
177 utils.CreateBackup(spicecert_file)
178 if spice_cacert_exists:
179 utils.CreateBackup(spicecacert_file)
181 logging.debug("Generating new self-signed SPICE certificate at %s",
183 (_, cert_pem) = utils.GenerateSelfSignedSslCert(spicecert_file)
185 # Self-signed certificate -> the public certificate is also the CA public
187 logging.debug("Writing the public certificate to %s",
189 utils.io.WriteFile(spicecacert_file, mode=0400, data=cert_pem)
191 # Cluster domain secret
193 logging.debug("Writing cluster domain secret to %s", cds_file)
194 utils.WriteFile(cds_file, data=cds, backup=True)
196 elif new_cds or not os.path.exists(cds_file):
197 logging.debug("Generating new cluster domain secret at %s", cds_file)
198 GenerateHmacKey(cds_file)
201 def _InitGanetiServerSetup(master_name):
202 """Setup the necessary configuration for the initial node daemon.
204 This creates the nodepass file containing the shared password for
205 the cluster, generates the SSL certificate and starts the node daemon.
207 @type master_name: str
208 @param master_name: Name of the master node
211 # Generate cluster secrets
212 GenerateClusterCrypto(True, False, False, False, False)
214 result = utils.RunCmd([pathutils.DAEMON_UTIL, "start", constants.NODED])
216 raise errors.OpExecError("Could not start the node daemon, command %s"
217 " had exitcode %s and error %s" %
218 (result.cmd, result.exit_code, result.output))
220 _WaitForNodeDaemon(master_name)
223 def _WaitForNodeDaemon(node_name):
224 """Wait for node daemon to become responsive.
227 def _CheckNodeDaemon():
228 # Pylint bug <http://www.logilab.org/ticket/35642>
229 # pylint: disable=E1101
230 result = rpc.BootstrapRunner().call_version([node_name])[node_name]
232 raise utils.RetryAgain()
235 utils.Retry(_CheckNodeDaemon, 1.0, _DAEMON_READY_TIMEOUT)
236 except utils.RetryTimeout:
237 raise errors.OpExecError("Node daemon on %s didn't answer queries within"
238 " %s seconds" % (node_name, _DAEMON_READY_TIMEOUT))
241 def _WaitForMasterDaemon():
242 """Wait for master daemon to become responsive.
245 def _CheckMasterDaemon():
248 (cluster_name, ) = cl.QueryConfigValues(["cluster_name"])
250 raise utils.RetryAgain()
252 logging.debug("Received cluster name %s from master", cluster_name)
255 utils.Retry(_CheckMasterDaemon, 1.0, _DAEMON_READY_TIMEOUT)
256 except utils.RetryTimeout:
257 raise errors.OpExecError("Master daemon didn't answer queries within"
258 " %s seconds" % _DAEMON_READY_TIMEOUT)
261 def _WaitForSshDaemon(hostname, port, family):
262 """Wait for SSH daemon to become responsive.
265 hostip = netutils.GetHostname(name=hostname, family=family).ip
267 def _CheckSshDaemon():
268 if netutils.TcpPing(hostip, port, timeout=1.0, live_port_needed=True):
269 logging.debug("SSH daemon on %s:%s (IP address %s) has become"
270 " responsive", hostname, port, hostip)
272 raise utils.RetryAgain()
275 utils.Retry(_CheckSshDaemon, 1.0, _DAEMON_READY_TIMEOUT)
276 except utils.RetryTimeout:
277 raise errors.OpExecError("SSH daemon on %s:%s (IP address %s) didn't"
278 " become responsive within %s seconds" %
279 (hostname, port, hostip, _DAEMON_READY_TIMEOUT))
282 def RunNodeSetupCmd(cluster_name, node, basecmd, debug, verbose,
283 use_cluster_key, ask_key, strict_host_check, data):
284 """Runs a command to configure something on a remote machine.
286 @type cluster_name: string
287 @param cluster_name: Cluster name
289 @param node: Node name
290 @type basecmd: string
291 @param basecmd: Base command (path on the remote machine)
293 @param debug: Enable debug output
295 @param verbose: Enable verbose output
296 @type use_cluster_key: bool
297 @param use_cluster_key: See L{ssh.SshRunner.BuildCmd}
299 @param ask_key: See L{ssh.SshRunner.BuildCmd}
300 @type strict_host_check: bool
301 @param strict_host_check: See L{ssh.SshRunner.BuildCmd}
302 @param data: JSON-serializable input data for script (passed to stdin)
307 # Pass --debug/--verbose to the external script if set on our invocation
309 cmd.append("--debug")
312 cmd.append("--verbose")
314 family = ssconf.SimpleStore().GetPrimaryIPFamily()
315 srun = ssh.SshRunner(cluster_name,
316 ipv6=(family == netutils.IP6Address.family))
317 scmd = srun.BuildCmd(node, constants.SSH_LOGIN_USER,
318 utils.ShellQuoteArgs(cmd),
319 batch=False, ask_key=ask_key, quiet=False,
320 strict_host_check=strict_host_check,
321 use_cluster_key=use_cluster_key)
323 tempfh = tempfile.TemporaryFile()
325 tempfh.write(serializer.DumpJson(data))
328 result = utils.RunCmd(scmd, interactive=True, input_fd=tempfh)
333 raise errors.OpExecError("Command '%s' failed: %s" %
334 (result.cmd, result.fail_reason))
336 _WaitForSshDaemon(node, netutils.GetDaemonPort(constants.SSH), family)
339 def _InitFileStorageDir(file_storage_dir):
340 """Initialize if needed the file storage.
342 @param file_storage_dir: the user-supplied value
343 @return: either empty string (if file storage was disabled at build
344 time) or the normalized path to the storage directory
347 file_storage_dir = os.path.normpath(file_storage_dir)
349 if not os.path.isabs(file_storage_dir):
350 raise errors.OpPrereqError("File storage directory '%s' is not an absolute"
351 " path" % file_storage_dir, errors.ECODE_INVAL)
353 if not os.path.exists(file_storage_dir):
355 os.makedirs(file_storage_dir, 0750)
357 raise errors.OpPrereqError("Cannot create file storage directory"
358 " '%s': %s" % (file_storage_dir, err),
359 errors.ECODE_ENVIRON)
361 if not os.path.isdir(file_storage_dir):
362 raise errors.OpPrereqError("The file storage directory '%s' is not"
363 " a directory." % file_storage_dir,
364 errors.ECODE_ENVIRON)
366 return file_storage_dir
369 def _PrepareFileBasedStorage(
370 enabled_disk_templates, file_storage_dir,
371 default_dir, file_disk_template,
372 init_fn=_InitFileStorageDir, acceptance_fn=None):
373 """Checks if a file-base storage type is enabled and inits the dir.
375 @type enabled_disk_templates: list of string
376 @param enabled_disk_templates: list of enabled disk templates
377 @type file_storage_dir: string
378 @param file_storage_dir: the file storage directory
379 @type default_dir: string
380 @param default_dir: default file storage directory when C{file_storage_dir}
382 @type file_disk_template: string
383 @param file_disk_template: a disk template whose storage type is 'ST_FILE'
385 @returns: the name of the actual file storage directory
388 assert (file_disk_template in
389 utils.storage.GetDiskTemplatesOfStorageType(constants.ST_FILE))
390 if file_storage_dir is None:
391 file_storage_dir = default_dir
392 if not acceptance_fn:
394 lambda path: filestorage.CheckFileStoragePathAcceptance(
395 path, exact_match_ok=True)
397 cluster.CheckFileStoragePathVsEnabledDiskTemplates(
398 logging.warning, file_storage_dir, enabled_disk_templates)
400 file_storage_enabled = file_disk_template in enabled_disk_templates
401 if file_storage_enabled:
403 acceptance_fn(file_storage_dir)
404 except errors.FileStoragePathError as e:
405 raise errors.OpPrereqError(str(e))
406 result_file_storage_dir = init_fn(file_storage_dir)
408 result_file_storage_dir = file_storage_dir
409 return result_file_storage_dir
412 def _PrepareFileStorage(
413 enabled_disk_templates, file_storage_dir, init_fn=_InitFileStorageDir,
415 """Checks if file storage is enabled and inits the dir.
417 @see: C{_PrepareFileBasedStorage}
420 return _PrepareFileBasedStorage(
421 enabled_disk_templates, file_storage_dir,
422 pathutils.DEFAULT_FILE_STORAGE_DIR, constants.DT_FILE,
423 init_fn=init_fn, acceptance_fn=acceptance_fn)
426 def _PrepareSharedFileStorage(
427 enabled_disk_templates, file_storage_dir, init_fn=_InitFileStorageDir,
429 """Checks if shared file storage is enabled and inits the dir.
431 @see: C{_PrepareFileBasedStorage}
434 return _PrepareFileBasedStorage(
435 enabled_disk_templates, file_storage_dir,
436 pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR, constants.DT_SHARED_FILE,
437 init_fn=init_fn, acceptance_fn=acceptance_fn)
440 def _InitCheckEnabledDiskTemplates(enabled_disk_templates):
441 """Checks the sanity of the enabled disk templates.
444 if not enabled_disk_templates:
445 raise errors.OpPrereqError("Enabled disk templates list must contain at"
446 " least one member", errors.ECODE_INVAL)
447 invalid_disk_templates = \
448 set(enabled_disk_templates) - constants.DISK_TEMPLATES
449 if invalid_disk_templates:
450 raise errors.OpPrereqError("Enabled disk templates list contains invalid"
451 " entries: %s" % invalid_disk_templates,
455 def InitCluster(cluster_name, mac_prefix, # pylint: disable=R0913, R0914
456 master_netmask, master_netdev, file_storage_dir,
457 shared_file_storage_dir, candidate_pool_size, secondary_ip=None,
458 vg_name=None, beparams=None, nicparams=None, ndparams=None,
459 hvparams=None, diskparams=None, enabled_hypervisors=None,
460 modify_etc_hosts=True, modify_ssh_setup=True,
461 maintain_node_health=False, drbd_helper=None, uid_pool=None,
462 default_iallocator=None, primary_ip_version=None, ipolicy=None,
463 prealloc_wipe_disks=False, use_external_mip_script=False,
464 hv_state=None, disk_state=None, enabled_disk_templates=None):
465 """Initialise the cluster.
467 @type candidate_pool_size: int
468 @param candidate_pool_size: master candidate pool size
469 @type enabled_disk_templates: list of string
470 @param enabled_disk_templates: list of disk_templates to be used in this
474 # TODO: complete the docstring
475 if config.ConfigWriter.IsCluster():
476 raise errors.OpPrereqError("Cluster is already initialised",
479 if not enabled_hypervisors:
480 raise errors.OpPrereqError("Enabled hypervisors list must contain at"
481 " least one member", errors.ECODE_INVAL)
482 invalid_hvs = set(enabled_hypervisors) - constants.HYPER_TYPES
484 raise errors.OpPrereqError("Enabled hypervisors contains invalid"
485 " entries: %s" % invalid_hvs,
488 _InitCheckEnabledDiskTemplates(enabled_disk_templates)
491 ipcls = netutils.IPAddress.GetClassFromIpVersion(primary_ip_version)
492 except errors.ProgrammerError:
493 raise errors.OpPrereqError("Invalid primary ip version: %d." %
494 primary_ip_version, errors.ECODE_INVAL)
496 hostname = netutils.GetHostname(family=ipcls.family)
497 if not ipcls.IsValid(hostname.ip):
498 raise errors.OpPrereqError("This host's IP (%s) is not a valid IPv%d"
499 " address." % (hostname.ip, primary_ip_version),
502 if ipcls.IsLoopback(hostname.ip):
503 raise errors.OpPrereqError("This host's IP (%s) resolves to a loopback"
504 " address. Please fix DNS or %s." %
505 (hostname.ip, pathutils.ETC_HOSTS),
506 errors.ECODE_ENVIRON)
508 if not ipcls.Own(hostname.ip):
509 raise errors.OpPrereqError("Inconsistency: this host's name resolves"
510 " to %s,\nbut this ip address does not"
511 " belong to this host" %
512 hostname.ip, errors.ECODE_ENVIRON)
514 clustername = netutils.GetHostname(name=cluster_name, family=ipcls.family)
516 if netutils.TcpPing(clustername.ip, constants.DEFAULT_NODED_PORT, timeout=5):
517 raise errors.OpPrereqError("Cluster IP already active",
518 errors.ECODE_NOTUNIQUE)
521 if primary_ip_version == constants.IP6_VERSION:
522 raise errors.OpPrereqError("When using a IPv6 primary address, a valid"
523 " IPv4 address must be given as secondary",
525 secondary_ip = hostname.ip
527 if not netutils.IP4Address.IsValid(secondary_ip):
528 raise errors.OpPrereqError("Secondary IP address (%s) has to be a valid"
529 " IPv4 address." % secondary_ip,
532 if not netutils.IP4Address.Own(secondary_ip):
533 raise errors.OpPrereqError("You gave %s as secondary IP,"
534 " but it does not belong to this host." %
535 secondary_ip, errors.ECODE_ENVIRON)
537 if master_netmask is not None:
538 if not ipcls.ValidateNetmask(master_netmask):
539 raise errors.OpPrereqError("CIDR netmask (%s) not valid for IPv%s " %
540 (master_netmask, primary_ip_version),
543 master_netmask = ipcls.iplen
546 # Check if volume group is valid
547 vgstatus = utils.CheckVolumeGroupSize(utils.ListVolumeGroups(), vg_name,
548 constants.MIN_VG_SIZE)
550 raise errors.OpPrereqError("Error: %s" % vgstatus, errors.ECODE_INVAL)
552 if drbd_helper is not None:
554 curr_helper = drbd.DRBD8.GetUsermodeHelper()
555 except errors.BlockDeviceError, err:
556 raise errors.OpPrereqError("Error while checking drbd helper"
557 " (specify --no-drbd-storage if you are not"
558 " using drbd): %s" % str(err),
559 errors.ECODE_ENVIRON)
560 if drbd_helper != curr_helper:
561 raise errors.OpPrereqError("Error: requiring %s as drbd helper but %s"
562 " is the current helper" % (drbd_helper,
566 logging.debug("Stopping daemons (if any are running)")
567 result = utils.RunCmd([pathutils.DAEMON_UTIL, "stop-all"])
569 raise errors.OpExecError("Could not stop daemons, command %s"
570 " had exitcode %s and error '%s'" %
571 (result.cmd, result.exit_code, result.output))
573 file_storage_dir = _PrepareFileStorage(enabled_disk_templates,
575 shared_file_storage_dir = _PrepareSharedFileStorage(enabled_disk_templates,
576 shared_file_storage_dir)
578 if not re.match("^[0-9a-z]{2}:[0-9a-z]{2}:[0-9a-z]{2}$", mac_prefix):
579 raise errors.OpPrereqError("Invalid mac prefix given '%s'" % mac_prefix,
582 result = utils.RunCmd(["ip", "link", "show", "dev", master_netdev])
584 raise errors.OpPrereqError("Invalid master netdev given (%s): '%s'" %
586 result.output.strip()), errors.ECODE_INVAL)
588 dirs = [(pathutils.RUN_DIR, constants.RUN_DIRS_MODE)]
589 utils.EnsureDirs(dirs)
591 objects.UpgradeBeParams(beparams)
592 utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
593 utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
595 objects.NIC.CheckParameterSyntax(nicparams)
597 full_ipolicy = objects.FillIPolicy(constants.IPOLICY_DEFAULTS, ipolicy)
599 if ndparams is not None:
600 utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
602 ndparams = dict(constants.NDC_DEFAULTS)
604 # This is ugly, as we modify the dict itself
605 # FIXME: Make utils.ForceDictType pure functional or write a wrapper
608 for hvname, hvs_data in hv_state.items():
609 utils.ForceDictType(hvs_data, constants.HVSTS_PARAMETER_TYPES)
610 hv_state[hvname] = objects.Cluster.SimpleFillHvState(hvs_data)
612 hv_state = dict((hvname, constants.HVST_DEFAULTS)
613 for hvname in enabled_hypervisors)
615 # FIXME: disk_state has no default values yet
617 for storage, ds_data in disk_state.items():
618 if storage not in constants.DS_VALID_TYPES:
619 raise errors.OpPrereqError("Invalid storage type in disk state: %s" %
620 storage, errors.ECODE_INVAL)
621 for ds_name, state in ds_data.items():
622 utils.ForceDictType(state, constants.DSS_PARAMETER_TYPES)
623 ds_data[ds_name] = objects.Cluster.SimpleFillDiskState(state)
625 # hvparams is a mapping of hypervisor->hvparams dict
626 for hv_name, hv_params in hvparams.iteritems():
627 utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
628 hv_class = hypervisor.GetHypervisor(hv_name)
629 hv_class.CheckParameterSyntax(hv_params)
631 # diskparams is a mapping of disk-template->diskparams dict
632 for template, dt_params in diskparams.items():
633 param_keys = set(dt_params.keys())
634 default_param_keys = set(constants.DISK_DT_DEFAULTS[template].keys())
635 if not (param_keys <= default_param_keys):
636 unknown_params = param_keys - default_param_keys
637 raise errors.OpPrereqError("Invalid parameters for disk template %s:"
639 utils.CommaJoin(unknown_params)),
641 utils.ForceDictType(dt_params, constants.DISK_DT_TYPES)
642 if template == constants.DT_DRBD8 and vg_name is not None:
643 # The default METAVG value is equal to the VG name set at init time,
645 dt_params[constants.DRBD_DEFAULT_METAVG] = vg_name
648 utils.VerifyDictOptions(diskparams, constants.DISK_DT_DEFAULTS)
649 except errors.OpPrereqError, err:
650 raise errors.OpPrereqError("While verify diskparam options: %s" % err,
653 # set up ssh config and /etc/hosts
654 sshline = utils.ReadFile(pathutils.SSH_HOST_RSA_PUB)
655 sshkey = sshline.split(" ")[1]
658 utils.AddHostToEtcHosts(hostname.name, hostname.ip)
663 if default_iallocator is not None:
664 alloc_script = utils.FindFile(default_iallocator,
665 constants.IALLOCATOR_SEARCH_PATH,
667 if alloc_script is None:
668 raise errors.OpPrereqError("Invalid default iallocator script '%s'"
669 " specified" % default_iallocator,
671 elif constants.HTOOLS:
672 # htools was enabled at build-time, we default to it
673 if utils.FindFile(constants.IALLOC_HAIL,
674 constants.IALLOCATOR_SEARCH_PATH,
676 default_iallocator = constants.IALLOC_HAIL
680 # init of cluster config file
681 cluster_config = objects.Cluster(
683 rsahostkeypub=sshkey,
684 highest_used_port=(constants.FIRST_DRBD_PORT - 1),
685 mac_prefix=mac_prefix,
686 volume_group_name=vg_name,
687 tcpudp_port_pool=set(),
688 master_ip=clustername.ip,
689 master_netmask=master_netmask,
690 master_netdev=master_netdev,
691 cluster_name=clustername.name,
692 file_storage_dir=file_storage_dir,
693 shared_file_storage_dir=shared_file_storage_dir,
694 enabled_hypervisors=enabled_hypervisors,
695 beparams={constants.PP_DEFAULT: beparams},
696 nicparams={constants.PP_DEFAULT: nicparams},
699 diskparams=diskparams,
700 candidate_pool_size=candidate_pool_size,
701 modify_etc_hosts=modify_etc_hosts,
702 modify_ssh_setup=modify_ssh_setup,
706 maintain_node_health=maintain_node_health,
707 drbd_usermode_helper=drbd_helper,
708 default_iallocator=default_iallocator,
709 primary_ip_family=ipcls.family,
710 prealloc_wipe_disks=prealloc_wipe_disks,
711 use_external_mip_script=use_external_mip_script,
712 ipolicy=full_ipolicy,
713 hv_state_static=hv_state,
714 disk_state_static=disk_state,
715 enabled_disk_templates=enabled_disk_templates,
717 master_node_config = objects.Node(name=hostname.name,
718 primary_ip=hostname.ip,
719 secondary_ip=secondary_ip,
721 master_candidate=True,
722 offline=False, drained=False,
723 ctime=now, mtime=now,
725 InitConfig(constants.CONFIG_VERSION, cluster_config, master_node_config)
726 cfg = config.ConfigWriter(offline=True)
727 ssh.WriteKnownHostsFile(cfg, pathutils.SSH_KNOWN_HOSTS_FILE)
728 cfg.Update(cfg.GetClusterInfo(), logging.error)
729 ssconf.WriteSsconfFiles(cfg.GetSsconfValues())
731 # set up the inter-node password and certificate
732 _InitGanetiServerSetup(hostname.name)
734 logging.debug("Starting daemons")
735 result = utils.RunCmd([pathutils.DAEMON_UTIL, "start-all"])
737 raise errors.OpExecError("Could not start daemons, command %s"
738 " had exitcode %s and error %s" %
739 (result.cmd, result.exit_code, result.output))
741 _WaitForMasterDaemon()
744 def InitConfig(version, cluster_config, master_node_config,
745 cfg_file=pathutils.CLUSTER_CONF_FILE):
746 """Create the initial cluster configuration.
748 It will contain the current node, which will also be the master
749 node, and no instances.
752 @param version: configuration version
753 @type cluster_config: L{objects.Cluster}
754 @param cluster_config: cluster configuration
755 @type master_node_config: L{objects.Node}
756 @param master_node_config: master node configuration
757 @type cfg_file: string
758 @param cfg_file: configuration file path
761 uuid_generator = config.TemporaryReservationManager()
762 cluster_config.uuid = uuid_generator.Generate([], utils.NewUUID,
764 master_node_config.uuid = uuid_generator.Generate([], utils.NewUUID,
766 cluster_config.master_node = master_node_config.uuid
768 master_node_config.uuid: master_node_config,
770 default_nodegroup = objects.NodeGroup(
771 uuid=uuid_generator.Generate([], utils.NewUUID, _INITCONF_ECID),
772 name=constants.INITIAL_NODE_GROUP_NAME,
773 members=[master_node_config.uuid],
777 default_nodegroup.uuid: default_nodegroup,
780 config_data = objects.ConfigData(version=version,
781 cluster=cluster_config,
782 nodegroups=nodegroups,
787 ctime=now, mtime=now)
788 utils.WriteFile(cfg_file,
789 data=serializer.Dump(config_data.ToDict()),
793 def FinalizeClusterDestroy(master_uuid):
794 """Execute the last steps of cluster destroy
796 This function shuts down all the daemons, completing the destroy
797 begun in cmdlib.LUDestroyOpcode.
800 cfg = config.ConfigWriter()
801 modify_ssh_setup = cfg.GetClusterInfo().modify_ssh_setup
802 runner = rpc.BootstrapRunner()
804 master_name = cfg.GetNodeName(master_uuid)
806 master_params = cfg.GetMasterNetworkParameters()
807 master_params.uuid = master_uuid
808 ems = cfg.GetUseExternalMipScript()
809 result = runner.call_node_deactivate_master_ip(master_name, master_params,
812 msg = result.fail_msg
814 logging.warning("Could not disable the master IP: %s", msg)
816 result = runner.call_node_stop_master(master_name)
817 msg = result.fail_msg
819 logging.warning("Could not disable the master role: %s", msg)
821 result = runner.call_node_leave_cluster(master_name, modify_ssh_setup)
822 msg = result.fail_msg
824 logging.warning("Could not shutdown the node daemon and cleanup"
825 " the node: %s", msg)
828 def SetupNodeDaemon(opts, cluster_name, node):
829 """Add a node to the cluster.
831 This function must be called before the actual opcode, and will ssh
832 to the remote node, copy the needed files, and start ganeti-noded,
833 allowing the master to do the rest via normal rpc calls.
835 @param cluster_name: the cluster name
836 @param node: the name of the new node
840 constants.NDS_CLUSTER_NAME: cluster_name,
841 constants.NDS_NODE_DAEMON_CERTIFICATE:
842 utils.ReadFile(pathutils.NODED_CERT_FILE),
843 constants.NDS_SSCONF: ssconf.SimpleStore().ReadAll(),
844 constants.NDS_START_NODE_DAEMON: True,
847 RunNodeSetupCmd(cluster_name, node, pathutils.NODE_DAEMON_SETUP,
848 opts.debug, opts.verbose,
849 True, opts.ssh_key_check, opts.ssh_key_check, data)
851 _WaitForNodeDaemon(node)
854 def MasterFailover(no_voting=False):
855 """Failover the master node.
857 This checks that we are not already the master, and will cause the
858 current master to cease being master, and the non-master to become
861 @type no_voting: boolean
862 @param no_voting: force the operation without remote nodes agreement
866 sstore = ssconf.SimpleStore()
868 old_master, new_master = ssconf.GetMasterAndMyself(sstore)
869 node_names = sstore.GetNodeList()
870 mc_list = sstore.GetMasterCandidates()
872 if old_master == new_master:
873 raise errors.OpPrereqError("This commands must be run on the node"
874 " where you want the new master to be."
875 " %s is already the master" %
876 old_master, errors.ECODE_INVAL)
878 if new_master not in mc_list:
879 mc_no_master = [name for name in mc_list if name != old_master]
880 raise errors.OpPrereqError("This node is not among the nodes marked"
881 " as master candidates. Only these nodes"
882 " can become masters. Current list of"
883 " master candidates is:\n"
884 "%s" % ("\n".join(mc_no_master)),
888 vote_list = GatherMasterVotes(node_names)
891 voted_master = vote_list[0][0]
892 if voted_master is None:
893 raise errors.OpPrereqError("Cluster is inconsistent, most nodes did"
894 " not respond.", errors.ECODE_ENVIRON)
895 elif voted_master != old_master:
896 raise errors.OpPrereqError("I have a wrong configuration, I believe"
897 " the master is %s but the other nodes"
898 " voted %s. Please resync the configuration"
900 (old_master, voted_master),
906 logging.info("Setting master to %s, old master: %s", new_master, old_master)
909 # instantiate a real config writer, as we now know we have the
911 cfg = config.ConfigWriter(accept_foreign=True)
913 old_master_node = cfg.GetNodeInfoByName(old_master)
914 if old_master_node is None:
915 raise errors.OpPrereqError("Could not find old master node '%s' in"
916 " cluster configuration." % old_master,
919 cluster_info = cfg.GetClusterInfo()
920 new_master_node = cfg.GetNodeInfoByName(new_master)
921 if new_master_node is None:
922 raise errors.OpPrereqError("Could not find new master node '%s' in"
923 " cluster configuration." % new_master,
926 cluster_info.master_node = new_master_node.uuid
927 # this will also regenerate the ssconf files, since we updated the
929 cfg.Update(cluster_info, logging.error)
930 except errors.ConfigurationError, err:
931 logging.error("Error while trying to set the new master: %s",
935 # if cfg.Update worked, then it means the old master daemon won't be
936 # able now to write its own config file (we rely on locking in both
937 # backend.UploadFile() and ConfigWriter._Write(); hence the next
938 # step is to kill the old master
940 logging.info("Stopping the master daemon on node %s", old_master)
942 runner = rpc.BootstrapRunner()
943 master_params = cfg.GetMasterNetworkParameters()
944 master_params.uuid = old_master_node.uuid
945 ems = cfg.GetUseExternalMipScript()
946 result = runner.call_node_deactivate_master_ip(old_master,
949 msg = result.fail_msg
951 logging.warning("Could not disable the master IP: %s", msg)
953 result = runner.call_node_stop_master(old_master)
954 msg = result.fail_msg
956 logging.error("Could not disable the master role on the old master"
957 " %s, please disable manually: %s", old_master, msg)
959 logging.info("Checking master IP non-reachability...")
961 master_ip = sstore.GetMasterIP()
964 # Here we have a phase where no master should be running
966 if netutils.TcpPing(master_ip, constants.DEFAULT_NODED_PORT):
967 raise utils.RetryAgain()
970 utils.Retry(_check_ip, (1, 1.5, 5), total_timeout)
971 except utils.RetryTimeout:
972 logging.warning("The master IP is still reachable after %s seconds,"
973 " continuing but activating the master on the current"
974 " node will probably fail", total_timeout)
976 if jstore.CheckDrainFlag():
977 logging.info("Undraining job queue")
978 jstore.SetDrainFlag(False)
980 logging.info("Starting the master daemons on the new master")
982 result = rpc.BootstrapRunner().call_node_start_master_daemons(new_master,
984 msg = result.fail_msg
986 logging.error("Could not start the master role on the new master"
987 " %s, please check: %s", new_master, msg)
990 logging.info("Master failed over from %s to %s", old_master, new_master)
995 """Returns the current master node.
997 This is a separate function in bootstrap since it's needed by
998 gnt-cluster, and instead of importing directly ssconf, it's better
999 to abstract it in bootstrap, where we do use ssconf in other
1003 sstore = ssconf.SimpleStore()
1005 old_master, _ = ssconf.GetMasterAndMyself(sstore)
1010 def GatherMasterVotes(node_names):
1011 """Check the agreement on who is the master.
1013 This function will return a list of (node, number of votes), ordered
1014 by the number of votes. Errors will be denoted by the key 'None'.
1016 Note that the sum of votes is the number of nodes this machine
1017 knows, whereas the number of entries in the list could be different
1018 (if some nodes vote for another master).
1020 We remove ourselves from the list since we know that (bugs aside)
1021 since we use the same source for configuration information for both
1022 backend and boostrap, we'll always vote for ourselves.
1024 @type node_names: list
1025 @param node_names: the list of nodes to query for master info; the current
1026 node will be removed if it is in the list
1028 @return: list of (node, votes)
1031 myself = netutils.Hostname.GetSysName()
1033 node_names.remove(myself)
1037 # no nodes left (eventually after removing myself)
1039 results = rpc.BootstrapRunner().call_master_info(node_names)
1040 if not isinstance(results, dict):
1041 # this should not happen (unless internal error in rpc)
1042 logging.critical("Can't complete rpc call, aborting master startup")
1043 return [(None, len(node_names))]
1045 for node_name in results:
1046 nres = results[node_name]
1051 logging.warning("Error contacting node %s: %s", node_name, msg)
1053 # for now we accept both length 3, 4 and 5 (data[3] is primary ip version
1054 # and data[4] is the master netmask)
1055 elif not isinstance(data, (tuple, list)) or len(data) < 3:
1056 logging.warning("Invalid data received from node %s: %s",
1060 if None not in votes:
1064 master_node = data[2]
1065 if master_node not in votes:
1066 votes[master_node] = 0
1067 votes[master_node] += 1
1069 vote_list = [v for v in votes.items()]
1070 # sort first on number of votes then on name, since we want None
1071 # sorted later if we have the half of the nodes not responding, and
1072 # half voting all for the same master
1073 vote_list.sort(key=lambda x: (x[1], x[0]), reverse=True)