4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Cluster related commands"""
23 # pylint: disable=W0401,W0613,W0614,C0103
24 # W0401: Wildcard import ganeti.cli
25 # W0613: Unused argument, since all functions follow the same API
26 # W0614: Unused import %s from wildcard import (since we need cli)
27 # C0103: Invalid name gnt-cluster
34 from ganeti.cli import *
35 from ganeti import opcodes
36 from ganeti import constants
37 from ganeti import errors
38 from ganeti import utils
39 from ganeti import bootstrap
40 from ganeti import ssh
41 from ganeti import objects
42 from ganeti import uidpool
43 from ganeti import compat
44 from ganeti import netutils
47 ON_OPT = cli_option("--on", default=False,
48 action="store_true", dest="on",
49 help="Recover from an EPO")
51 GROUPS_OPT = cli_option("--groups", default=False,
52 action="store_true", dest="groups",
53 help="Arguments are node groups instead of nodes")
55 _EPO_PING_INTERVAL = 30 # 30 seconds between pings
56 _EPO_PING_TIMEOUT = 1 # 1 second
57 _EPO_REACHABLE_TIMEOUT = 15 * 60 # 15 minutes
61 def InitCluster(opts, args):
62 """Initialize the cluster.
64 @param opts: the command line options selected by the user
66 @param args: should contain only one element, the desired
69 @return: the desired exit code
72 if not opts.lvm_storage and opts.vg_name:
73 ToStderr("Options --no-lvm-storage and --vg-name conflict.")
76 vg_name = opts.vg_name
77 if opts.lvm_storage and not opts.vg_name:
78 vg_name = constants.DEFAULT_VG
80 if not opts.drbd_storage and opts.drbd_helper:
81 ToStderr("Options --no-drbd-storage and --drbd-usermode-helper conflict.")
84 drbd_helper = opts.drbd_helper
85 if opts.drbd_storage and not opts.drbd_helper:
86 drbd_helper = constants.DEFAULT_DRBD_HELPER
88 master_netdev = opts.master_netdev
89 if master_netdev is None:
90 master_netdev = constants.DEFAULT_BRIDGE
92 hvlist = opts.enabled_hypervisors
94 hvlist = constants.DEFAULT_ENABLED_HYPERVISOR
95 hvlist = hvlist.split(",")
97 hvparams = dict(opts.hvparams)
98 beparams = opts.beparams
99 nicparams = opts.nicparams
101 # prepare beparams dict
102 beparams = objects.FillDict(constants.BEC_DEFAULTS, beparams)
103 utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
105 # prepare nicparams dict
106 nicparams = objects.FillDict(constants.NICC_DEFAULTS, nicparams)
107 utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
109 # prepare ndparams dict
110 if opts.ndparams is None:
111 ndparams = dict(constants.NDC_DEFAULTS)
113 ndparams = objects.FillDict(constants.NDC_DEFAULTS, opts.ndparams)
114 utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
116 # prepare hvparams dict
117 for hv in constants.HYPER_TYPES:
118 if hv not in hvparams:
120 hvparams[hv] = objects.FillDict(constants.HVC_DEFAULTS[hv], hvparams[hv])
121 utils.ForceDictType(hvparams[hv], constants.HVS_PARAMETER_TYPES)
123 if opts.candidate_pool_size is None:
124 opts.candidate_pool_size = constants.MASTER_POOL_SIZE_DEFAULT
126 if opts.mac_prefix is None:
127 opts.mac_prefix = constants.DEFAULT_MAC_PREFIX
129 uid_pool = opts.uid_pool
130 if uid_pool is not None:
131 uid_pool = uidpool.ParseUidPool(uid_pool)
133 if opts.prealloc_wipe_disks is None:
134 opts.prealloc_wipe_disks = False
137 primary_ip_version = int(opts.primary_ip_version)
138 except (ValueError, TypeError), err:
139 ToStderr("Invalid primary ip version value: %s" % str(err))
142 bootstrap.InitCluster(cluster_name=args[0],
143 secondary_ip=opts.secondary_ip,
145 mac_prefix=opts.mac_prefix,
146 master_netdev=master_netdev,
147 file_storage_dir=opts.file_storage_dir,
148 shared_file_storage_dir=opts.shared_file_storage_dir,
149 enabled_hypervisors=hvlist,
154 candidate_pool_size=opts.candidate_pool_size,
155 modify_etc_hosts=opts.modify_etc_hosts,
156 modify_ssh_setup=opts.modify_ssh_setup,
157 maintain_node_health=opts.maintain_node_health,
158 drbd_helper=drbd_helper,
160 default_iallocator=opts.default_iallocator,
161 primary_ip_version=primary_ip_version,
162 prealloc_wipe_disks=opts.prealloc_wipe_disks,
164 op = opcodes.OpClusterPostInit()
165 SubmitOpCode(op, opts=opts)
170 def DestroyCluster(opts, args):
171 """Destroy the cluster.
173 @param opts: the command line options selected by the user
175 @param args: should be an empty list
177 @return: the desired exit code
180 if not opts.yes_do_it:
181 ToStderr("Destroying a cluster is irreversible. If you really want"
182 " destroy this cluster, supply the --yes-do-it option.")
185 op = opcodes.OpClusterDestroy()
186 master = SubmitOpCode(op, opts=opts)
187 # if we reached this, the opcode didn't fail; we can proceed to
188 # shutdown all the daemons
189 bootstrap.FinalizeClusterDestroy(master)
193 def RenameCluster(opts, args):
194 """Rename the cluster.
196 @param opts: the command line options selected by the user
198 @param args: should contain only one element, the new cluster name
200 @return: the desired exit code
205 (cluster_name, ) = cl.QueryConfigValues(["cluster_name"])
209 usertext = ("This will rename the cluster from '%s' to '%s'. If you are"
210 " connected over the network to the cluster name, the"
211 " operation is very dangerous as the IP address will be"
212 " removed from the node and the change may not go through."
213 " Continue?") % (cluster_name, new_name)
214 if not AskUser(usertext):
217 op = opcodes.OpClusterRename(name=new_name)
218 result = SubmitOpCode(op, opts=opts, cl=cl)
221 ToStdout("Cluster renamed from '%s' to '%s'", cluster_name, result)
226 def ActivateMasterIp(opts, args):
227 """Activates the master IP.
230 op = opcodes.OpClusterActivateMasterIp()
235 def DeactivateMasterIp(opts, args):
236 """Deactivates the master IP.
240 usertext = ("This will disable the master IP. All the open connections to"
241 " the master IP will be closed. To reach the master you will"
242 " need to use its node IP."
244 if not AskUser(usertext):
247 op = opcodes.OpClusterDeactivateMasterIp()
252 def RedistributeConfig(opts, args):
253 """Forces push of the cluster configuration.
255 @param opts: the command line options selected by the user
257 @param args: empty list
259 @return: the desired exit code
262 op = opcodes.OpClusterRedistConf()
263 SubmitOrSend(op, opts)
267 def ShowClusterVersion(opts, args):
268 """Write version of ganeti software to the standard output.
270 @param opts: the command line options selected by the user
272 @param args: should be an empty list
274 @return: the desired exit code
278 result = cl.QueryClusterInfo()
279 ToStdout("Software version: %s", result["software_version"])
280 ToStdout("Internode protocol: %s", result["protocol_version"])
281 ToStdout("Configuration format: %s", result["config_version"])
282 ToStdout("OS api version: %s", result["os_api_version"])
283 ToStdout("Export interface: %s", result["export_version"])
287 def ShowClusterMaster(opts, args):
288 """Write name of master node to the standard output.
290 @param opts: the command line options selected by the user
292 @param args: should be an empty list
294 @return: the desired exit code
297 master = bootstrap.GetMaster()
302 def _PrintGroupedParams(paramsdict, level=1, roman=False):
303 """Print Grouped parameters (be, nic, disk) by group.
305 @type paramsdict: dict of dicts
306 @param paramsdict: {group: {param: value, ...}, ...}
308 @param level: Level of indention
312 for item, val in sorted(paramsdict.items()):
313 if isinstance(val, dict):
314 ToStdout("%s- %s:", indent, item)
315 _PrintGroupedParams(val, level=level + 1, roman=roman)
316 elif roman and isinstance(val, int):
317 ToStdout("%s %s: %s", indent, item, compat.TryToRoman(val))
319 ToStdout("%s %s: %s", indent, item, val)
322 def ShowClusterConfig(opts, args):
323 """Shows cluster information.
325 @param opts: the command line options selected by the user
327 @param args: should be an empty list
329 @return: the desired exit code
333 result = cl.QueryClusterInfo()
335 ToStdout("Cluster name: %s", result["name"])
336 ToStdout("Cluster UUID: %s", result["uuid"])
338 ToStdout("Creation time: %s", utils.FormatTime(result["ctime"]))
339 ToStdout("Modification time: %s", utils.FormatTime(result["mtime"]))
341 ToStdout("Master node: %s", result["master"])
343 ToStdout("Architecture (this node): %s (%s)",
344 result["architecture"][0], result["architecture"][1])
347 tags = utils.CommaJoin(utils.NiceSort(result["tags"]))
351 ToStdout("Tags: %s", tags)
353 ToStdout("Default hypervisor: %s", result["default_hypervisor"])
354 ToStdout("Enabled hypervisors: %s",
355 utils.CommaJoin(result["enabled_hypervisors"]))
357 ToStdout("Hypervisor parameters:")
358 _PrintGroupedParams(result["hvparams"])
360 ToStdout("OS-specific hypervisor parameters:")
361 _PrintGroupedParams(result["os_hvp"])
363 ToStdout("OS parameters:")
364 _PrintGroupedParams(result["osparams"])
366 ToStdout("Hidden OSes: %s", utils.CommaJoin(result["hidden_os"]))
367 ToStdout("Blacklisted OSes: %s", utils.CommaJoin(result["blacklisted_os"]))
369 ToStdout("Cluster parameters:")
370 ToStdout(" - candidate pool size: %s",
371 compat.TryToRoman(result["candidate_pool_size"],
372 convert=opts.roman_integers))
373 ToStdout(" - master netdev: %s", result["master_netdev"])
374 ToStdout(" - lvm volume group: %s", result["volume_group_name"])
375 if result["reserved_lvs"]:
376 reserved_lvs = utils.CommaJoin(result["reserved_lvs"])
378 reserved_lvs = "(none)"
379 ToStdout(" - lvm reserved volumes: %s", reserved_lvs)
380 ToStdout(" - drbd usermode helper: %s", result["drbd_usermode_helper"])
381 ToStdout(" - file storage path: %s", result["file_storage_dir"])
382 ToStdout(" - shared file storage path: %s",
383 result["shared_file_storage_dir"])
384 ToStdout(" - maintenance of node health: %s",
385 result["maintain_node_health"])
386 ToStdout(" - uid pool: %s",
387 uidpool.FormatUidPool(result["uid_pool"],
388 roman=opts.roman_integers))
389 ToStdout(" - default instance allocator: %s", result["default_iallocator"])
390 ToStdout(" - primary ip version: %d", result["primary_ip_version"])
391 ToStdout(" - preallocation wipe disks: %s", result["prealloc_wipe_disks"])
392 ToStdout(" - OS search path: %s", utils.CommaJoin(constants.OS_SEARCH_PATH))
394 ToStdout("Default node parameters:")
395 _PrintGroupedParams(result["ndparams"], roman=opts.roman_integers)
397 ToStdout("Default instance parameters:")
398 _PrintGroupedParams(result["beparams"], roman=opts.roman_integers)
400 ToStdout("Default nic parameters:")
401 _PrintGroupedParams(result["nicparams"], roman=opts.roman_integers)
406 def ClusterCopyFile(opts, args):
407 """Copy a file from master to some nodes.
409 @param opts: the command line options selected by the user
411 @param args: should contain only one element, the path of
412 the file to be copied
414 @return: the desired exit code
418 if not os.path.exists(filename):
419 raise errors.OpPrereqError("No such filename '%s'" % filename,
424 cluster_name = cl.QueryConfigValues(["cluster_name"])[0]
426 results = GetOnlineNodes(nodes=opts.nodes, cl=cl, filter_master=True,
427 secondary_ips=opts.use_replication_network,
428 nodegroup=opts.nodegroup)
430 srun = ssh.SshRunner(cluster_name=cluster_name)
432 if not srun.CopyFileToNode(node, filename):
433 ToStderr("Copy of file %s to node %s failed", filename, node)
438 def RunClusterCommand(opts, args):
439 """Run a command on some nodes.
441 @param opts: the command line options selected by the user
443 @param args: should contain the command to be run and its arguments
445 @return: the desired exit code
450 command = " ".join(args)
452 nodes = GetOnlineNodes(nodes=opts.nodes, cl=cl, nodegroup=opts.nodegroup)
454 cluster_name, master_node = cl.QueryConfigValues(["cluster_name",
457 srun = ssh.SshRunner(cluster_name=cluster_name)
459 # Make sure master node is at list end
460 if master_node in nodes:
461 nodes.remove(master_node)
462 nodes.append(master_node)
465 result = srun.Run(name, "root", command)
466 ToStdout("------------------------------------------------")
467 ToStdout("node: %s", name)
468 ToStdout("%s", result.output)
469 ToStdout("return code = %s", result.exit_code)
474 def VerifyCluster(opts, args):
475 """Verify integrity of cluster, performing various test on nodes.
477 @param opts: the command line options selected by the user
479 @param args: should be an empty list
481 @return: the desired exit code
486 if opts.skip_nplusone_mem:
487 skip_checks.append(constants.VERIFY_NPLUSONE_MEM)
491 op = opcodes.OpClusterVerify(verbose=opts.verbose,
492 error_codes=opts.error_codes,
493 debug_simulate_errors=opts.simulate_errors,
494 skip_checks=skip_checks,
495 group_name=opts.nodegroup)
496 result = SubmitOpCode(op, cl=cl, opts=opts)
498 # Keep track of submitted jobs
499 jex = JobExecutor(cl=cl, opts=opts)
501 for (status, job_id) in result[constants.JOB_IDS_KEY]:
502 jex.AddJobId(None, status, job_id)
504 results = jex.GetResults()
506 (bad_jobs, bad_results) = \
508 # Convert iterators to lists
511 map(compat.partial(itertools.ifilterfalse, bool),
512 # Convert result to booleans in a tuple
513 zip(*((job_success, len(op_results) == 1 and op_results[0])
514 for (job_success, op_results) in results)))))
516 if bad_jobs == 0 and bad_results == 0:
517 rcode = constants.EXIT_SUCCESS
519 rcode = constants.EXIT_FAILURE
521 ToStdout("%s job(s) failed while verifying the cluster.", bad_jobs)
526 def VerifyDisks(opts, args):
527 """Verify integrity of cluster disks.
529 @param opts: the command line options selected by the user
531 @param args: should be an empty list
533 @return: the desired exit code
538 op = opcodes.OpClusterVerifyDisks()
540 result = SubmitOpCode(op, cl=cl, opts=opts)
542 # Keep track of submitted jobs
543 jex = JobExecutor(cl=cl, opts=opts)
545 for (status, job_id) in result[constants.JOB_IDS_KEY]:
546 jex.AddJobId(None, status, job_id)
548 retcode = constants.EXIT_SUCCESS
550 for (status, result) in jex.GetResults():
552 ToStdout("Job failed: %s", result)
555 ((bad_nodes, instances, missing), ) = result
557 for node, text in bad_nodes.items():
558 ToStdout("Error gathering data on node %s: %s",
559 node, utils.SafeEncode(text[-400:]))
560 retcode = constants.EXIT_FAILURE
561 ToStdout("You need to fix these nodes first before fixing instances")
563 for iname in instances:
566 op = opcodes.OpInstanceActivateDisks(instance_name=iname)
568 ToStdout("Activating disks for instance '%s'", iname)
569 SubmitOpCode(op, opts=opts, cl=cl)
570 except errors.GenericError, err:
571 nret, msg = FormatError(err)
573 ToStderr("Error activating disks for instance %s: %s", iname, msg)
576 for iname, ival in missing.iteritems():
577 all_missing = compat.all(x[0] in bad_nodes for x in ival)
579 ToStdout("Instance %s cannot be verified as it lives on"
580 " broken nodes", iname)
582 ToStdout("Instance %s has missing logical volumes:", iname)
584 for node, vol in ival:
585 if node in bad_nodes:
586 ToStdout("\tbroken node %s /dev/%s", node, vol)
588 ToStdout("\t%s /dev/%s", node, vol)
590 ToStdout("You need to replace or recreate disks for all the above"
591 " instances if this message persists after fixing broken nodes.")
592 retcode = constants.EXIT_FAILURE
597 def RepairDiskSizes(opts, args):
598 """Verify sizes of cluster disks.
600 @param opts: the command line options selected by the user
602 @param args: optional list of instances to restrict check to
604 @return: the desired exit code
607 op = opcodes.OpClusterRepairDiskSizes(instances=args)
608 SubmitOpCode(op, opts=opts)
612 def MasterFailover(opts, args):
613 """Failover the master node.
615 This command, when run on a non-master node, will cause the current
616 master to cease being master, and the non-master to become new
619 @param opts: the command line options selected by the user
621 @param args: should be an empty list
623 @return: the desired exit code
627 usertext = ("This will perform the failover even if most other nodes"
628 " are down, or if this node is outdated. This is dangerous"
629 " as it can lead to a non-consistent cluster. Check the"
630 " gnt-cluster(8) man page before proceeding. Continue?")
631 if not AskUser(usertext):
634 return bootstrap.MasterFailover(no_voting=opts.no_voting)
637 def MasterPing(opts, args):
638 """Checks if the master is alive.
640 @param opts: the command line options selected by the user
642 @param args: should be an empty list
644 @return: the desired exit code
649 cl.QueryClusterInfo()
651 except Exception: # pylint: disable=W0703
655 def SearchTags(opts, args):
656 """Searches the tags on all the cluster.
658 @param opts: the command line options selected by the user
660 @param args: should contain only one element, the tag pattern
662 @return: the desired exit code
665 op = opcodes.OpTagsSearch(pattern=args[0])
666 result = SubmitOpCode(op, opts=opts)
669 result = list(result)
671 for path, tag in result:
672 ToStdout("%s %s", path, tag)
675 def _ReadAndVerifyCert(cert_filename, verify_private_key=False):
676 """Reads and verifies an X509 certificate.
678 @type cert_filename: string
679 @param cert_filename: the path of the file containing the certificate to
680 verify encoded in PEM format
681 @type verify_private_key: bool
682 @param verify_private_key: whether to verify the private key in addition to
683 the public certificate
685 @return: a string containing the PEM-encoded certificate.
689 pem = utils.ReadFile(cert_filename)
691 raise errors.X509CertError(cert_filename,
692 "Unable to read certificate: %s" % str(err))
695 OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem)
696 except Exception, err:
697 raise errors.X509CertError(cert_filename,
698 "Unable to load certificate: %s" % str(err))
700 if verify_private_key:
702 OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, pem)
703 except Exception, err:
704 raise errors.X509CertError(cert_filename,
705 "Unable to load private key: %s" % str(err))
710 def _RenewCrypto(new_cluster_cert, new_rapi_cert, #pylint: disable=R0911
711 rapi_cert_filename, new_spice_cert, spice_cert_filename,
712 spice_cacert_filename, new_confd_hmac_key, new_cds,
713 cds_filename, force):
714 """Renews cluster certificates, keys and secrets.
716 @type new_cluster_cert: bool
717 @param new_cluster_cert: Whether to generate a new cluster certificate
718 @type new_rapi_cert: bool
719 @param new_rapi_cert: Whether to generate a new RAPI certificate
720 @type rapi_cert_filename: string
721 @param rapi_cert_filename: Path to file containing new RAPI certificate
722 @type new_spice_cert: bool
723 @param new_spice_cert: Whether to generate a new SPICE certificate
724 @type spice_cert_filename: string
725 @param spice_cert_filename: Path to file containing new SPICE certificate
726 @type spice_cacert_filename: string
727 @param spice_cacert_filename: Path to file containing the certificate of the
728 CA that signed the SPICE certificate
729 @type new_confd_hmac_key: bool
730 @param new_confd_hmac_key: Whether to generate a new HMAC key
732 @param new_cds: Whether to generate a new cluster domain secret
733 @type cds_filename: string
734 @param cds_filename: Path to file containing new cluster domain secret
736 @param force: Whether to ask user for confirmation
739 if new_rapi_cert and rapi_cert_filename:
740 ToStderr("Only one of the --new-rapi-certificate and --rapi-certificate"
741 " options can be specified at the same time.")
744 if new_cds and cds_filename:
745 ToStderr("Only one of the --new-cluster-domain-secret and"
746 " --cluster-domain-secret options can be specified at"
750 if new_spice_cert and (spice_cert_filename or spice_cacert_filename):
751 ToStderr("When using --new-spice-certificate, the --spice-certificate"
752 " and --spice-ca-certificate must not be used.")
755 if bool(spice_cacert_filename) ^ bool(spice_cert_filename):
756 ToStderr("Both --spice-certificate and --spice-ca-certificate must be"
760 rapi_cert_pem, spice_cert_pem, spice_cacert_pem = (None, None, None)
762 if rapi_cert_filename:
763 rapi_cert_pem = _ReadAndVerifyCert(rapi_cert_filename, True)
764 if spice_cert_filename:
765 spice_cert_pem = _ReadAndVerifyCert(spice_cert_filename, True)
766 spice_cacert_pem = _ReadAndVerifyCert(spice_cacert_filename)
767 except errors.X509CertError, err:
768 ToStderr("Unable to load X509 certificate from %s: %s", err[0], err[1])
773 cds = utils.ReadFile(cds_filename)
774 except Exception, err: # pylint: disable=W0703
775 ToStderr("Can't load new cluster domain secret from %s: %s" %
776 (cds_filename, str(err)))
782 usertext = ("This requires all daemons on all nodes to be restarted and"
783 " may take some time. Continue?")
784 if not AskUser(usertext):
787 def _RenewCryptoInner(ctx):
788 ctx.feedback_fn("Updating certificates and keys")
789 bootstrap.GenerateClusterCrypto(new_cluster_cert,
794 rapi_cert_pem=rapi_cert_pem,
795 spice_cert_pem=spice_cert_pem,
796 spice_cacert_pem=spice_cacert_pem,
802 files_to_copy.append(constants.NODED_CERT_FILE)
804 if new_rapi_cert or rapi_cert_pem:
805 files_to_copy.append(constants.RAPI_CERT_FILE)
807 if new_spice_cert or spice_cert_pem:
808 files_to_copy.append(constants.SPICE_CERT_FILE)
809 files_to_copy.append(constants.SPICE_CACERT_FILE)
811 if new_confd_hmac_key:
812 files_to_copy.append(constants.CONFD_HMAC_KEY)
815 files_to_copy.append(constants.CLUSTER_DOMAIN_SECRET_FILE)
818 for node_name in ctx.nonmaster_nodes:
819 ctx.feedback_fn("Copying %s to %s" %
820 (", ".join(files_to_copy), node_name))
821 for file_name in files_to_copy:
822 ctx.ssh.CopyFileToNode(node_name, file_name)
824 RunWhileClusterStopped(ToStdout, _RenewCryptoInner)
826 ToStdout("All requested certificates and keys have been replaced."
827 " Running \"gnt-cluster verify\" now is recommended.")
832 def RenewCrypto(opts, args):
833 """Renews cluster certificates, keys and secrets.
836 return _RenewCrypto(opts.new_cluster_cert,
842 opts.new_confd_hmac_key,
843 opts.new_cluster_domain_secret,
844 opts.cluster_domain_secret,
848 def SetClusterParams(opts, args):
849 """Modify the cluster.
851 @param opts: the command line options selected by the user
853 @param args: should be an empty list
855 @return: the desired exit code
858 if not (not opts.lvm_storage or opts.vg_name or
859 not opts.drbd_storage or opts.drbd_helper or
860 opts.enabled_hypervisors or opts.hvparams or
861 opts.beparams or opts.nicparams or opts.ndparams or
862 opts.candidate_pool_size is not None or
863 opts.uid_pool is not None or
864 opts.maintain_node_health is not None or
865 opts.add_uids is not None or
866 opts.remove_uids is not None or
867 opts.default_iallocator is not None or
868 opts.reserved_lvs is not None or
869 opts.master_netdev is not None or
870 opts.prealloc_wipe_disks is not None):
871 ToStderr("Please give at least one of the parameters.")
874 vg_name = opts.vg_name
875 if not opts.lvm_storage and opts.vg_name:
876 ToStderr("Options --no-lvm-storage and --vg-name conflict.")
879 if not opts.lvm_storage:
882 drbd_helper = opts.drbd_helper
883 if not opts.drbd_storage and opts.drbd_helper:
884 ToStderr("Options --no-drbd-storage and --drbd-usermode-helper conflict.")
887 if not opts.drbd_storage:
890 hvlist = opts.enabled_hypervisors
891 if hvlist is not None:
892 hvlist = hvlist.split(",")
894 # a list of (name, dict) we can pass directly to dict() (or [])
895 hvparams = dict(opts.hvparams)
896 for hv_params in hvparams.values():
897 utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES)
899 beparams = opts.beparams
900 utils.ForceDictType(beparams, constants.BES_PARAMETER_TYPES)
902 nicparams = opts.nicparams
903 utils.ForceDictType(nicparams, constants.NICS_PARAMETER_TYPES)
905 ndparams = opts.ndparams
906 if ndparams is not None:
907 utils.ForceDictType(ndparams, constants.NDS_PARAMETER_TYPES)
909 mnh = opts.maintain_node_health
911 uid_pool = opts.uid_pool
912 if uid_pool is not None:
913 uid_pool = uidpool.ParseUidPool(uid_pool)
915 add_uids = opts.add_uids
916 if add_uids is not None:
917 add_uids = uidpool.ParseUidPool(add_uids)
919 remove_uids = opts.remove_uids
920 if remove_uids is not None:
921 remove_uids = uidpool.ParseUidPool(remove_uids)
923 if opts.reserved_lvs is not None:
924 if opts.reserved_lvs == "":
925 opts.reserved_lvs = []
927 opts.reserved_lvs = utils.UnescapeAndSplit(opts.reserved_lvs, sep=",")
929 op = opcodes.OpClusterSetParams(vg_name=vg_name,
930 drbd_helper=drbd_helper,
931 enabled_hypervisors=hvlist,
937 candidate_pool_size=opts.candidate_pool_size,
938 maintain_node_health=mnh,
941 remove_uids=remove_uids,
942 default_iallocator=opts.default_iallocator,
943 prealloc_wipe_disks=opts.prealloc_wipe_disks,
944 master_netdev=opts.master_netdev,
945 reserved_lvs=opts.reserved_lvs)
946 SubmitOpCode(op, opts=opts)
950 def QueueOps(opts, args):
953 @param opts: the command line options selected by the user
955 @param args: should contain only one element, the subcommand
957 @return: the desired exit code
962 if command in ("drain", "undrain"):
963 drain_flag = command == "drain"
964 client.SetQueueDrainFlag(drain_flag)
965 elif command == "info":
966 result = client.QueryConfigValues(["drain_flag"])
971 ToStdout("The drain flag is %s" % val)
973 raise errors.OpPrereqError("Command '%s' is not valid." % command,
979 def _ShowWatcherPause(until):
980 if until is None or until < time.time():
981 ToStdout("The watcher is not paused.")
983 ToStdout("The watcher is paused until %s.", time.ctime(until))
986 def WatcherOps(opts, args):
987 """Watcher operations.
989 @param opts: the command line options selected by the user
991 @param args: should contain only one element, the subcommand
993 @return: the desired exit code
999 if command == "continue":
1000 client.SetWatcherPause(None)
1001 ToStdout("The watcher is no longer paused.")
1003 elif command == "pause":
1005 raise errors.OpPrereqError("Missing pause duration", errors.ECODE_INVAL)
1007 result = client.SetWatcherPause(time.time() + ParseTimespec(args[1]))
1008 _ShowWatcherPause(result)
1010 elif command == "info":
1011 result = client.QueryConfigValues(["watcher_pause"])
1012 _ShowWatcherPause(result[0])
1015 raise errors.OpPrereqError("Command '%s' is not valid." % command,
1021 def _OobPower(opts, node_list, power):
1022 """Puts the node in the list to desired power state.
1024 @param opts: The command line options selected by the user
1025 @param node_list: The list of nodes to operate on
1026 @param power: True if they should be powered on, False otherwise
1027 @return: The success of the operation (none failed)
1031 command = constants.OOB_POWER_ON
1033 command = constants.OOB_POWER_OFF
1035 op = opcodes.OpOobCommand(node_names=node_list,
1038 timeout=opts.oob_timeout,
1039 power_delay=opts.power_delay)
1040 result = SubmitOpCode(op, opts=opts)
1042 for node_result in result:
1043 (node_tuple, data_tuple) = node_result
1044 (_, node_name) = node_tuple
1045 (data_status, _) = data_tuple
1046 if data_status != constants.RS_NORMAL:
1047 assert data_status != constants.RS_UNAVAIL
1049 ToStderr("There was a problem changing power for %s, please investigate",
1058 def _InstanceStart(opts, inst_list, start):
1059 """Puts the instances in the list to desired state.
1061 @param opts: The command line options selected by the user
1062 @param inst_list: The list of instances to operate on
1063 @param start: True if they should be started, False for shutdown
1064 @return: The success of the operation (none failed)
1068 opcls = opcodes.OpInstanceStartup
1069 text_submit, text_success, text_failed = ("startup", "started", "starting")
1071 opcls = compat.partial(opcodes.OpInstanceShutdown,
1072 timeout=opts.shutdown_timeout)
1073 text_submit, text_success, text_failed = ("shutdown", "stopped", "stopping")
1075 jex = JobExecutor(opts=opts)
1077 for inst in inst_list:
1078 ToStdout("Submit %s of instance %s", text_submit, inst)
1079 op = opcls(instance_name=inst)
1080 jex.QueueJob(inst, op)
1082 results = jex.GetResults()
1083 bad_cnt = len([1 for (success, _) in results if not success])
1086 ToStdout("All instances have been %s successfully", text_success)
1088 ToStderr("There were errors while %s instances:\n"
1089 "%d error(s) out of %d instance(s)", text_failed, bad_cnt,
1096 class _RunWhenNodesReachableHelper:
1097 """Helper class to make shared internal state sharing easier.
1099 @ivar success: Indicates if all action_cb calls were successful
1102 def __init__(self, node_list, action_cb, node2ip, port, feedback_fn,
1103 _ping_fn=netutils.TcpPing, _sleep_fn=time.sleep):
1106 @param node_list: The list of nodes to be reachable
1107 @param action_cb: Callback called when a new host is reachable
1109 @param node2ip: Node to ip mapping
1110 @param port: The port to use for the TCP ping
1111 @param feedback_fn: The function used for feedback
1112 @param _ping_fn: Function to check reachabilty (for unittest use only)
1113 @param _sleep_fn: Function to sleep (for unittest use only)
1116 self.down = set(node_list)
1118 self.node2ip = node2ip
1120 self.action_cb = action_cb
1122 self.feedback_fn = feedback_fn
1123 self._ping_fn = _ping_fn
1124 self._sleep_fn = _sleep_fn
1127 """When called we run action_cb.
1129 @raises utils.RetryAgain: When there are still down nodes
1132 if not self.action_cb(self.up):
1133 self.success = False
1136 raise utils.RetryAgain()
1140 def Wait(self, secs):
1141 """Checks if a host is up or waits remaining seconds.
1143 @param secs: The secs remaining
1147 for node in self.down:
1148 if self._ping_fn(self.node2ip[node], self.port, timeout=_EPO_PING_TIMEOUT,
1149 live_port_needed=True):
1150 self.feedback_fn("Node %s became available" % node)
1152 self.down -= self.up
1153 # If we have a node available there is the possibility to run the
1154 # action callback successfully, therefore we don't wait and return
1157 self._sleep_fn(max(0.0, start + secs - time.time()))
1160 def _RunWhenNodesReachable(node_list, action_cb, interval):
1161 """Run action_cb when nodes become reachable.
1163 @param node_list: The list of nodes to be reachable
1164 @param action_cb: Callback called when a new host is reachable
1165 @param interval: The earliest time to retry
1168 client = GetClient()
1169 cluster_info = client.QueryClusterInfo()
1170 if cluster_info["primary_ip_version"] == constants.IP4_VERSION:
1171 family = netutils.IPAddress.family
1173 family = netutils.IP6Address.family
1175 node2ip = dict((node, netutils.GetHostname(node, family=family).ip)
1176 for node in node_list)
1178 port = netutils.GetDaemonPort(constants.NODED)
1179 helper = _RunWhenNodesReachableHelper(node_list, action_cb, node2ip, port,
1183 return utils.Retry(helper, interval, _EPO_REACHABLE_TIMEOUT,
1184 wait_fn=helper.Wait)
1185 except utils.RetryTimeout:
1186 ToStderr("Time exceeded while waiting for nodes to become reachable"
1187 " again:\n - %s", " - ".join(helper.down))
1191 def _MaybeInstanceStartup(opts, inst_map, nodes_online,
1192 _instance_start_fn=_InstanceStart):
1193 """Start the instances conditional based on node_states.
1195 @param opts: The command line options selected by the user
1196 @param inst_map: A dict of inst -> nodes mapping
1197 @param nodes_online: A list of nodes online
1198 @param _instance_start_fn: Callback to start instances (unittest use only)
1199 @return: Success of the operation on all instances
1202 start_inst_list = []
1203 for (inst, nodes) in inst_map.items():
1204 if not (nodes - nodes_online):
1205 # All nodes the instance lives on are back online
1206 start_inst_list.append(inst)
1208 for inst in start_inst_list:
1212 return _instance_start_fn(opts, start_inst_list, True)
1217 def _EpoOn(opts, full_node_list, node_list, inst_map):
1218 """Does the actual power on.
1220 @param opts: The command line options selected by the user
1221 @param full_node_list: All nodes to operate on (includes nodes not supporting
1223 @param node_list: The list of nodes to operate on (all need to support OOB)
1224 @param inst_map: A dict of inst -> nodes mapping
1225 @return: The desired exit status
1228 if node_list and not _OobPower(opts, node_list, False):
1229 ToStderr("Not all nodes seem to get back up, investigate and start"
1230 " manually if needed")
1232 # Wait for the nodes to be back up
1233 action_cb = compat.partial(_MaybeInstanceStartup, opts, dict(inst_map))
1235 ToStdout("Waiting until all nodes are available again")
1236 if not _RunWhenNodesReachable(full_node_list, action_cb, _EPO_PING_INTERVAL):
1237 ToStderr("Please investigate and start stopped instances manually")
1238 return constants.EXIT_FAILURE
1240 return constants.EXIT_SUCCESS
1243 def _EpoOff(opts, node_list, inst_map):
1244 """Does the actual power off.
1246 @param opts: The command line options selected by the user
1247 @param node_list: The list of nodes to operate on (all need to support OOB)
1248 @param inst_map: A dict of inst -> nodes mapping
1249 @return: The desired exit status
1252 if not _InstanceStart(opts, inst_map.keys(), False):
1253 ToStderr("Please investigate and stop instances manually before continuing")
1254 return constants.EXIT_FAILURE
1257 return constants.EXIT_SUCCESS
1259 if _OobPower(opts, node_list, False):
1260 return constants.EXIT_SUCCESS
1262 return constants.EXIT_FAILURE
1265 def Epo(opts, args):
1268 @param opts: the command line options selected by the user
1270 @param args: should contain only one element, the subcommand
1272 @return: the desired exit code
1275 if opts.groups and opts.show_all:
1276 ToStderr("Only one of --groups or --all are allowed")
1277 return constants.EXIT_FAILURE
1278 elif args and opts.show_all:
1279 ToStderr("Arguments in combination with --all are not allowed")
1280 return constants.EXIT_FAILURE
1282 client = GetClient()
1285 node_query_list = itertools.chain(*client.QueryGroups(names=args,
1286 fields=["node_list"],
1289 node_query_list = args
1291 result = client.QueryNodes(names=node_query_list,
1292 fields=["name", "master", "pinst_list",
1293 "sinst_list", "powered", "offline"],
1297 for (idx, (node, master, pinsts, sinsts, powered,
1298 offline)) in enumerate(result):
1299 # Normalize the node_query_list as well
1300 if not opts.show_all:
1301 node_query_list[idx] = node
1303 for inst in (pinsts + sinsts):
1304 if inst in inst_map:
1306 inst_map[inst].add(node)
1308 inst_map[inst] = set()
1310 inst_map[inst] = set([node])
1312 if master and opts.on:
1313 # We ignore the master for turning on the machines, in fact we are
1314 # already operating on the master at this point :)
1316 elif master and not opts.show_all:
1317 ToStderr("%s is the master node, please do a master-failover to another"
1318 " node not affected by the EPO or use --all if you intend to"
1319 " shutdown the whole cluster", node)
1320 return constants.EXIT_FAILURE
1321 elif powered is None:
1322 ToStdout("Node %s does not support out-of-band handling, it can not be"
1323 " handled in a fully automated manner", node)
1324 elif powered == opts.on:
1325 ToStdout("Node %s is already in desired power state, skipping", node)
1326 elif not offline or (offline and powered):
1327 node_list.append(node)
1329 if not opts.force and not ConfirmOperation(node_query_list, "nodes", "epo"):
1330 return constants.EXIT_FAILURE
1333 return _EpoOn(opts, node_query_list, node_list, inst_map)
1335 return _EpoOff(opts, node_list, inst_map)
1340 InitCluster, [ArgHost(min=1, max=1)],
1341 [BACKEND_OPT, CP_SIZE_OPT, ENABLED_HV_OPT, GLOBAL_FILEDIR_OPT,
1342 HVLIST_OPT, MAC_PREFIX_OPT, MASTER_NETDEV_OPT, NIC_PARAMS_OPT,
1343 NOLVM_STORAGE_OPT, NOMODIFY_ETCHOSTS_OPT, NOMODIFY_SSH_SETUP_OPT,
1344 SECONDARY_IP_OPT, VG_NAME_OPT, MAINTAIN_NODE_HEALTH_OPT,
1345 UIDPOOL_OPT, DRBD_HELPER_OPT, NODRBD_STORAGE_OPT,
1346 DEFAULT_IALLOCATOR_OPT, PRIMARY_IP_VERSION_OPT, PREALLOC_WIPE_DISKS_OPT,
1347 NODE_PARAMS_OPT, GLOBAL_SHARED_FILEDIR_OPT],
1348 "[opts...] <cluster_name>", "Initialises a new cluster configuration"),
1350 DestroyCluster, ARGS_NONE, [YES_DOIT_OPT],
1351 "", "Destroy cluster"),
1353 RenameCluster, [ArgHost(min=1, max=1)],
1354 [FORCE_OPT, DRY_RUN_OPT],
1356 "Renames the cluster"),
1358 RedistributeConfig, ARGS_NONE, [SUBMIT_OPT, DRY_RUN_OPT, PRIORITY_OPT],
1359 "", "Forces a push of the configuration file and ssconf files"
1360 " to the nodes in the cluster"),
1362 VerifyCluster, ARGS_NONE,
1363 [VERBOSE_OPT, DEBUG_SIMERR_OPT, ERROR_CODES_OPT, NONPLUS1_OPT,
1364 DRY_RUN_OPT, PRIORITY_OPT, NODEGROUP_OPT],
1365 "", "Does a check on the cluster configuration"),
1367 VerifyDisks, ARGS_NONE, [PRIORITY_OPT],
1368 "", "Does a check on the cluster disk status"),
1369 "repair-disk-sizes": (
1370 RepairDiskSizes, ARGS_MANY_INSTANCES, [DRY_RUN_OPT, PRIORITY_OPT],
1371 "", "Updates mismatches in recorded disk sizes"),
1372 "master-failover": (
1373 MasterFailover, ARGS_NONE, [NOVOTING_OPT],
1374 "", "Makes the current node the master"),
1376 MasterPing, ARGS_NONE, [],
1377 "", "Checks if the master is alive"),
1379 ShowClusterVersion, ARGS_NONE, [],
1380 "", "Shows the cluster version"),
1382 ShowClusterMaster, ARGS_NONE, [],
1383 "", "Shows the cluster master"),
1385 ClusterCopyFile, [ArgFile(min=1, max=1)],
1386 [NODE_LIST_OPT, USE_REPL_NET_OPT, NODEGROUP_OPT],
1387 "[-n node...] <filename>", "Copies a file to all (or only some) nodes"),
1389 RunClusterCommand, [ArgCommand(min=1)],
1390 [NODE_LIST_OPT, NODEGROUP_OPT],
1391 "[-n node...] <command>", "Runs a command on all (or only some) nodes"),
1393 ShowClusterConfig, ARGS_NONE, [ROMAN_OPT],
1394 "[--roman]", "Show cluster configuration"),
1396 ListTags, ARGS_NONE, [], "", "List the tags of the cluster"),
1398 AddTags, [ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT],
1399 "tag...", "Add tags to the cluster"),
1401 RemoveTags, [ArgUnknown()], [TAG_SRC_OPT, PRIORITY_OPT],
1402 "tag...", "Remove tags from the cluster"),
1404 SearchTags, [ArgUnknown(min=1, max=1)], [PRIORITY_OPT], "",
1405 "Searches the tags on all objects on"
1406 " the cluster for a given pattern (regex)"),
1409 [ArgChoice(min=1, max=1, choices=["drain", "undrain", "info"])],
1410 [], "drain|undrain|info", "Change queue properties"),
1413 [ArgChoice(min=1, max=1, choices=["pause", "continue", "info"]),
1414 ArgSuggest(min=0, max=1, choices=["30m", "1h", "4h"])],
1416 "{pause <timespec>|continue|info}", "Change watcher properties"),
1418 SetClusterParams, ARGS_NONE,
1419 [BACKEND_OPT, CP_SIZE_OPT, ENABLED_HV_OPT, HVLIST_OPT, MASTER_NETDEV_OPT,
1420 NIC_PARAMS_OPT, NOLVM_STORAGE_OPT, VG_NAME_OPT, MAINTAIN_NODE_HEALTH_OPT,
1421 UIDPOOL_OPT, ADD_UIDS_OPT, REMOVE_UIDS_OPT, DRBD_HELPER_OPT,
1422 NODRBD_STORAGE_OPT, DEFAULT_IALLOCATOR_OPT, RESERVED_LVS_OPT,
1423 DRY_RUN_OPT, PRIORITY_OPT, PREALLOC_WIPE_DISKS_OPT, NODE_PARAMS_OPT],
1425 "Alters the parameters of the cluster"),
1427 RenewCrypto, ARGS_NONE,
1428 [NEW_CLUSTER_CERT_OPT, NEW_RAPI_CERT_OPT, RAPI_CERT_OPT,
1429 NEW_CONFD_HMAC_KEY_OPT, FORCE_OPT,
1430 NEW_CLUSTER_DOMAIN_SECRET_OPT, CLUSTER_DOMAIN_SECRET_OPT,
1431 NEW_SPICE_CERT_OPT, SPICE_CERT_OPT, SPICE_CACERT_OPT],
1433 "Renews cluster certificates, keys and secrets"),
1435 Epo, [ArgUnknown()],
1436 [FORCE_OPT, ON_OPT, GROUPS_OPT, ALL_OPT, OOB_TIMEOUT_OPT,
1437 SHUTDOWN_TIMEOUT_OPT, POWER_DELAY_OPT],
1439 "Performs an emergency power-off on given args"),
1440 "activate-master-ip": (
1441 ActivateMasterIp, ARGS_NONE, [], "", "Activates the master IP"),
1442 "deactivate-master-ip": (
1443 DeactivateMasterIp, ARGS_NONE, [CONFIRM_OPT], "",
1444 "Deactivates the master IP"),
1448 #: dictionary with aliases for commands
1450 "masterfailover": "master-failover",
1455 return GenericMain(commands, override={"tag_type": constants.TAG_CLUSTER},