X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/e9c487becfd474c1b1b19ce192e6380606fa37ed..a182a3ed71ac3fc17f265b9d6372d77e026dc054:/lib/cli.py diff --git a/lib/cli.py b/lib/cli.py index c1a7287..30f6771 100644 --- a/lib/cli.py +++ b/lib/cli.py @@ -27,6 +27,9 @@ import textwrap import os.path import time import logging +import errno +import itertools +import shlex from cStringIO import StringIO from ganeti import utils @@ -66,6 +69,7 @@ __all__ = [ "DEBUG_SIMERR_OPT", "DISKIDX_OPT", "DISK_OPT", + "DISK_PARAMS_OPT", "DISK_TEMPLATE_OPT", "DRAINED_OPT", "DRY_RUN_OPT", @@ -77,6 +81,7 @@ __all__ = [ "FIELDS_OPT", "FILESTORE_DIR_OPT", "FILESTORE_DRIVER_OPT", + "FORCE_FILTER_OPT", "FORCE_OPT", "FORCE_VARIANT_OPT", "GLOBAL_FILEDIR_OPT", @@ -89,6 +94,7 @@ __all__ = [ "DEFAULT_IALLOCATOR_OPT", "IDENTIFY_DEFAULTS_OPT", "IGNORE_CONSIST_OPT", + "IGNORE_ERRORS_OPT", "IGNORE_FAILURES_OPT", "IGNORE_OFFLINE_OPT", "IGNORE_REMOVE_FAILURES_OPT", @@ -98,6 +104,7 @@ __all__ = [ "MAC_PREFIX_OPT", "MAINTAIN_NODE_HEALTH_OPT", "MASTER_NETDEV_OPT", + "MASTER_NETMASK_OPT", "MC_OPT", "MIGRATION_MODE_OPT", "NET_OPT", @@ -106,6 +113,7 @@ __all__ = [ "NEW_CONFD_HMAC_KEY_OPT", "NEW_RAPI_CERT_OPT", "NEW_SECONDARY_OPT", + "NEW_SPICE_CERT_OPT", "NIC_PARAMS_OPT", "NODE_FORCE_JOIN_OPT", "NODE_LIST_OPT", @@ -128,7 +136,10 @@ __all__ = [ "NOSTART_OPT", "NOSSH_KEYCHECK_OPT", "NOVOTING_OPT", + "NO_REMEMBER_OPT", "NWSYNC_OPT", + "OFFLINE_INST_OPT", + "ONLINE_INST_OPT", "ON_PRIMARY_OPT", "ON_SECONDARY_OPT", "OFFLINE_OPT", @@ -139,6 +150,7 @@ __all__ = [ "POWER_DELAY_OPT", "PREALLOC_WIPE_DISKS_OPT", "PRIMARY_IP_VERSION_OPT", + "PRIMARY_ONLY_OPT", "PRIORITY_OPT", "RAPI_CERT_OPT", "READD_OPT", @@ -148,24 +160,38 @@ __all__ = [ "RESERVED_LVS_OPT", "ROMAN_OPT", "SECONDARY_IP_OPT", + "SECONDARY_ONLY_OPT", "SELECT_OS_OPT", "SEP_OPT", "SHOWCMD_OPT", "SHUTDOWN_TIMEOUT_OPT", "SINGLE_NODE_OPT", + "SPECS_CPU_COUNT_OPT", + "SPECS_DISK_COUNT_OPT", + "SPECS_DISK_SIZE_OPT", + "SPECS_MEM_SIZE_OPT", + "SPECS_NIC_COUNT_OPT", + "SPICE_CACERT_OPT", + "SPICE_CERT_OPT", "SRC_DIR_OPT", "SRC_NODE_OPT", "SUBMIT_OPT", + "STARTUP_PAUSED_OPT", "STATIC_OPT", "SYNC_OPT", + "TAG_ADD_OPT", "TAG_SRC_OPT", "TIMEOUT_OPT", + "TO_GROUP_OPT", "UIDPOOL_OPT", "USEUNITS_OPT", + "USE_EXTERNAL_MIP_SCRIPT", "USE_REPL_NET_OPT", "VERBOSE_OPT", "VG_NAME_OPT", "YES_DOIT_OPT", + "DISK_STATE_OPT", + "HV_STATE_OPT", # Generic functions for CLI programs "ConfirmOperation", "GenericMain", @@ -248,9 +274,12 @@ _PRIONAME_TO_VALUE = dict(_PRIORITY_NAMES) QR_UNKNOWN, QR_INCOMPLETE) = range(3) +#: Maximum batch size for ChooseJob +_CHOOSE_BATCH = 25 + class _Argument: - def __init__(self, min=0, max=None): # pylint: disable-msg=W0622 + def __init__(self, min=0, max=None): # pylint: disable=W0622 self.min = min self.max = max @@ -265,7 +294,7 @@ class ArgSuggest(_Argument): Value can be any of the ones passed to the constructor. """ - # pylint: disable-msg=W0622 + # pylint: disable=W0622 def __init__(self, min=0, max=None, choices=None): _Argument.__init__(self, min=min, max=max) self.choices = choices @@ -344,7 +373,8 @@ ARGS_MANY_NODES = [ArgNode()] ARGS_MANY_GROUPS = [ArgGroup()] ARGS_ONE_INSTANCE = [ArgInstance(min=1, max=1)] ARGS_ONE_NODE = [ArgNode(min=1, max=1)] -ARGS_ONE_GROUP = [ArgInstance(min=1, max=1)] +# TODO +ARGS_ONE_GROUP = [ArgGroup(min=1, max=1)] ARGS_ONE_OS = [ArgOs(min=1, max=1)] @@ -359,7 +389,9 @@ def _ExtractTagsObject(opts, args): kind = opts.tag_type if kind == constants.TAG_CLUSTER: retval = kind, kind - elif kind == constants.TAG_NODE or kind == constants.TAG_INSTANCE: + elif kind in (constants.TAG_NODEGROUP, + constants.TAG_NODE, + constants.TAG_INSTANCE): if not args: raise errors.OpPrereqError("no arguments passed to the command") name = args.pop(0) @@ -450,7 +482,7 @@ def RemoveTags(opts, args): SubmitOpCode(op, opts=opts) -def check_unit(option, opt, value): # pylint: disable-msg=W0613 +def check_unit(option, opt, value): # pylint: disable=W0613 """OptParsers custom converter for units. """ @@ -497,7 +529,7 @@ def _SplitKeyVal(opt, data): return kv_dict -def check_ident_key_val(option, opt, value): # pylint: disable-msg=W0613 +def check_ident_key_val(option, opt, value): # pylint: disable=W0613 """Custom parser for ident:key=val,key=val options. This will store the parsed values as a tuple (ident, {key: val}). As such, @@ -505,7 +537,7 @@ def check_ident_key_val(option, opt, value): # pylint: disable-msg=W0613 """ if ":" not in value: - ident, rest = value, '' + ident, rest = value, "" else: ident, rest = value.split(":", 1) @@ -525,7 +557,7 @@ def check_ident_key_val(option, opt, value): # pylint: disable-msg=W0613 return retval -def check_key_val(option, opt, value): # pylint: disable-msg=W0613 +def check_key_val(option, opt, value): # pylint: disable=W0613 """Custom parser class for key=val,key=val options. This will store the parsed values as a dict {key: val}. @@ -534,7 +566,7 @@ def check_key_val(option, opt, value): # pylint: disable-msg=W0613 return _SplitKeyVal(opt, value) -def check_bool(option, opt, value): # pylint: disable-msg=W0613 +def check_bool(option, opt, value): # pylint: disable=W0613 """Custom parser for yes/no options. This will store the parsed value as either True or False. @@ -609,7 +641,7 @@ SEP_OPT = cli_option("--separator", default=None, " (defaults to one space)")) USEUNITS_OPT = cli_option("--units", default=None, - dest="units", choices=('h', 'm', 'g', 't'), + dest="units", choices=("h", "m", "g", "t"), help="Specify units for output (one of h/m/g/t)") FIELDS_OPT = cli_option("-o", "--output", dest="output", action="store", @@ -627,6 +659,10 @@ IGNORE_OFFLINE_OPT = cli_option("--ignore-offline", dest="ignore_offline", help=("Ignore offline nodes and do as much" " as possible")) +TAG_ADD_OPT = cli_option("--tags", dest="tags", + default=None, help="Comma-separated list of instance" + " tags") + TAG_SRC_OPT = cli_option("--from", dest="tags_source", default=None, help="File with tag names") @@ -659,9 +695,17 @@ NWSYNC_OPT = cli_option("--no-wait-for-sync", dest="wait_for_sync", default=True, action="store_false", help="Don't wait for sync (DANGEROUS!)") +ONLINE_INST_OPT = cli_option("--online", dest="online_inst", + action="store_true", default=False, + help="Enable offline instance") + +OFFLINE_INST_OPT = cli_option("--offline", dest="offline_inst", + action="store_true", default=False, + help="Disable down instance") + DISK_TEMPLATE_OPT = cli_option("-t", "--disk-template", dest="disk_template", - help="Custom disk setup (diskless, file," - " plain or drbd)", + help=("Custom disk setup (%s)" % + utils.CommaJoin(constants.DISK_TEMPLATES)), default=None, metavar="TEMPL", choices=list(constants.DISK_TEMPLATES)) @@ -712,9 +756,36 @@ BACKEND_OPT = cli_option("-B", "--backend-parameters", dest="beparams", type="keyval", default={}, help="Backend parameters") -HVOPTS_OPT = cli_option("-H", "--hypervisor-parameters", type="keyval", - default={}, dest="hvparams", - help="Hypervisor parameters") +HVOPTS_OPT = cli_option("-H", "--hypervisor-parameters", type="keyval", + default={}, dest="hvparams", + help="Hypervisor parameters") + +DISK_PARAMS_OPT = cli_option("-D", "--disk-parameters", dest="diskparams", + help="Disk template parameters, in the format" + " template:option=value,option=value,...", + type="identkeyval", action="append", default=[]) + +SPECS_MEM_SIZE_OPT = cli_option("--specs-mem-size", dest="ispecs_mem_size", + type="keyval", default={}, + help="Memory count specs: min, max, std" + " (in MB)") + +SPECS_CPU_COUNT_OPT = cli_option("--specs-cpu-count", dest="ispecs_cpu_count", + type="keyval", default={}, + help="CPU count specs: min, max, std") + +SPECS_DISK_COUNT_OPT = cli_option("--specs-disk-count", + dest="ispecs_disk_count", + type="keyval", default={}, + help="Disk count specs: min, max, std") + +SPECS_DISK_SIZE_OPT = cli_option("--specs-disk-size", dest="ispecs_disk_size", + type="keyval", default={}, + help="Disk size specs: min, max, std (in MB)") + +SPECS_NIC_COUNT_OPT = cli_option("--specs-nic-count", dest="ispecs_nic_count", + type="keyval", default={}, + help="NIC count specs: min, max, std") HYPERVISOR_OPT = cli_option("-H", "--hypervisor-parameters", dest="hypervisor", help="Hypervisor and hypervisor options, in the" @@ -788,7 +859,8 @@ NODE_LIST_OPT = cli_option("-n", "--node", dest="nodes", default=[], " times, if not given defaults to all nodes)", completion_suggest=OPT_COMPL_ONE_NODE) -NODEGROUP_OPT = cli_option("-g", "--node-group", +NODEGROUP_OPT_NAME = "--node-group" +NODEGROUP_OPT = cli_option("-g", NODEGROUP_OPT_NAME, dest="nodegroup", help="Node group (name or uuid)", metavar="", @@ -861,12 +933,16 @@ NEW_SECONDARY_OPT = cli_option("-n", "--new-secondary", dest="dst_node", ON_PRIMARY_OPT = cli_option("-p", "--on-primary", dest="on_primary", default=False, action="store_true", help="Replace the disk(s) on the primary" - " node (only for the drbd template)") + " node (applies only to internally mirrored" + " disk templates, e.g. %s)" % + utils.CommaJoin(constants.DTS_INT_MIRROR)) ON_SECONDARY_OPT = cli_option("-s", "--on-secondary", dest="on_secondary", default=False, action="store_true", help="Replace the disk(s) on the secondary" - " node (only for the drbd template)") + " node (applies only to internally mirrored" + " disk templates, e.g. %s)" % + utils.CommaJoin(constants.DTS_INT_MIRROR)) AUTO_PROMOTE_OPT = cli_option("--auto-promote", dest="auto_promote", default=False, action="store_true", @@ -876,7 +952,9 @@ AUTO_PROMOTE_OPT = cli_option("--auto-promote", dest="auto_promote", AUTO_REPLACE_OPT = cli_option("-a", "--auto", dest="auto", default=False, action="store_true", help="Automatically replace faulty disks" - " (only for the drbd template)") + " (applies only to internally mirrored" + " disk templates, e.g. %s)" % + utils.CommaJoin(constants.DTS_INT_MIRROR)) IGNORE_SIZE_OPT = cli_option("--ignore-size", dest="ignore_size", default=False, action="store_true", @@ -905,8 +983,7 @@ NOSSH_KEYCHECK_OPT = cli_option("--no-ssh-key-check", dest="ssh_key_check", NODE_FORCE_JOIN_OPT = cli_option("--force-join", dest="force_join", default=False, action="store_true", - help="Force the joining of a node," - " needed when merging clusters") + help="Force the joining of a node") MC_OPT = cli_option("-C", "--master-candidate", dest="master_candidate", type="bool", default=None, metavar=_YORNO, @@ -959,7 +1036,7 @@ VG_NAME_OPT = cli_option("--vg-name", dest="vg_name", " [%s]" % constants.DEFAULT_VG), metavar="VG", default=None) -YES_DOIT_OPT = cli_option("--yes-do-it", dest="yes_do_it", +YES_DOIT_OPT = cli_option("--yes-do-it", "--ya-rly", dest="yes_do_it", help="Destroy cluster", action="store_true") NOVOTING_OPT = cli_option("--no-voting", dest="no_voting", @@ -980,6 +1057,18 @@ MASTER_NETDEV_OPT = cli_option("--master-netdev", dest="master_netdev", metavar="NETDEV", default=None) +MASTER_NETMASK_OPT = cli_option("--master-netmask", dest="master_netmask", + help="Specify the netmask of the master IP", + metavar="NETMASK", + default=None) + +USE_EXTERNAL_MIP_SCRIPT = cli_option("--use-external-mip-script", + dest="use_external_mip_script", + help="Specify whether to run a user-provided" + " script for the master IP address turnup and" + " turndown operations", + type="bool", metavar=_YORNO, default=None) + GLOBAL_FILEDIR_OPT = cli_option("--file-storage-dir", dest="file_storage_dir", help="Specify the default directory (cluster-" "wide) for storing the file-based disks [%s]" % @@ -1061,6 +1150,21 @@ NEW_RAPI_CERT_OPT = cli_option("--new-rapi-certificate", dest="new_rapi_cert", help=("Generate a new self-signed RAPI" " certificate")) +SPICE_CERT_OPT = cli_option("--spice-certificate", dest="spice_cert", + default=None, + help="File containing new SPICE certificate") + +SPICE_CACERT_OPT = cli_option("--spice-ca-certificate", dest="spice_cacert", + default=None, + help="File containing the certificate of the CA" + " which signed the SPICE certificate") + +NEW_SPICE_CERT_OPT = cli_option("--new-spice-certificate", + dest="new_spice_cert", default=None, + action="store_true", + help=("Generate a new self-signed SPICE" + " certificate")) + NEW_CONFD_HMAC_KEY_OPT = cli_option("--new-confd-hmac-key", dest="new_confd_hmac_key", default=False, action="store_true", @@ -1182,6 +1286,54 @@ POWER_DELAY_OPT = cli_option("--power-delay", dest="power_delay", type="float", default=constants.OOB_POWER_DELAY, help="Time in seconds to wait between power-ons") +FORCE_FILTER_OPT = cli_option("-F", "--filter", dest="force_filter", + action="store_true", default=False, + help=("Whether command argument should be treated" + " as filter")) + +NO_REMEMBER_OPT = cli_option("--no-remember", + dest="no_remember", + action="store_true", default=False, + help="Perform but do not record the change" + " in the configuration") + +PRIMARY_ONLY_OPT = cli_option("-p", "--primary-only", + default=False, action="store_true", + help="Evacuate primary instances only") + +SECONDARY_ONLY_OPT = cli_option("-s", "--secondary-only", + default=False, action="store_true", + help="Evacuate secondary instances only" + " (applies only to internally mirrored" + " disk templates, e.g. %s)" % + utils.CommaJoin(constants.DTS_INT_MIRROR)) + +STARTUP_PAUSED_OPT = cli_option("--paused", dest="startup_paused", + action="store_true", default=False, + help="Pause instance at startup") + +TO_GROUP_OPT = cli_option("--to", dest="to", metavar="", + help="Destination node group (name or uuid)", + default=None, action="append", + completion_suggest=OPT_COMPL_ONE_NODEGROUP) + +IGNORE_ERRORS_OPT = cli_option("-I", "--ignore-errors", default=[], + action="append", dest="ignore_errors", + choices=list(constants.CV_ALL_ECODES_STRINGS), + help="Error code to be ignored") + +DISK_STATE_OPT = cli_option("--disk-state", default=[], dest="disk_state", + action="append", + help=("Specify disk state information in the format" + " storage_type/identifier:option=value,..."), + type="identkeyval") + +HV_STATE_OPT = cli_option("--hypervisor-state", default=[], dest="hv_state", + action="append", + help=("Specify hypervisor state information in the" + " format hypervisor:option=value,..."), + type="identkeyval") + #: Options provided by all commands COMMON_OPTS = [DEBUG_OPT] @@ -1205,12 +1357,13 @@ COMMON_CREATE_OPTS = [ OSPARAMS_OPT, OS_SIZE_OPT, SUBMIT_OPT, + TAG_ADD_OPT, DRY_RUN_OPT, PRIORITY_OPT, ] -def _ParseArgs(argv, commands, aliases): +def _ParseArgs(argv, commands, aliases, env_override): """Parser for the command line arguments. This function parses the arguments and returns the function which @@ -1220,8 +1373,11 @@ def _ParseArgs(argv, commands, aliases): @param commands: dictionary with special contents, see the design doc for cmdline handling @param aliases: dictionary with command aliases {'alias': 'target, ...} + @param env_override: list of env variables allowed for default args """ + assert not (env_override - set(commands)) + if len(argv) == 0: binary = "" else: @@ -1275,13 +1431,19 @@ def _ParseArgs(argv, commands, aliases): cmd = aliases[cmd] + if cmd in env_override: + args_env_name = ("%s_%s" % (binary.replace("-", "_"), cmd)).upper() + env_args = os.environ.get(args_env_name) + if env_args: + argv = utils.InsertAtPos(argv, 1, shlex.split(env_args)) + func, args_def, parser_opts, usage, description = commands[cmd] parser = OptionParser(option_list=parser_opts + COMMON_OPTS, description=description, formatter=TitledHelpFormatter(), usage="%%prog %s %s" % (cmd, usage)) parser.disable_interspersed_args() - options, args = parser.parse_args() + options, args = parser.parse_args(args=argv[1:]) if not _CheckArguments(cmd, args_def, args): return None, None, None @@ -1364,8 +1526,8 @@ def SplitNodeOption(value): """Splits the value of a --node option. """ - if value and ':' in value: - return value.split(':', 1) + if value and ":" in value: + return value.split(":", 1) else: return (value, None) @@ -1382,7 +1544,7 @@ def CalculateOSNames(os_name, os_variants): """ if os_variants: - return ['%s+%s' % (os_name, v) for v in os_variants] + return ["%s+%s" % (os_name, v) for v in os_variants] else: return [os_name] @@ -1424,12 +1586,12 @@ def AskUser(text, choices=None): """ if choices is None: - choices = [('y', True, 'Perform the operation'), - ('n', False, 'Do not perform the operation')] + choices = [("y", True, "Perform the operation"), + ("n", False, "Do not perform the operation")] if not choices or not isinstance(choices, list): raise errors.ProgrammerError("Invalid choices argument to AskUser") for entry in choices: - if not isinstance(entry, tuple) or len(entry) < 3 or entry[0] == '?': + if not isinstance(entry, tuple) or len(entry) < 3 or entry[0] == "?": raise errors.ProgrammerError("Invalid choices element to AskUser") answer = choices[-1][1] @@ -1444,18 +1606,18 @@ def AskUser(text, choices=None): try: chars = [entry[0] for entry in choices] chars[-1] = "[%s]" % chars[-1] - chars.append('?') + chars.append("?") maps = dict([(entry[0], entry[1]) for entry in choices]) while True: f.write(text) - f.write('\n') + f.write("\n") f.write("/".join(chars)) f.write(": ") line = f.readline(2).strip().lower() if line in maps: answer = maps[line] break - elif line == '?': + elif line == "?": for entry in choices: f.write(" %s - %s\n" % (entry[0], entry[2])) f.write("\n") @@ -1702,7 +1864,7 @@ class StdioJobPollReportCb(JobPollReportCbBase): ToStderr("Job %s is waiting in queue", job_id) self.notified_queued = True - elif status == constants.JOB_STATUS_WAITLOCK and not self.notified_waitlock: + elif status == constants.JOB_STATUS_WAITING and not self.notified_waitlock: ToStderr("Job %s is trying to acquire all necessary locks", job_id) self.notified_waitlock = True @@ -1902,6 +2064,9 @@ def FormatError(err): "%s" % msg) elif isinstance(err, errors.JobLost): obuf.write("Error checking job status: %s" % msg) + elif isinstance(err, errors.QueryFilterParseError): + obuf.write("Error while parsing query filter: %s\n" % err.args[0]) + obuf.write("\n".join(err.GetDetails())) elif isinstance(err, errors.GenericError): obuf.write("Unhandled Ganeti error: %s" % msg) elif isinstance(err, JobSubmittedException): @@ -1909,19 +2074,21 @@ def FormatError(err): retcode = 0 else: obuf.write("Unhandled exception: %s" % msg) - return retcode, obuf.getvalue().rstrip('\n') + return retcode, obuf.getvalue().rstrip("\n") -def GenericMain(commands, override=None, aliases=None): +def GenericMain(commands, override=None, aliases=None, + env_override=frozenset()): """Generic main function for all the gnt-* commands. - Arguments: - - commands: a dictionary with a special structure, see the design doc - for command line handling. - - override: if not None, we expect a dictionary with keys that will - override command line options; this can be used to pass - options from the scripts to generic functions - - aliases: dictionary with command aliases {'alias': 'target, ...} + @param commands: a dictionary with a special structure, see the design doc + for command line handling. + @param override: if not None, we expect a dictionary with keys that will + override command line options; this can be used to pass + options from the scripts to generic functions + @param aliases: dictionary with command aliases {'alias': 'target, ...} + @param env_override: list of environment names which are allowed to submit + default args for commands """ # save the program name and the entire command line for later logging @@ -1940,7 +2107,7 @@ def GenericMain(commands, override=None, aliases=None): aliases = {} try: - func, options, args = _ParseArgs(sys.argv, commands, aliases) + func, options, args = _ParseArgs(sys.argv, commands, aliases, env_override) except errors.ParameterError, err: result, err_msg = FormatError(err) ToStderr(err_msg) @@ -1973,6 +2140,12 @@ def GenericMain(commands, override=None, aliases=None): ToStderr("Aborted. Note that if the operation created any jobs, they" " might have been submitted and" " will continue to run in the background.") + except IOError, err: + if err.errno == errno.EPIPE: + # our terminal went away, we'll exit + sys.exit(constants.EXIT_FAILURE) + else: + raise return result @@ -2046,7 +2219,7 @@ def GenericInstanceCreate(mode, opts, args): raise errors.OpPrereqError("Please use either the '--disk' or" " '-s' option") if opts.sd_size is not None: - opts.disks = [(0, {"size": opts.sd_size})] + opts.disks = [(0, {constants.IDISK_SIZE: opts.sd_size})] if opts.disks: try: @@ -2061,26 +2234,32 @@ def GenericInstanceCreate(mode, opts, args): if not isinstance(ddict, dict): msg = "Invalid disk/%d value: expected dict, got %s" % (didx, ddict) raise errors.OpPrereqError(msg) - elif "size" in ddict: - if "adopt" in ddict: + elif constants.IDISK_SIZE in ddict: + if constants.IDISK_ADOPT in ddict: raise errors.OpPrereqError("Only one of 'size' and 'adopt' allowed" " (disk %d)" % didx) try: - ddict["size"] = utils.ParseUnit(ddict["size"]) + ddict[constants.IDISK_SIZE] = \ + utils.ParseUnit(ddict[constants.IDISK_SIZE]) except ValueError, err: raise errors.OpPrereqError("Invalid disk size for disk %d: %s" % (didx, err)) - elif "adopt" in ddict: + elif constants.IDISK_ADOPT in ddict: if mode == constants.INSTANCE_IMPORT: raise errors.OpPrereqError("Disk adoption not allowed for instance" " import") - ddict["size"] = 0 + ddict[constants.IDISK_SIZE] = 0 else: raise errors.OpPrereqError("Missing size or adoption source for" " disk %d" % didx) disks[didx] = ddict - utils.ForceDictType(opts.beparams, constants.BES_PARAMETER_TYPES) + if opts.tags is not None: + tags = opts.tags.split(",") + else: + tags = [] + + utils.ForceDictType(opts.beparams, constants.BES_PARAMETER_COMPAT) utils.ForceDictType(hvparams, constants.HVS_PARAMETER_TYPES) if mode == constants.INSTANCE_CREATE: @@ -2123,6 +2302,7 @@ def GenericInstanceCreate(mode, opts, args): force_variant=force_variant, src_node=src_node, src_path=src_path, + tags=tags, no_install=no_install, identify_defaults=identify_defaults) @@ -2191,7 +2371,7 @@ class _RunWhileClusterStoppedHelper: """ # Pause watcher by acquiring an exclusive lock on watcher state file self.feedback_fn("Blocking watcher") - watcher_block = utils.FileLock.Open(constants.WATCHER_STATEFILE) + watcher_block = utils.FileLock.Open(constants.WATCHER_LOCK_FILE) try: # TODO: Currently, this just blocks. There's no timeout. # TODO: Should it be a shared lock? @@ -2293,8 +2473,8 @@ def GenerateTable(headers, fields, separator, data, if unitfields is None: unitfields = [] - numfields = utils.FieldSet(*numfields) # pylint: disable-msg=W0142 - unitfields = utils.FieldSet(*unitfields) # pylint: disable-msg=W0142 + numfields = utils.FieldSet(*numfields) # pylint: disable=W0142 + unitfields = utils.FieldSet(*unitfields) # pylint: disable=W0142 format_fields = [] for field in fields: @@ -2312,7 +2492,7 @@ def GenerateTable(headers, fields, separator, data, if separator is None: mlens = [0 for name in fields] - format_str = ' '.join(format_fields) + format_str = " ".join(format_fields) else: format_str = separator.replace("%", "%%").join(format_fields) @@ -2351,7 +2531,7 @@ def GenerateTable(headers, fields, separator, data, for line in data: args = [] if line is None: - line = ['-' for _ in fields] + line = ["-" for _ in fields] for idx in range(len(fields)): if separator is None: args.append(mlens[idx]) @@ -2560,7 +2740,7 @@ def _WarnUnknownFields(fdefs): def GenericList(resource, fields, names, unit, separator, header, cl=None, - format_override=None, verbose=False): + format_override=None, verbose=False, force_filter=False): """Generic implementation for listing all items of a resource. @param resource: One of L{constants.QR_VIA_LUXI} @@ -2576,6 +2756,8 @@ def GenericList(resource, fields, names, unit, separator, header, cl=None, @param separator: String used to separate fields @type header: bool @param header: Whether to show header row + @type force_filter: bool + @param force_filter: Whether to always treat names as filter @type format_override: dict @param format_override: Dictionary for overriding field formatting functions, indexed by field name, contents like L{_DEFAULT_FORMAT_QUERY} @@ -2583,13 +2765,15 @@ def GenericList(resource, fields, names, unit, separator, header, cl=None, @param verbose: whether to use verbose field descriptions or not """ - if cl is None: - cl = GetClient() - if not names: names = None - response = cl.Query(resource, fields, qlang.MakeSimpleFilter("name", names)) + qfilter = qlang.MakeFilter(names, force_filter) + + if cl is None: + cl = GetClient() + + response = cl.Query(resource, fields, qfilter) found_unknown = _WarnUnknownFields(response.fields) @@ -2741,8 +2925,8 @@ def FormatTimestamp(ts): @return: a string with the formatted timestamp """ - if not isinstance (ts, (tuple, list)) or len(ts) != 2: - return '?' + if not isinstance(ts, (tuple, list)) or len(ts) != 2: + return "?" sec, usec = ts return time.strftime("%F %T", time.localtime(sec)) + ".%06d" % usec @@ -2765,11 +2949,11 @@ def ParseTimespec(value): if not value: raise errors.OpPrereqError("Empty time specification passed") suffix_map = { - 's': 1, - 'm': 60, - 'h': 3600, - 'd': 86400, - 'w': 604800, + "s": 1, + "m": 60, + "h": 3600, + "d": 86400, + "w": 604800, } if value[-1] not in suffix_map: try: @@ -2790,7 +2974,7 @@ def ParseTimespec(value): def GetOnlineNodes(nodes, cl=None, nowarn=False, secondary_ips=False, - filter_master=False): + filter_master=False, nodegroup=None): """Returns the names of online nodes. This function will also log a warning on stderr with the names of @@ -2811,28 +2995,60 @@ def GetOnlineNodes(nodes, cl=None, nowarn=False, secondary_ips=False, @param filter_master: if True, do not return the master node in the list (useful in coordination with secondary_ips where we cannot check our node name against the list) + @type nodegroup: string + @param nodegroup: If set, only return nodes in this node group """ if cl is None: cl = GetClient() - if secondary_ips: - name_idx = 2 - else: - name_idx = 0 + qfilter = [] + + if nodes: + qfilter.append(qlang.MakeSimpleFilter("name", nodes)) + + if nodegroup is not None: + qfilter.append([qlang.OP_OR, [qlang.OP_EQUAL, "group", nodegroup], + [qlang.OP_EQUAL, "group.uuid", nodegroup]]) if filter_master: - master_node = cl.QueryConfigValues(["master_node"])[0] - filter_fn = lambda x: x != master_node + qfilter.append([qlang.OP_NOT, [qlang.OP_TRUE, "master"]]) + + if qfilter: + if len(qfilter) > 1: + final_filter = [qlang.OP_AND] + qfilter + else: + assert len(qfilter) == 1 + final_filter = qfilter[0] else: - filter_fn = lambda _: True + final_filter = None + + result = cl.Query(constants.QR_NODE, ["name", "offline", "sip"], final_filter) + + def _IsOffline(row): + (_, (_, offline), _) = row + return offline + + def _GetName(row): + ((_, name), _, _) = row + return name + + def _GetSip(row): + (_, _, (_, sip)) = row + return sip + + (offline, online) = compat.partition(result.data, _IsOffline) - result = cl.QueryNodes(names=nodes, fields=["name", "offline", "sip"], - use_locking=False) - offline = [row[0] for row in result if row[1]] if offline and not nowarn: - ToStderr("Note: skipping offline node(s): %s" % utils.CommaJoin(offline)) - return [row[name_idx] for row in result if not row[1] and filter_fn(row[0])] + ToStderr("Note: skipping offline node(s): %s" % + utils.CommaJoin(map(_GetName, offline))) + + if secondary_ips: + fn = _GetSip + else: + fn = _GetName + + return map(fn, online) def _ToStream(stream, txt, *args): @@ -2844,13 +3060,20 @@ def _ToStream(stream, txt, *args): @param txt: the message """ - if args: - args = tuple(args) - stream.write(txt % args) - else: - stream.write(txt) - stream.write('\n') - stream.flush() + try: + if args: + args = tuple(args) + stream.write(txt % args) + else: + stream.write(txt) + stream.write("\n") + stream.flush() + except IOError, err: + if err.errno == errno.EPIPE: + # our terminal went away, we'll exit + sys.exit(constants.EXIT_FAILURE) + else: + raise def ToStdout(txt, *args): @@ -2893,15 +3116,33 @@ class JobExecutor(object): self.jobs = [] self.opts = opts self.feedback_fn = feedback_fn + self._counter = itertools.count() + + @staticmethod + def _IfName(name, fmt): + """Helper function for formatting name. + + """ + if name: + return fmt % name + + return "" def QueueJob(self, name, *ops): """Record a job for later submit. @type name: string @param name: a description of the job, will be used in WaitJobSet + """ SetGenericOpcodeOpts(ops, self.opts) - self.queue.append((name, ops)) + self.queue.append((self._counter.next(), name, ops)) + + def AddJobId(self, name, status, job_id): + """Adds a job ID to the internal queue. + + """ + self.jobs.append((self._counter.next(), status, job_id, name)) def SubmitPending(self, each=False): """Submit all pending jobs. @@ -2909,14 +3150,13 @@ class JobExecutor(object): """ if each: results = [] - for row in self.queue: + for (_, _, ops) in self.queue: # SubmitJob will remove the success status, but raise an exception if # the submission fails, so we'll notice that anyway. - results.append([True, self.cl.SubmitJob(row[1])]) + results.append([True, self.cl.SubmitJob(ops)[0]]) else: - results = self.cl.SubmitManyJobs([row[1] for row in self.queue]) - for (idx, ((status, data), (name, _))) in enumerate(zip(results, - self.queue)): + results = self.cl.SubmitManyJobs([ops for (_, _, ops) in self.queue]) + for ((status, data), (idx, name, _)) in zip(results, self.queue): self.jobs.append((idx, status, data, name)) def _ChooseJob(self): @@ -2925,13 +3165,14 @@ class JobExecutor(object): """ assert self.jobs, "_ChooseJob called with empty job list" - result = self.cl.QueryJobs([i[2] for i in self.jobs], ["status"]) + result = self.cl.QueryJobs([i[2] for i in self.jobs[:_CHOOSE_BATCH]], + ["status"]) assert result for job_data, status in zip(self.jobs, result): if (isinstance(status, list) and status and status[0] in (constants.JOB_STATUS_QUEUED, - constants.JOB_STATUS_WAITLOCK, + constants.JOB_STATUS_WAITING, constants.JOB_STATUS_CANCELING)): # job is still present and waiting continue @@ -2962,25 +3203,26 @@ class JobExecutor(object): # first, remove any non-submitted jobs self.jobs, failures = compat.partition(self.jobs, lambda x: x[1]) for idx, _, jid, name in failures: - ToStderr("Failed to submit job for %s: %s", name, jid) + ToStderr("Failed to submit job%s: %s", self._IfName(name, " for %s"), jid) results.append((idx, False, jid)) while self.jobs: (idx, _, jid, name) = self._ChooseJob() - ToStdout("Waiting for job %s for %s...", jid, name) + ToStdout("Waiting for job %s%s ...", jid, self._IfName(name, " for %s")) try: job_result = PollJob(jid, cl=self.cl, feedback_fn=self.feedback_fn) success = True except errors.JobLost, err: _, job_result = FormatError(err) - ToStderr("Job %s for %s has been archived, cannot check its result", - jid, name) + ToStderr("Job %s%s has been archived, cannot check its result", + jid, self._IfName(name, " for %s")) success = False except (errors.GenericError, luxi.ProtocolError), err: _, job_result = FormatError(err) success = False # the error message will always be shown, verbose or not - ToStderr("Job %s for %s has failed: %s", jid, name, job_result) + ToStderr("Job %s%s has failed: %s", + jid, self._IfName(name, " for %s"), job_result) results.append((idx, success, job_result))