import logging
import errno
import itertools
+import shlex
+import optparse
from cStringIO import StringIO
from ganeti import utils
from ganeti import netutils
from ganeti import qlang
-from optparse import (OptionParser, TitledHelpFormatter,
+from optparse import (TitledHelpFormatter,
Option, OptionValueError)
"DEBUG_SIMERR_OPT",
"DISKIDX_OPT",
"DISK_OPT",
+ "DISK_PARAMS_OPT",
"DISK_TEMPLATE_OPT",
"DRAINED_OPT",
"DRY_RUN_OPT",
"DEFAULT_IALLOCATOR_OPT",
"IDENTIFY_DEFAULTS_OPT",
"IGNORE_CONSIST_OPT",
+ "IGNORE_ERRORS_OPT",
"IGNORE_FAILURES_OPT",
"IGNORE_OFFLINE_OPT",
"IGNORE_REMOVE_FAILURES_OPT",
"MAC_PREFIX_OPT",
"MAINTAIN_NODE_HEALTH_OPT",
"MASTER_NETDEV_OPT",
+ "MASTER_NETMASK_OPT",
"MC_OPT",
"MIGRATION_MODE_OPT",
"NET_OPT",
"NEW_CONFD_HMAC_KEY_OPT",
"NEW_RAPI_CERT_OPT",
"NEW_SECONDARY_OPT",
+ "NEW_SPICE_CERT_OPT",
"NIC_PARAMS_OPT",
"NODE_FORCE_JOIN_OPT",
"NODE_LIST_OPT",
"NOVOTING_OPT",
"NO_REMEMBER_OPT",
"NWSYNC_OPT",
+ "OFFLINE_INST_OPT",
+ "ONLINE_INST_OPT",
"ON_PRIMARY_OPT",
"ON_SECONDARY_OPT",
"OFFLINE_OPT",
"REMOVE_INSTANCE_OPT",
"REMOVE_UIDS_OPT",
"RESERVED_LVS_OPT",
+ "RUNTIME_MEM_OPT",
"ROMAN_OPT",
"SECONDARY_IP_OPT",
"SECONDARY_ONLY_OPT",
"SHOWCMD_OPT",
"SHUTDOWN_TIMEOUT_OPT",
"SINGLE_NODE_OPT",
+ "SPECS_CPU_COUNT_OPT",
+ "SPECS_DISK_COUNT_OPT",
+ "SPECS_DISK_SIZE_OPT",
+ "SPECS_MEM_SIZE_OPT",
+ "SPECS_NIC_COUNT_OPT",
+ "SPECS_DISK_TEMPLATES",
+ "SPICE_CACERT_OPT",
+ "SPICE_CERT_OPT",
"SRC_DIR_OPT",
"SRC_NODE_OPT",
"SUBMIT_OPT",
"TO_GROUP_OPT",
"UIDPOOL_OPT",
"USEUNITS_OPT",
+ "USE_EXTERNAL_MIP_SCRIPT",
"USE_REPL_NET_OPT",
"VERBOSE_OPT",
"VG_NAME_OPT",
"YES_DOIT_OPT",
+ "DISK_STATE_OPT",
+ "HV_STATE_OPT",
+ "IGNORE_IPOLICY_OPT",
+ "INSTANCE_POLICY_OPTS",
# Generic functions for CLI programs
"ConfirmOperation",
"GenericMain",
QR_UNKNOWN,
QR_INCOMPLETE) = range(3)
+#: Maximum batch size for ChooseJob
+_CHOOSE_BATCH = 25
+
class _Argument:
- def __init__(self, min=0, max=None): # pylint: disable-msg=W0622
+ def __init__(self, min=0, max=None): # pylint: disable=W0622
self.min = min
self.max = max
Value can be any of the ones passed to the constructor.
"""
- # pylint: disable-msg=W0622
+ # pylint: disable=W0622
def __init__(self, min=0, max=None, choices=None):
_Argument.__init__(self, min=min, max=max)
self.choices = choices
SubmitOpCode(op, opts=opts)
-def check_unit(option, opt, value): # pylint: disable-msg=W0613
+def check_unit(option, opt, value): # pylint: disable=W0613
"""OptParsers custom converter for units.
"""
return kv_dict
-def check_ident_key_val(option, opt, value): # pylint: disable-msg=W0613
+def check_ident_key_val(option, opt, value): # pylint: disable=W0613
"""Custom parser for ident:key=val,key=val options.
This will store the parsed values as a tuple (ident, {key: val}). As such,
return retval
-def check_key_val(option, opt, value): # pylint: disable-msg=W0613
+def check_key_val(option, opt, value): # pylint: disable=W0613
"""Custom parser class for key=val,key=val options.
This will store the parsed values as a dict {key: val}.
return _SplitKeyVal(opt, value)
-def check_bool(option, opt, value): # pylint: disable-msg=W0613
+def check_bool(option, opt, value): # pylint: disable=W0613
"""Custom parser for yes/no options.
This will store the parsed value as either True or False.
raise errors.ParameterError("Invalid boolean value '%s'" % value)
+def check_list(option, opt, value): # pylint: disable=W0613
+ """Custom parser for comma-separated lists.
+
+ """
+ # we have to make this explicit check since "".split(",") is [""],
+ # not an empty list :(
+ if not value:
+ return []
+ else:
+ return utils.UnescapeAndSplit(value)
+
+
# completion_suggestion is normally a list. Using numeric values not evaluating
# to False for dynamic completion.
(OPT_COMPL_MANY_NODES,
"keyval",
"unit",
"bool",
+ "list",
)
TYPE_CHECKER = Option.TYPE_CHECKER.copy()
TYPE_CHECKER["identkeyval"] = check_ident_key_val
TYPE_CHECKER["keyval"] = check_key_val
TYPE_CHECKER["unit"] = check_unit
TYPE_CHECKER["bool"] = check_bool
+ TYPE_CHECKER["list"] = check_list
# optparse.py sets make_option, so we do it for our own option class, too
default=True, action="store_false",
help="Don't wait for sync (DANGEROUS!)")
+ONLINE_INST_OPT = cli_option("--online", dest="online_inst",
+ action="store_true", default=False,
+ help="Enable offline instance")
+
+OFFLINE_INST_OPT = cli_option("--offline", dest="offline_inst",
+ action="store_true", default=False,
+ help="Disable down instance")
+
DISK_TEMPLATE_OPT = cli_option("-t", "--disk-template", dest="disk_template",
help=("Custom disk setup (%s)" %
utils.CommaJoin(constants.DISK_TEMPLATES)),
type="keyval", default={},
help="Backend parameters")
-HVOPTS_OPT = cli_option("-H", "--hypervisor-parameters", type="keyval",
- default={}, dest="hvparams",
- help="Hypervisor parameters")
+HVOPTS_OPT = cli_option("-H", "--hypervisor-parameters", type="keyval",
+ default={}, dest="hvparams",
+ help="Hypervisor parameters")
+
+DISK_PARAMS_OPT = cli_option("-D", "--disk-parameters", dest="diskparams",
+ help="Disk template parameters, in the format"
+ " template:option=value,option=value,...",
+ type="identkeyval", action="append", default=[])
+
+SPECS_MEM_SIZE_OPT = cli_option("--specs-mem-size", dest="ispecs_mem_size",
+ type="keyval", default={},
+ help="Memory count specs: min, max, std"
+ " (in MB)")
+
+SPECS_CPU_COUNT_OPT = cli_option("--specs-cpu-count", dest="ispecs_cpu_count",
+ type="keyval", default={},
+ help="CPU count specs: min, max, std")
+
+SPECS_DISK_COUNT_OPT = cli_option("--specs-disk-count",
+ dest="ispecs_disk_count",
+ type="keyval", default={},
+ help="Disk count specs: min, max, std")
+
+SPECS_DISK_SIZE_OPT = cli_option("--specs-disk-size", dest="ispecs_disk_size",
+ type="keyval", default={},
+ help="Disk size specs: min, max, std (in MB)")
+
+SPECS_NIC_COUNT_OPT = cli_option("--specs-nic-count", dest="ispecs_nic_count",
+ type="keyval", default={},
+ help="NIC count specs: min, max, std")
+
+SPECS_DISK_TEMPLATES = cli_option("--specs-disk-templates",
+ dest="ispecs_disk_templates",
+ type="list", default=None,
+ help="Comma-separated list of"
+ " enabled disk templates")
HYPERVISOR_OPT = cli_option("-H", "--hypervisor-parameters", dest="hypervisor",
help="Hypervisor and hypervisor options, in the"
" times, if not given defaults to all nodes)",
completion_suggest=OPT_COMPL_ONE_NODE)
-NODEGROUP_OPT = cli_option("-g", "--node-group",
+NODEGROUP_OPT_NAME = "--node-group"
+NODEGROUP_OPT = cli_option("-g", NODEGROUP_OPT_NAME,
dest="nodegroup",
help="Node group (name or uuid)",
metavar="<nodegroup>",
metavar="NETDEV",
default=None)
+MASTER_NETMASK_OPT = cli_option("--master-netmask", dest="master_netmask",
+ help="Specify the netmask of the master IP",
+ metavar="NETMASK",
+ default=None)
+
+USE_EXTERNAL_MIP_SCRIPT = cli_option("--use-external-mip-script",
+ dest="use_external_mip_script",
+ help="Specify whether to run a user-provided"
+ " script for the master IP address turnup and"
+ " turndown operations",
+ type="bool", metavar=_YORNO, default=None)
+
GLOBAL_FILEDIR_OPT = cli_option("--file-storage-dir", dest="file_storage_dir",
help="Specify the default directory (cluster-"
"wide) for storing the file-based disks [%s]" %
help=("Generate a new self-signed RAPI"
" certificate"))
+SPICE_CERT_OPT = cli_option("--spice-certificate", dest="spice_cert",
+ default=None,
+ help="File containing new SPICE certificate")
+
+SPICE_CACERT_OPT = cli_option("--spice-ca-certificate", dest="spice_cacert",
+ default=None,
+ help="File containing the certificate of the CA"
+ " which signed the SPICE certificate")
+
+NEW_SPICE_CERT_OPT = cli_option("--new-spice-certificate",
+ dest="new_spice_cert", default=None,
+ action="store_true",
+ help=("Generate a new self-signed SPICE"
+ " certificate"))
+
NEW_CONFD_HMAC_KEY_OPT = cli_option("--new-confd-hmac-key",
dest="new_confd_hmac_key",
default=False, action="store_true",
default=None, action="append",
completion_suggest=OPT_COMPL_ONE_NODEGROUP)
+IGNORE_ERRORS_OPT = cli_option("-I", "--ignore-errors", default=[],
+ action="append", dest="ignore_errors",
+ choices=list(constants.CV_ALL_ECODES_STRINGS),
+ help="Error code to be ignored")
+
+DISK_STATE_OPT = cli_option("--disk-state", default=[], dest="disk_state",
+ action="append",
+ help=("Specify disk state information in the format"
+ " storage_type/identifier:option=value,..."),
+ type="identkeyval")
+
+HV_STATE_OPT = cli_option("--hypervisor-state", default=[], dest="hv_state",
+ action="append",
+ help=("Specify hypervisor state information in the"
+ " format hypervisor:option=value,..."),
+ type="identkeyval")
+
+IGNORE_IPOLICY_OPT = cli_option("--ignore-ipolicy", dest="ignore_ipolicy",
+ action="store_true", default=False,
+ help="Ignore instance policy violations")
+
+RUNTIME_MEM_OPT = cli_option("-m", "--runtime-memory", dest="runtime_mem",
+ help="Sets the instance's runtime memory,"
+ " ballooning it up or down to the new value",
+ default=None, type="unit", metavar="<size>")
#: Options provided by all commands
COMMON_OPTS = [DEBUG_OPT]
PRIORITY_OPT,
]
+# common instance policy options
+INSTANCE_POLICY_OPTS = [
+ SPECS_CPU_COUNT_OPT,
+ SPECS_DISK_COUNT_OPT,
+ SPECS_DISK_SIZE_OPT,
+ SPECS_MEM_SIZE_OPT,
+ SPECS_NIC_COUNT_OPT,
+ SPECS_DISK_TEMPLATES,
+ ]
-def _ParseArgs(argv, commands, aliases):
+
+def _ParseArgs(argv, commands, aliases, env_override):
"""Parser for the command line arguments.
This function parses the arguments and returns the function which
@param commands: dictionary with special contents, see the design
doc for cmdline handling
@param aliases: dictionary with command aliases {'alias': 'target, ...}
+ @param env_override: list of env variables allowed for default args
"""
+ assert not (env_override - set(commands))
+
if len(argv) == 0:
binary = "<command>"
else:
cmd = aliases[cmd]
+ if cmd in env_override:
+ args_env_name = ("%s_%s" % (binary.replace("-", "_"), cmd)).upper()
+ env_args = os.environ.get(args_env_name)
+ if env_args:
+ argv = utils.InsertAtPos(argv, 1, shlex.split(env_args))
+
func, args_def, parser_opts, usage, description = commands[cmd]
- parser = OptionParser(option_list=parser_opts + COMMON_OPTS,
- description=description,
- formatter=TitledHelpFormatter(),
- usage="%%prog %s %s" % (cmd, usage))
+ parser = CustomOptionParser(option_list=parser_opts + COMMON_OPTS,
+ description=description,
+ formatter=TitledHelpFormatter(),
+ usage="%%prog %s %s" % (cmd, usage))
parser.disable_interspersed_args()
- options, args = parser.parse_args()
+ options, args = parser.parse_args(args=argv[1:])
if not _CheckArguments(cmd, args_def, args):
return None, None, None
return func, options, args
+class CustomOptionParser(optparse.OptionParser):
+ def _match_long_opt(self, opt):
+ """Override C{OptionParser}'s function for matching long options.
+
+ The default implementation does prefix-based abbreviation matching. We
+ disable such behaviour as it can can lead to confusing conflicts (e.g.
+ C{--force} and C{--force-multi}).
+
+ """
+ if opt in self._long_opt:
+ return opt
+ else:
+ raise optparse.BadOptionError(opt)
+
+
def _CheckArguments(cmd, args_def, args):
"""Verifies the arguments using the argument definition.
return retcode, obuf.getvalue().rstrip("\n")
-def GenericMain(commands, override=None, aliases=None):
+def GenericMain(commands, override=None, aliases=None,
+ env_override=frozenset()):
"""Generic main function for all the gnt-* commands.
- Arguments:
- - commands: a dictionary with a special structure, see the design doc
- for command line handling.
- - override: if not None, we expect a dictionary with keys that will
- override command line options; this can be used to pass
- options from the scripts to generic functions
- - aliases: dictionary with command aliases {'alias': 'target, ...}
+ @param commands: a dictionary with a special structure, see the design doc
+ for command line handling.
+ @param override: if not None, we expect a dictionary with keys that will
+ override command line options; this can be used to pass
+ options from the scripts to generic functions
+ @param aliases: dictionary with command aliases {'alias': 'target, ...}
+ @param env_override: list of environment names which are allowed to submit
+ default args for commands
"""
# save the program name and the entire command line for later logging
aliases = {}
try:
- func, options, args = _ParseArgs(sys.argv, commands, aliases)
+ func, options, args = _ParseArgs(sys.argv, commands, aliases, env_override)
except errors.ParameterError, err:
result, err_msg = FormatError(err)
ToStderr(err_msg)
else:
tags = []
- utils.ForceDictType(opts.beparams, constants.BES_PARAMETER_TYPES)
+ utils.ForceDictType(opts.beparams, constants.BES_PARAMETER_COMPAT)
utils.ForceDictType(hvparams, constants.HVS_PARAMETER_TYPES)
if mode == constants.INSTANCE_CREATE:
src_path=src_path,
tags=tags,
no_install=no_install,
- identify_defaults=identify_defaults)
+ identify_defaults=identify_defaults,
+ ignore_ipolicy=opts.ignore_ipolicy)
SubmitOrSend(op, opts)
return 0
"""
# Pause watcher by acquiring an exclusive lock on watcher state file
self.feedback_fn("Blocking watcher")
- watcher_block = utils.FileLock.Open(constants.WATCHER_STATEFILE)
+ watcher_block = utils.FileLock.Open(constants.WATCHER_LOCK_FILE)
try:
# TODO: Currently, this just blocks. There's no timeout.
# TODO: Should it be a shared lock?
if unitfields is None:
unitfields = []
- numfields = utils.FieldSet(*numfields) # pylint: disable-msg=W0142
- unitfields = utils.FieldSet(*unitfields) # pylint: disable-msg=W0142
+ numfields = utils.FieldSet(*numfields) # pylint: disable=W0142
+ unitfields = utils.FieldSet(*unitfields) # pylint: disable=W0142
format_fields = []
for field in fields:
@param verbose: whether to use verbose field descriptions or not
"""
- if cl is None:
- cl = GetClient()
-
if not names:
names = None
- if (force_filter or
- (names and len(names) == 1 and qlang.MaybeFilter(names[0]))):
- try:
- (filter_text, ) = names
- except ValueError:
- raise errors.OpPrereqError("Exactly one argument must be given as a"
- " filter")
+ qfilter = qlang.MakeFilter(names, force_filter)
- logging.debug("Parsing '%s' as filter", filter_text)
- filter_ = qlang.ParseFilter(filter_text)
- else:
- filter_ = qlang.MakeSimpleFilter("name", names)
+ if cl is None:
+ cl = GetClient()
- response = cl.Query(resource, fields, filter_)
+ response = cl.Query(resource, fields, qfilter)
found_unknown = _WarnUnknownFields(response.fields)
@return: a string with the formatted timestamp
"""
- if not isinstance (ts, (tuple, list)) or len(ts) != 2:
+ if not isinstance(ts, (tuple, list)) or len(ts) != 2:
return "?"
sec, usec = ts
return time.strftime("%F %T", time.localtime(sec)) + ".%06d" % usec
if cl is None:
cl = GetClient()
- filter_ = []
+ qfilter = []
if nodes:
- filter_.append(qlang.MakeSimpleFilter("name", nodes))
+ qfilter.append(qlang.MakeSimpleFilter("name", nodes))
if nodegroup is not None:
- filter_.append([qlang.OP_OR, [qlang.OP_EQUAL, "group", nodegroup],
+ qfilter.append([qlang.OP_OR, [qlang.OP_EQUAL, "group", nodegroup],
[qlang.OP_EQUAL, "group.uuid", nodegroup]])
if filter_master:
- filter_.append([qlang.OP_NOT, [qlang.OP_TRUE, "master"]])
+ qfilter.append([qlang.OP_NOT, [qlang.OP_TRUE, "master"]])
- if filter_:
- if len(filter_) > 1:
- final_filter = [qlang.OP_AND] + filter_
+ if qfilter:
+ if len(qfilter) > 1:
+ final_filter = [qlang.OP_AND] + qfilter
else:
- assert len(filter_) == 1
- final_filter = filter_[0]
+ assert len(qfilter) == 1
+ final_filter = qfilter[0]
else:
final_filter = None
self.jobs = []
self.opts = opts
self.feedback_fn = feedback_fn
- if feedback_fn:
- self._info_fn = feedback_fn
- else:
- self._info_fn = ToStdout
self._counter = itertools.count()
@staticmethod
for (_, _, ops) in self.queue:
# SubmitJob will remove the success status, but raise an exception if
# the submission fails, so we'll notice that anyway.
- results.append([True, self.cl.SubmitJob(ops)])
+ results.append([True, self.cl.SubmitJob(ops)[0]])
else:
results = self.cl.SubmitManyJobs([ops for (_, _, ops) in self.queue])
for ((status, data), (idx, name, _)) in zip(results, self.queue):
"""
assert self.jobs, "_ChooseJob called with empty job list"
- result = self.cl.QueryJobs([i[2] for i in self.jobs], ["status"])
+ result = self.cl.QueryJobs([i[2] for i in self.jobs[:_CHOOSE_BATCH]],
+ ["status"])
assert result
for job_data, status in zip(self.jobs, result):
if self.verbose:
ok_jobs = [row[2] for row in self.jobs if row[1]]
if ok_jobs:
- self._info_fn("Submitted jobs %s", utils.CommaJoin(ok_jobs))
+ ToStdout("Submitted jobs %s", utils.CommaJoin(ok_jobs))
# first, remove any non-submitted jobs
self.jobs, failures = compat.partition(self.jobs, lambda x: x[1])
for idx, _, jid, name in failures:
- self._info_fn("Failed to submit job%s: %s",
- self._IfName(name, " for %s"), jid)
+ ToStderr("Failed to submit job%s: %s", self._IfName(name, " for %s"), jid)
results.append((idx, False, jid))
while self.jobs:
(idx, _, jid, name) = self._ChooseJob()
- self._info_fn("Waiting for job %s%s ...",
- jid, self._IfName(name, " for %s"))
+ ToStdout("Waiting for job %s%s ...", jid, self._IfName(name, " for %s"))
try:
job_result = PollJob(jid, cl=self.cl, feedback_fn=self.feedback_fn)
success = True
except errors.JobLost, err:
_, job_result = FormatError(err)
- self._info_fn("Job %s%s has been archived, cannot check its result",
- jid, self._IfName(name, " for %s"))
+ ToStderr("Job %s%s has been archived, cannot check its result",
+ jid, self._IfName(name, " for %s"))
success = False
except (errors.GenericError, luxi.ProtocolError), err:
_, job_result = FormatError(err)
success = False
# the error message will always be shown, verbose or not
- self._info_fn("Job %s%s has failed: %s",
- jid, self._IfName(name, " for %s"), job_result)
+ ToStderr("Job %s%s has failed: %s",
+ jid, self._IfName(name, " for %s"), job_result)
results.append((idx, success, job_result))
self.SubmitPending()
for _, status, result, name in self.jobs:
if status:
- self._info_fn("%s: %s", result, name)
+ ToStdout("%s: %s", result, name)
else:
- self._info_fn("Failure for %s: %s", name, result)
+ ToStderr("Failure for %s: %s", name, result)
return [row[1:3] for row in self.jobs]
projects / ganeti-local / blobdiff