X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/bafc1d9018a4d4b9b3fcffe8935a10ab5b973871..4dfd62661512563329e80bd5600714fab4e4f41f:/lib/cmdlib.py diff --git a/lib/cmdlib.py b/lib/cmdlib.py index 0b94d64..87ce76d 100644 --- a/lib/cmdlib.py +++ b/lib/cmdlib.py @@ -25,14 +25,11 @@ import os import os.path -import sha import time -import tempfile import re import platform import logging import copy -import random from ganeti import ssh from ganeti import utils @@ -41,7 +38,6 @@ from ganeti import hypervisor from ganeti import locking from ganeti import constants from ganeti import objects -from ganeti import opcodes from ganeti import serializer from ganeti import ssconf @@ -69,7 +65,7 @@ class LogicalUnit(object): def __init__(self, processor, op, context, rpc): """Constructor for LogicalUnit. - This needs to be overriden in derived classes in order to check op + This needs to be overridden in derived classes in order to check op validity. """ @@ -117,7 +113,7 @@ class LogicalUnit(object): CheckPrereq, doing these separate is better because: - ExpandNames is left as as purely a lock-related function - - CheckPrereq is run after we have aquired locks (and possible + - CheckPrereq is run after we have acquired locks (and possible waited for them) The function is allowed to change the self.op attribute so that @@ -434,15 +430,28 @@ def _CheckNodeOnline(lu, node): @param lu: the LU on behalf of which we make the check @param node: the node to check - @raise errors.OpPrereqError: if the nodes is offline + @raise errors.OpPrereqError: if the node is offline """ if lu.cfg.GetNodeInfo(node).offline: raise errors.OpPrereqError("Can't use offline node %s" % node) +def _CheckNodeNotDrained(lu, node): + """Ensure that a given node is not drained. + + @param lu: the LU on behalf of which we make the check + @param node: the node to check + @raise errors.OpPrereqError: if the node is drained + + """ + if lu.cfg.GetNodeInfo(node).drained: + raise errors.OpPrereqError("Can't use drained node %s" % node) + + def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status, - memory, vcpus, nics): + memory, vcpus, nics, disk_template, disks, + bep, hvp, hypervisor_name): """Builds instance related env variables for hooks This builds the hook environment from individual variables. @@ -464,6 +473,16 @@ def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status, @type nics: list @param nics: list of tuples (ip, bridge, mac) representing the NICs the instance has + @type disk_template: string + @param disk_template: the disk template of the instance + @type disks: list + @param disks: the list of (size, mode) pairs + @type bep: dict + @param bep: the backend parameters for the instance + @type hvp: dict + @param hvp: the hypervisor parameters for the instance + @type hypervisor_name: string + @param hypervisor_name: the hypervisor for the instance @rtype: dict @return: the hook environment for this instance @@ -481,6 +500,8 @@ def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status, "INSTANCE_STATUS": str_status, "INSTANCE_MEMORY": memory, "INSTANCE_VCPUS": vcpus, + "INSTANCE_DISK_TEMPLATE": disk_template, + "INSTANCE_HYPERVISOR": hypervisor_name, } if nics: @@ -490,12 +511,26 @@ def _BuildInstanceHookEnv(name, primary_node, secondary_nodes, os_type, status, ip = "" env["INSTANCE_NIC%d_IP" % idx] = ip env["INSTANCE_NIC%d_BRIDGE" % idx] = bridge - env["INSTANCE_NIC%d_HWADDR" % idx] = mac + env["INSTANCE_NIC%d_MAC" % idx] = mac else: nic_count = 0 env["INSTANCE_NIC_COUNT"] = nic_count + if disks: + disk_count = len(disks) + for idx, (size, mode) in enumerate(disks): + env["INSTANCE_DISK%d_SIZE" % idx] = size + env["INSTANCE_DISK%d_MODE" % idx] = mode + else: + disk_count = 0 + + env["INSTANCE_DISK_COUNT"] = disk_count + + for source, kind in [(bep, "BE"), (hvp, "HV")]: + for key, value in source.items(): + env["INSTANCE_%s_%s" % (kind, key)] = value + return env @@ -514,7 +549,9 @@ def _BuildInstanceHookEnvByObject(lu, instance, override=None): @return: the hook environment dictionary """ - bep = lu.cfg.GetClusterInfo().FillBE(instance) + cluster = lu.cfg.GetClusterInfo() + bep = cluster.FillBE(instance) + hvp = cluster.FillHV(instance) args = { 'name': instance.name, 'primary_node': instance.primary_node, @@ -524,6 +561,11 @@ def _BuildInstanceHookEnvByObject(lu, instance, override=None): 'memory': bep[constants.BE_MEMORY], 'vcpus': bep[constants.BE_VCPUS], 'nics': [(nic.ip, nic.bridge, nic.mac) for nic in instance.nics], + 'disk_template': instance.disk_template, + 'disks': [(disk.size, disk.mode) for disk in instance.disks], + 'bep': bep, + 'hvp': hvp, + 'hypervisor_name': instance.hypervisor, } if override: args.update(override) @@ -547,10 +589,10 @@ def _AdjustCandidatePool(lu): def _CheckInstanceBridgesExist(lu, instance): - """Check that the brigdes needed by an instance exist. + """Check that the bridges needed by an instance exist. """ - # check bridges existance + # check bridges existence brlist = [nic.bridge for nic in instance.nics] result = lu.rpc.call_bridges_exist(instance.primary_node, brlist) result.Raise() @@ -571,7 +613,7 @@ class LUDestroyCluster(NoHooksLU): This checks whether the cluster is empty. - Any errors are signalled by raising errors.OpPrereqError. + Any errors are signaled by raising errors.OpPrereqError. """ master = self.cfg.GetMasterNode() @@ -618,13 +660,13 @@ class LUVerifyCluster(LogicalUnit): def _VerifyNode(self, nodeinfo, file_list, local_cksum, node_result, feedback_fn, master_files, - drbd_map): + drbd_map, vg_name): """Run multiple tests against a node. Test list: - compares ganeti version - - checks vg existance and size > 20G + - checks vg existence and size > 20G - checks config file checksum - checks ssh to other nodes @@ -638,6 +680,7 @@ class LUVerifyCluster(LogicalUnit): @param drbd_map: the useddrbd minors for this node, in form of minor: (instance, must_exist) which correspond to instances and their running status + @param vg_name: Ganeti Volume Group (result of self.cfg.GetVGName()) """ node = nodeinfo.name @@ -671,18 +714,18 @@ class LUVerifyCluster(LogicalUnit): (constants.RELEASE_VERSION, node, remote_version[1])) # checks vg existence and size > 20G - - vglist = node_result.get(constants.NV_VGLIST, None) - if not vglist: - feedback_fn(" - ERROR: unable to check volume groups on node %s." % - (node,)) - bad = True - else: - vgstatus = utils.CheckVolumeGroupSize(vglist, self.cfg.GetVGName(), - constants.MIN_VG_SIZE) - if vgstatus: - feedback_fn(" - ERROR: %s on node %s" % (vgstatus, node)) + if vg_name is not None: + vglist = node_result.get(constants.NV_VGLIST, None) + if not vglist: + feedback_fn(" - ERROR: unable to check volume groups on node %s." % + (node,)) bad = True + else: + vgstatus = utils.CheckVolumeGroupSize(vglist, vg_name, + constants.MIN_VG_SIZE) + if vgstatus: + feedback_fn(" - ERROR: %s on node %s" % (vgstatus, node)) + bad = True # checks config file checksum @@ -705,8 +748,8 @@ class LUVerifyCluster(LogicalUnit): else: # not candidate and this is not a must-have file bad = True - feedback_fn(" - ERROR: non master-candidate has old/wrong file" - " '%s'" % file_name) + feedback_fn(" - ERROR: file '%s' should not exist on non master" + " candidates (and the file is outdated)" % file_name) else: # all good, except non-master/non-must have combination if not node_is_mc and not must_have_file: @@ -744,16 +787,22 @@ class LUVerifyCluster(LogicalUnit): (hv_name, hv_result)) # check used drbd list - used_minors = node_result.get(constants.NV_DRBDLIST, []) - for minor, (iname, must_exist) in drbd_map.items(): - if minor not in used_minors and must_exist: - feedback_fn(" - ERROR: drbd minor %d of instance %s is not active" % - (minor, iname)) - bad = True - for minor in used_minors: - if minor not in drbd_map: - feedback_fn(" - ERROR: unallocated drbd minor %d is in use" % minor) - bad = True + if vg_name is not None: + used_minors = node_result.get(constants.NV_DRBDLIST, []) + if not isinstance(used_minors, (tuple, list)): + feedback_fn(" - ERROR: cannot parse drbd status file: %s" % + str(used_minors)) + else: + for minor, (iname, must_exist) in drbd_map.items(): + if minor not in used_minors and must_exist: + feedback_fn(" - ERROR: drbd minor %d of instance %s is" + " not active" % (minor, iname)) + bad = True + for minor in used_minors: + if minor not in drbd_map: + feedback_fn(" - ERROR: unallocated drbd minor %d is in use" % + minor) + bad = True return bad @@ -856,7 +905,7 @@ class LUVerifyCluster(LogicalUnit): if bep[constants.BE_AUTO_BALANCE]: needed_mem += bep[constants.BE_MEMORY] if nodeinfo['mfree'] < needed_mem: - feedback_fn(" - ERROR: not enough memory on node %s to accomodate" + feedback_fn(" - ERROR: not enough memory on node %s to accommodate" " failovers should node %s fail" % (node, prinode)) bad = True return bad @@ -875,13 +924,17 @@ class LUVerifyCluster(LogicalUnit): def BuildHooksEnv(self): """Build hooks env. - Cluster-Verify hooks just rone in the post phase and their failure makes + Cluster-Verify hooks just ran in the post phase and their failure makes the output be logged in the verify output and the verification to fail. """ all_nodes = self.cfg.GetNodeList() - # TODO: populate the environment with useful information for verify hooks - env = {} + env = { + "CLUSTER_TAGS": " ".join(self.cfg.GetClusterInfo().GetTags()) + } + for node in self.cfg.GetAllNodesInfo().values(): + env["NODE_TAGS_%s" % node.name] = " ".join(node.GetTags()) + return env, [], all_nodes def Exec(self, feedback_fn): @@ -903,6 +956,7 @@ class LUVerifyCluster(LogicalUnit): i_non_redundant = [] # Non redundant instances i_non_a_balanced = [] # Non auto-balanced instances n_offline = [] # List of offline nodes + n_drained = [] # List of nodes being drained node_volume = {} node_instance = {} node_info = {} @@ -928,13 +982,14 @@ class LUVerifyCluster(LogicalUnit): constants.NV_NODENETTEST: [(node.name, node.primary_ip, node.secondary_ip) for node in nodeinfo if not node.offline], - constants.NV_LVLIST: vg_name, constants.NV_INSTANCELIST: hypervisors, - constants.NV_VGLIST: None, constants.NV_VERSION: None, constants.NV_HVINFO: self.cfg.GetHypervisorType(), - constants.NV_DRBDLIST: None, } + if vg_name is not None: + node_verify_param[constants.NV_VGLIST] = None + node_verify_param[constants.NV_LVLIST] = vg_name + node_verify_param[constants.NV_DRBDLIST] = None all_nvinfo = self.rpc.call_node_verify(nodelist, node_verify_param, self.cfg.GetClusterName()) @@ -955,6 +1010,9 @@ class LUVerifyCluster(LogicalUnit): ntype = "master" elif node_i.master_candidate: ntype = "master candidate" + elif node_i.drained: + ntype = "drained" + n_drained.append(node) else: ntype = "regular" feedback_fn("* Verifying node %s (%s)" % (node, ntype)) @@ -966,17 +1024,27 @@ class LUVerifyCluster(LogicalUnit): node_drbd = {} for minor, instance in all_drbd_map[node].items(): - instance = instanceinfo[instance] - node_drbd[minor] = (instance.name, instance.admin_up) + if instance not in instanceinfo: + feedback_fn(" - ERROR: ghost instance '%s' in temporary DRBD map" % + instance) + # ghost instance should not be running, but otherwise we + # don't give double warnings (both ghost instance and + # unallocated minor in use) + node_drbd[minor] = (instance, False) + else: + instance = instanceinfo[instance] + node_drbd[minor] = (instance.name, instance.admin_up) result = self._VerifyNode(node_i, file_names, local_checksums, nresult, feedback_fn, master_files, - node_drbd) + node_drbd, vg_name) bad = bad or result lvdata = nresult.get(constants.NV_LVLIST, "Missing LV data") - if isinstance(lvdata, basestring): + if vg_name is None: + node_volume[node] = {} + elif isinstance(lvdata, basestring): feedback_fn(" - ERROR: LVM problem on node %s: %s" % - (node, lvdata.encode('string_escape'))) + (node, utils.SafeEncode(lvdata))) bad = True node_volume[node] = {} elif not isinstance(lvdata, dict): @@ -1006,7 +1074,6 @@ class LUVerifyCluster(LogicalUnit): try: node_info[node] = { "mfree": int(nodeinfo['memory_free']), - "dfree": int(nresult[constants.NV_VGLIST][vg_name]), "pinst": [], "sinst": [], # dictionary holding all instances this node is secondary for, @@ -1017,8 +1084,19 @@ class LUVerifyCluster(LogicalUnit): # secondary. "sinst-by-pnode": {}, } - except ValueError: - feedback_fn(" - ERROR: invalid value returned from node %s" % (node,)) + # FIXME: devise a free space model for file based instances as well + if vg_name is not None: + if (constants.NV_VGLIST not in nresult or + vg_name not in nresult[constants.NV_VGLIST]): + feedback_fn(" - ERROR: node %s didn't return data for the" + " volume group '%s' - it is either missing or broken" % + (node, vg_name)) + bad = True + continue + node_info[node]["dfree"] = int(nresult[constants.NV_VGLIST][vg_name]) + except (ValueError, KeyError): + feedback_fn(" - ERROR: invalid nodeinfo value returned" + " from node %s" % (node,)) bad = True continue @@ -1107,10 +1185,13 @@ class LUVerifyCluster(LogicalUnit): if n_offline: feedback_fn(" - NOTICE: %d offline node(s) found." % len(n_offline)) + if n_drained: + feedback_fn(" - NOTICE: %d drained node(s) found." % len(n_drained)) + return not bad def HooksCallBack(self, phase, hooks_results, feedback_fn, lu_result): - """Analize the post-hooks' result + """Analyze the post-hooks' result This method analyses the hook result, handles it, and sends some nicely-formatted feedback back to the user. @@ -1209,7 +1290,6 @@ class LUVerifyDisks(NoHooksLU): node_lvs = self.rpc.call_volume_list(nodes, vg_name) - to_act = set() for node in nodes: # node_volume lvs = node_lvs[node] @@ -1222,6 +1302,7 @@ class LUVerifyDisks(NoHooksLU): if isinstance(lvs, basestring): logging.warning("Error enumerating LVs on node %s: %s", node, lvs) res_nlvm[node] = lvs + continue elif not isinstance(lvs, dict): logging.warning("Connection to node %s failed or invalid data" " returned", node) @@ -1244,6 +1325,128 @@ class LUVerifyDisks(NoHooksLU): return result +class LURepairDiskSizes(NoHooksLU): + """Verifies the cluster disks sizes. + + """ + _OP_REQP = ["instances"] + REQ_BGL = False + + def ExpandNames(self): + + if not isinstance(self.op.instances, list): + raise errors.OpPrereqError("Invalid argument type 'instances'") + + if self.op.instances: + self.wanted_names = [] + for name in self.op.instances: + full_name = self.cfg.ExpandInstanceName(name) + if full_name is None: + raise errors.OpPrereqError("Instance '%s' not known" % name) + self.wanted_names.append(full_name) + self.needed_locks = { + locking.LEVEL_NODE: [], + locking.LEVEL_INSTANCE: self.wanted_names, + } + self.recalculate_locks[locking.LEVEL_NODE] = constants.LOCKS_REPLACE + else: + self.wanted_names = None + self.needed_locks = { + locking.LEVEL_NODE: locking.ALL_SET, + locking.LEVEL_INSTANCE: locking.ALL_SET, + } + self.share_locks = dict(((i, 1) for i in locking.LEVELS)) + + def DeclareLocks(self, level): + if level == locking.LEVEL_NODE and self.wanted_names is not None: + self._LockInstancesNodes(primary_only=True) + + def CheckPrereq(self): + """Check prerequisites. + + This only checks the optional instance list against the existing names. + + """ + if self.wanted_names is None: + self.wanted_names = self.acquired_locks[locking.LEVEL_INSTANCE] + + self.wanted_instances = [self.cfg.GetInstanceInfo(name) for name + in self.wanted_names] + + def _EnsureChildSizes(self, disk): + """Ensure children of the disk have the needed disk size. + + This is valid mainly for DRBD8 and fixes an issue where the + children have smaller disk size. + + @param disk: an L{ganeti.objects.Disk} object + + """ + if disk.dev_type == constants.LD_DRBD8: + assert disk.children, "Empty children for DRBD8?" + fchild = disk.children[0] + mismatch = fchild.size < disk.size + if mismatch: + self.LogInfo("Child disk has size %d, parent %d, fixing", + fchild.size, disk.size) + fchild.size = disk.size + + # and we recurse on this child only, not on the metadev + return self._EnsureChildSizes(fchild) or mismatch + else: + return False + + def Exec(self, feedback_fn): + """Verify the size of cluster disks. + + """ + # TODO: check child disks too + # TODO: check differences in size between primary/secondary nodes + per_node_disks = {} + for instance in self.wanted_instances: + pnode = instance.primary_node + if pnode not in per_node_disks: + per_node_disks[pnode] = [] + for idx, disk in enumerate(instance.disks): + per_node_disks[pnode].append((instance, idx, disk)) + + changed = [] + for node, dskl in per_node_disks.items(): + newl = [v[2].Copy() for v in dskl] + for dsk in newl: + self.cfg.SetDiskID(dsk, node) + result = self.rpc.call_blockdev_getsizes(node, newl) + if result.failed: + self.LogWarning("Failure in blockdev_getsizes call to node" + " %s, ignoring", node) + continue + if len(result.data) != len(dskl): + self.LogWarning("Invalid result from node %s, ignoring node results", + node) + continue + for ((instance, idx, disk), size) in zip(dskl, result.data): + if size is None: + self.LogWarning("Disk %d of instance %s did not return size" + " information, ignoring", idx, instance.name) + continue + if not isinstance(size, (int, long)): + self.LogWarning("Disk %d of instance %s did not return valid" + " size information, ignoring", idx, instance.name) + continue + size = size >> 20 + if size != disk.size: + self.LogInfo("Disk %d of instance %s has mismatched size," + " correcting: recorded %d, actual %d", idx, + instance.name, disk.size, size) + disk.size = size + self.cfg.Update(instance) + changed.append((instance.name, idx, size)) + if self._EnsureChildSizes(disk): + self.cfg.Update(instance) + changed.append((instance.name, idx, disk.size)) + return changed + + class LURenameCluster(LogicalUnit): """Rename the cluster. @@ -1318,7 +1521,7 @@ class LURenameCluster(LogicalUnit): constants.SSH_KNOWN_HOSTS_FILE, to_node) finally: - result = self.rpc.call_node_start_master(master, False) + result = self.rpc.call_node_start_master(master, False, False) if result.failed or not result.data: self.LogWarning("Could not re-enable the master role on" " the master, please restart manually.") @@ -1329,7 +1532,7 @@ def _RecursiveCheckIfLVMBased(disk): @type disk: L{objects.Disk} @param disk: the disk to check - @rtype: booleean + @rtype: boolean @return: boolean indicating whether a LD_LV dev_type was found or not """ @@ -1349,7 +1552,7 @@ class LUSetClusterParams(LogicalUnit): _OP_REQP = [] REQ_BGL = False - def CheckParameters(self): + def CheckArguments(self): """Check parameters """ @@ -1358,7 +1561,7 @@ class LUSetClusterParams(LogicalUnit): if self.op.candidate_pool_size is not None: try: self.op.candidate_pool_size = int(self.op.candidate_pool_size) - except ValueError, err: + except (ValueError, TypeError), err: raise errors.OpPrereqError("Invalid candidate_pool_size value: %s" % str(err)) if self.op.candidate_pool_size < 1: @@ -1390,8 +1593,6 @@ class LUSetClusterParams(LogicalUnit): if the given volume group is valid. """ - # FIXME: This only works because there is only one parameter that can be - # changed or removed. if self.op.vg_name is not None and not self.op.vg_name: instances = self.cfg.GetAllInstancesInfo().values() for inst in instances: @@ -1420,7 +1621,7 @@ class LUSetClusterParams(LogicalUnit): self.cluster = cluster = self.cfg.GetClusterInfo() # validate beparams changes if self.op.beparams: - utils.CheckBEParams(self.op.beparams) + utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES) self.new_beparams = cluster.FillDict( cluster.beparams[constants.BEGR_DEFAULT], self.op.beparams) @@ -1437,6 +1638,13 @@ class LUSetClusterParams(LogicalUnit): if self.op.enabled_hypervisors is not None: self.hv_list = self.op.enabled_hypervisors + if not self.hv_list: + raise errors.OpPrereqError("Enabled hypervisors list must contain at" + " least one member") + invalid_hvs = set(self.hv_list) - constants.HYPER_TYPES + if invalid_hvs: + raise errors.OpPrereqError("Enabled hypervisors contains invalid" + " entries: %s" % invalid_hvs) else: self.hv_list = cluster.enabled_hypervisors @@ -1448,6 +1656,7 @@ class LUSetClusterParams(LogicalUnit): hv_name in self.op.enabled_hypervisors)): # either this is a new hypervisor, or its parameters have changed hv_class = hypervisor.GetHypervisor(hv_name) + utils.ForceDictType(hv_params, constants.HVS_PARAMETER_TYPES) hv_class.CheckParameterSyntax(hv_params) _CheckHVParams(self, node_list, hv_name, hv_params) @@ -1456,8 +1665,11 @@ class LUSetClusterParams(LogicalUnit): """ if self.op.vg_name is not None: - if self.op.vg_name != self.cfg.GetVGName(): - self.cfg.SetVGName(self.op.vg_name) + new_volume = self.op.vg_name + if not new_volume: + new_volume = None + if new_volume != self.cfg.GetVGName(): + self.cfg.SetVGName(new_volume) else: feedback_fn("Cluster LVM configuration already in desired" " state, not changing") @@ -1469,14 +1681,11 @@ class LUSetClusterParams(LogicalUnit): self.cluster.beparams[constants.BEGR_DEFAULT] = self.new_beparams if self.op.candidate_pool_size is not None: self.cluster.candidate_pool_size = self.op.candidate_pool_size + # we need to update the pool size here, otherwise the save will fail + _AdjustCandidatePool(self) self.cfg.Update(self.cluster) - # we want to update nodes after the cluster so that if any errors - # happen, we have recorded and saved the cluster info - if self.op.candidate_pool_size is not None: - _AdjustCandidatePool(self) - class LURedistributeConfig(NoHooksLU): """Force the redistribution of cluster configuration. @@ -1521,6 +1730,7 @@ def _WaitForSync(lu, instance, oneshot=False, unlock=False): lu.cfg.SetDiskID(dev, node) retries = 0 + degr_retries = 10 # in seconds, as we sleep 1 second each time while True: max_time = 0 done = True @@ -1553,6 +1763,16 @@ def _WaitForSync(lu, instance, oneshot=False, unlock=False): rem_time = "no time estimate" lu.proc.LogInfo("- device %s: %5.2f%% done, %s" % (instance.disks[i].iv_name, perc_done, rem_time)) + + # if we're done but degraded, let's do a few small retries, to + # make sure we see a stable and not transient situation; therefore + # we force restart of the loop + if (done or oneshot) and cumul_degraded and degr_retries > 0: + logging.info("Degraded disks found, %d retries left", degr_retries) + degr_retries -= 1 + time.sleep(1) + continue + if done or oneshot: break @@ -1580,11 +1800,15 @@ def _CheckDiskConsistency(lu, dev, node, on_primary, ldisk=False): result = True if on_primary or dev.AssembleOnSecondary(): rstats = lu.rpc.call_blockdev_find(node, dev) - if rstats.failed or not rstats.data: - logging.warning("Node %s: disk degraded, not found or node down", node) + msg = rstats.RemoteFailMsg() + if msg: + lu.LogWarning("Can't find disk on node %s: %s", node, msg) + result = False + elif not rstats.payload: + lu.LogWarning("Can't find disk on node %s", node) result = False else: - result = result and (not rstats.data[idx]) + result = result and (not rstats.payload[idx]) if dev.children: for child in dev.children: result = result and _CheckDiskConsistency(lu, child, node, on_primary) @@ -1610,9 +1834,11 @@ class LUDiagnoseOS(NoHooksLU): selected=self.op.output_fields) # Lock all nodes, in shared mode + # Temporary removal of locks, should be reverted later + # TODO: reintroduce locks when they are lighter-weight self.needed_locks = {} - self.share_locks[locking.LEVEL_NODE] = 1 - self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET + #self.share_locks[locking.LEVEL_NODE] = 1 + #self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET def CheckPrereq(self): """Check prerequisites. @@ -1627,7 +1853,7 @@ class LUDiagnoseOS(NoHooksLU): @param rlist: a map with node names as keys and OS objects as values @rtype: dict - @returns: a dictionary with osnames as keys and as value another map, with + @return: a dictionary with osnames as keys and as value another map, with nodes as keys and list of OS objects as values, eg:: {"debian-etch": {"node1": [,...], @@ -1636,6 +1862,11 @@ class LUDiagnoseOS(NoHooksLU): """ all_os = {} + # we build here the list of nodes that didn't fail the RPC (at RPC + # level), so that nodes with a non-responding node daemon don't + # make all OSes invalid + good_nodes = [node_name for node_name in rlist + if not rlist[node_name].failed] for node_name, nr in rlist.iteritems(): if nr.failed or not nr.data: continue @@ -1644,7 +1875,7 @@ class LUDiagnoseOS(NoHooksLU): # build a list of nodes for this os containing empty lists # for each node in node_list all_os[os_obj.name] = {} - for nname in node_list: + for nname in good_nodes: all_os[os_obj.name][nname] = [] all_os[os_obj.name][node_name].append(os_obj) return all_os @@ -1653,9 +1884,7 @@ class LUDiagnoseOS(NoHooksLU): """Compute the list of OSes. """ - node_list = self.acquired_locks[locking.LEVEL_NODE] - valid_nodes = [node for node in self.cfg.GetOnlineNodeList() - if node in node_list] + valid_nodes = [node for node in self.cfg.GetOnlineNodeList()] node_data = self.rpc.call_os_diagnose(valid_nodes) if node_data == False: raise errors.OpExecError("Can't gather the list of OSes") @@ -1711,7 +1940,7 @@ class LURemoveNode(LogicalUnit): - it does not have primary or secondary instances - it's not the master - Any errors are signalled by raising errors.OpPrereqError. + Any errors are signaled by raising errors.OpPrereqError. """ node = self.cfg.GetNodeInfo(self.cfg.ExpandNodeName(self.op.node_name)) @@ -1753,13 +1982,13 @@ class LUQueryNodes(NoHooksLU): """Logical unit for querying nodes. """ - _OP_REQP = ["output_fields", "names"] + _OP_REQP = ["output_fields", "names", "use_locking"] REQ_BGL = False _FIELDS_DYNAMIC = utils.FieldSet( "dtotal", "dfree", "mtotal", "mnode", "mfree", "bootid", - "ctotal", + "ctotal", "cnodes", "csockets", ) _FIELDS_STATIC = utils.FieldSet( @@ -1770,6 +1999,8 @@ class LUQueryNodes(NoHooksLU): "master_candidate", "master", "offline", + "drained", + "role", ) def ExpandNames(self): @@ -1785,7 +2016,8 @@ class LUQueryNodes(NoHooksLU): else: self.wanted = locking.ALL_SET - self.do_locking = self._FIELDS_STATIC.NonMatching(self.op.output_fields) + self.do_node_query = self._FIELDS_STATIC.NonMatching(self.op.output_fields) + self.do_locking = self.do_node_query and self.op.use_locking if self.do_locking: # if we don't request only static fields, we need to lock the nodes self.needed_locks[locking.LEVEL_NODE] = self.wanted @@ -1820,7 +2052,7 @@ class LUQueryNodes(NoHooksLU): # begin data gathering - if self.do_locking: + if self.do_node_query: live_data = {} node_data = self.rpc.call_node_info(nodenames, self.cfg.GetVGName(), self.cfg.GetHypervisorType()) @@ -1837,6 +2069,8 @@ class LUQueryNodes(NoHooksLU): "dfree": fn(int, nodeinfo.get('vg_free', None)), "ctotal": fn(int, nodeinfo.get('cpu_total', None)), "bootid": nodeinfo.get('bootid', None), + "cnodes": fn(int, nodeinfo.get('cpu_nodes', None)), + "csockets": fn(int, nodeinfo.get('cpu_sockets', None)), } else: live_data[name] = {} @@ -1849,10 +2083,9 @@ class LUQueryNodes(NoHooksLU): inst_fields = frozenset(("pinst_cnt", "pinst_list", "sinst_cnt", "sinst_list")) if inst_fields & frozenset(self.op.output_fields): - instancelist = self.cfg.GetInstanceList() + inst_data = self.cfg.GetAllInstancesInfo() - for instance_name in instancelist: - inst = self.cfg.GetInstanceInfo(instance_name) + for instance_name, inst in inst_data.items(): if inst.primary_node in node_to_primary: node_to_primary[inst.primary_node].add(inst.name) for secnode in inst.secondary_nodes: @@ -1891,8 +2124,21 @@ class LUQueryNodes(NoHooksLU): val = node.name == master_node elif field == "offline": val = node.offline + elif field == "drained": + val = node.drained elif self._FIELDS_DYNAMIC.Matches(field): val = live_data[node.name].get(field, None) + elif field == "role": + if node.name == master_node: + val = "M" + elif node.master_candidate: + val = "C" + elif node.drained: + val = "D" + elif node.offline: + val = "O" + else: + val = "R" else: raise errors.ParameterError(field) node_output.append(val) @@ -2014,7 +2260,7 @@ class LUAddNode(LogicalUnit): - it is resolvable - its parameters (single/dual homed) matches the cluster - Any errors are signalled by raising errors.OpPrereqError. + Any errors are signaled by raising errors.OpPrereqError. """ node_name = self.op.node_name @@ -2068,7 +2314,7 @@ class LUAddNode(LogicalUnit): raise errors.OpPrereqError("The master has a private ip but the" " new node doesn't have one") - # checks reachablity + # checks reachability if not utils.TcpPing(primary_ip, constants.DEFAULT_NODED_PORT): raise errors.OpPrereqError("Node not reachable by ping") @@ -2080,14 +2326,24 @@ class LUAddNode(LogicalUnit): " based ping to noded port") cp_size = self.cfg.GetClusterInfo().candidate_pool_size - mc_now, _ = self.cfg.GetMasterCandidateStats() - master_candidate = mc_now < cp_size + if self.op.readd: + exceptions = [node] + else: + exceptions = [] + mc_now, mc_max = self.cfg.GetMasterCandidateStats(exceptions) + # the new node will increase mc_max with one, so: + mc_max = min(mc_max + 1, cp_size) + self.master_candidate = mc_now < mc_max - self.new_node = objects.Node(name=node, - primary_ip=primary_ip, - secondary_ip=secondary_ip, - master_candidate=master_candidate, - offline=False) + if self.op.readd: + self.new_node = self.cfg.GetNodeInfo(node) + assert self.new_node is not None, "Can't retrieve locked node %s" % node + else: + self.new_node = objects.Node(name=node, + primary_ip=primary_ip, + secondary_ip=secondary_ip, + master_candidate=self.master_candidate, + offline=False, drained=False) def Exec(self, feedback_fn): """Adds the new node to the cluster. @@ -2096,6 +2352,20 @@ class LUAddNode(LogicalUnit): new_node = self.new_node node = new_node.name + # for re-adds, reset the offline/drained/master-candidate flags; + # we need to reset here, otherwise offline would prevent RPC calls + # later in the procedure; this also means that if the re-add + # fails, we are left with a non-offlined, broken node + if self.op.readd: + new_node.drained = new_node.offline = False + self.LogInfo("Readding a node, the offline/drained flags were reset") + # if we demote the node, we do cleanup later in the procedure + new_node.master_candidate = self.master_candidate + + # notify the user about any possible mc promotion + if new_node.master_candidate: + self.LogInfo("Node will be a master candidate") + # check connectivity result = self.rpc.call_version([node])[node] result.Raise() @@ -2135,7 +2405,8 @@ class LUAddNode(LogicalUnit): " new node: %s" % msg) # Add node to our /etc/hosts, and add key to known_hosts - utils.AddHostToEtcHosts(new_node.name) + if self.cfg.GetClusterInfo().modify_etc_hosts: + utils.AddHostToEtcHosts(new_node.name) if new_node.secondary_ip != new_node.primary_ip: result = self.rpc.call_node_has_ip_address(new_node.name, @@ -2159,7 +2430,8 @@ class LUAddNode(LogicalUnit): " for remote verification" % verifier) if result[verifier].data['nodelist']: for failed in result[verifier].data['nodelist']: - feedback_fn("ssh/hostname verification failed %s -> %s" % + feedback_fn("ssh/hostname verification failed" + " (checking from %s): %s" % (verifier, result[verifier].data['nodelist'][failed])) raise errors.OpExecError("ssh/hostname verification failed.") @@ -2181,7 +2453,7 @@ class LUAddNode(LogicalUnit): to_copy = [] enabled_hypervisors = self.cfg.GetClusterInfo().enabled_hypervisors - if constants.HTS_USE_VNC.intersection(enabled_hypervisors): + if constants.HTS_COPY_VNC_PASSWORD.intersection(enabled_hypervisors): to_copy.append(constants.VNC_PASSWORD_FILE) for fname in to_copy: @@ -2191,6 +2463,15 @@ class LUAddNode(LogicalUnit): if self.op.readd: self.context.ReaddNode(new_node) + # make sure we redistribute the config + self.cfg.Update(new_node) + # and make sure the new node will not have old files around + if not new_node.master_candidate: + result = self.rpc.call_node_demote_from_mc(new_node.name) + msg = result.RemoteFailMsg() + if msg: + self.LogWarning("Node failed to demote itself from master" + " candidate status: %s" % msg) else: self.context.AddNode(new_node) @@ -2211,11 +2492,13 @@ class LUSetNodeParams(LogicalUnit): self.op.node_name = node_name _CheckBooleanOpField(self.op, 'master_candidate') _CheckBooleanOpField(self.op, 'offline') - if self.op.master_candidate is None and self.op.offline is None: + _CheckBooleanOpField(self.op, 'drained') + all_mods = [self.op.offline, self.op.master_candidate, self.op.drained] + if all_mods.count(None) == 3: raise errors.OpPrereqError("Please pass at least one modification") - if self.op.offline == True and self.op.master_candidate == True: - raise errors.OpPrereqError("Can't set the node into offline and" - " master_candidate at the same time") + if all_mods.count(True) > 1: + raise errors.OpPrereqError("Can't set the node into more than one" + " state at the same time") def ExpandNames(self): self.needed_locks = {locking.LEVEL_NODE: self.op.node_name} @@ -2230,6 +2513,7 @@ class LUSetNodeParams(LogicalUnit): "OP_TARGET": self.op.node_name, "MASTER_CANDIDATE": str(self.op.master_candidate), "OFFLINE": str(self.op.offline), + "DRAINED": str(self.op.drained), } nl = [self.cfg.GetMasterNode(), self.op.node_name] @@ -2243,12 +2527,16 @@ class LUSetNodeParams(LogicalUnit): """ node = self.node = self.cfg.GetNodeInfo(self.op.node_name) - if ((self.op.master_candidate == False or self.op.offline == True) - and node.master_candidate): - # we will demote the node from master_candidate + if (self.op.master_candidate is not None or + self.op.drained is not None or + self.op.offline is not None): + # we can't change the master's node flags if self.op.node_name == self.cfg.GetMasterNode(): - raise errors.OpPrereqError("The master node has to be a" - " master candidate and online") + raise errors.OpPrereqError("The master role can be changed" + " only via masterfailover") + + if ((self.op.master_candidate == False or self.op.offline == True or + self.op.drained == True) and node.master_candidate): cp_size = self.cfg.GetClusterInfo().candidate_pool_size num_candidates, _ = self.cfg.GetMasterCandidateStats() if num_candidates <= cp_size: @@ -2259,10 +2547,11 @@ class LUSetNodeParams(LogicalUnit): else: raise errors.OpPrereqError(msg) - if (self.op.master_candidate == True and node.offline and - not self.op.offline == False): - raise errors.OpPrereqError("Can't set an offline node to" - " master_candidate") + if (self.op.master_candidate == True and + ((node.offline and not self.op.offline == False) or + (node.drained and not self.op.drained == False))): + raise errors.OpPrereqError("Node '%s' is offline or drained, can't set" + " to master_candidate" % node.name) return @@ -2273,29 +2562,50 @@ class LUSetNodeParams(LogicalUnit): node = self.node result = [] + changed_mc = False if self.op.offline is not None: node.offline = self.op.offline result.append(("offline", str(self.op.offline))) - if self.op.offline == True and node.master_candidate: - node.master_candidate = False - result.append(("master_candidate", "auto-demotion due to offline")) + if self.op.offline == True: + if node.master_candidate: + node.master_candidate = False + changed_mc = True + result.append(("master_candidate", "auto-demotion due to offline")) + if node.drained: + node.drained = False + result.append(("drained", "clear drained status due to offline")) if self.op.master_candidate is not None: node.master_candidate = self.op.master_candidate + changed_mc = True result.append(("master_candidate", str(self.op.master_candidate))) if self.op.master_candidate == False: rrc = self.rpc.call_node_demote_from_mc(node.name) - if (rrc.failed or not isinstance(rrc.data, (tuple, list)) - or len(rrc.data) != 2): - self.LogWarning("Node rpc error: %s" % rrc.error) - elif not rrc.data[0]: - self.LogWarning("Node failed to demote itself: %s" % rrc.data[1]) + msg = rrc.RemoteFailMsg() + if msg: + self.LogWarning("Node failed to demote itself: %s" % msg) + + if self.op.drained is not None: + node.drained = self.op.drained + result.append(("drained", str(self.op.drained))) + if self.op.drained == True: + if node.master_candidate: + node.master_candidate = False + changed_mc = True + result.append(("master_candidate", "auto-demotion due to drain")) + rrc = self.rpc.call_node_demote_from_mc(node.name) + msg = rrc.RemoteFailMsg() + if msg: + self.LogWarning("Node failed to demote itself: %s" % msg) + if node.offline: + node.offline = False + result.append(("offline", "clear offline status due to drain")) # this will trigger configuration file update, if needed self.cfg.Update(node) # this will trigger job queue propagation or cleanup - if self.op.node_name != self.cfg.GetMasterNode(): + if changed_mc: self.context.ReaddNode(node) return result @@ -2333,10 +2643,15 @@ class LUQueryClusterInfo(NoHooksLU): "master": cluster.master_node, "default_hypervisor": cluster.default_hypervisor, "enabled_hypervisors": cluster.enabled_hypervisors, - "hvparams": dict([(hypervisor, cluster.hvparams[hypervisor]) - for hypervisor in cluster.enabled_hypervisors]), + "hvparams": dict([(hypervisor_name, cluster.hvparams[hypervisor_name]) + for hypervisor_name in cluster.enabled_hypervisors]), "beparams": cluster.beparams, "candidate_pool_size": cluster.candidate_pool_size, + "default_bridge": cluster.default_bridge, + "master_netdev": cluster.master_netdev, + "volume_group_name": cluster.volume_group_name, + "file_storage_dir": cluster.file_storage_dir, + "tags": list(cluster.GetTags()), } return result @@ -2408,19 +2723,24 @@ class LUActivateInstanceDisks(NoHooksLU): assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name _CheckNodeOnline(self, self.instance.primary_node) + if not hasattr(self.op, "ignore_size"): + self.op.ignore_size = False def Exec(self, feedback_fn): """Activate the disks. """ - disks_ok, disks_info = _AssembleInstanceDisks(self, self.instance) + disks_ok, disks_info = \ + _AssembleInstanceDisks(self, self.instance, + ignore_size=self.op.ignore_size) if not disks_ok: raise errors.OpExecError("Cannot activate block devices") return disks_info -def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False): +def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False, + ignore_size=False): """Prepare the block devices for an instance. This sets up the block devices on all nodes. @@ -2432,6 +2752,10 @@ def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False): @type ignore_secondaries: boolean @param ignore_secondaries: if true, errors on secondary nodes won't result in an error return from the function + @type ignore_size: boolean + @param ignore_size: if true, the current known size of the disk + will not be used during the disk activation, useful for cases + when the size is wrong @return: False if the operation failed, otherwise a list of (host, instance_visible_name, node_visible_name) with the mapping from node devices to instance devices @@ -2452,12 +2776,16 @@ def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False): # 1st pass, assemble on all nodes in secondary mode for inst_disk in instance.disks: for node, node_disk in inst_disk.ComputeNodeTree(instance.primary_node): + if ignore_size: + node_disk = node_disk.Copy() + node_disk.UnsetSize() lu.cfg.SetDiskID(node_disk, node) result = lu.rpc.call_blockdev_assemble(node, node_disk, iname, False) - if result.failed or not result: + msg = result.RemoteFailMsg() + if msg: lu.proc.LogWarning("Could not prepare block device %s on node %s" - " (is_primary=False, pass=1)", - inst_disk.iv_name, node) + " (is_primary=False, pass=1): %s", + inst_disk.iv_name, node, msg) if not ignore_secondaries: disks_ok = False @@ -2468,14 +2796,19 @@ def _AssembleInstanceDisks(lu, instance, ignore_secondaries=False): for node, node_disk in inst_disk.ComputeNodeTree(instance.primary_node): if node != instance.primary_node: continue + if ignore_size: + node_disk = node_disk.Copy() + node_disk.UnsetSize() lu.cfg.SetDiskID(node_disk, node) result = lu.rpc.call_blockdev_assemble(node, node_disk, iname, True) - if result.failed or not result: + msg = result.RemoteFailMsg() + if msg: lu.proc.LogWarning("Could not prepare block device %s on node %s" - " (is_primary=True, pass=2)", - inst_disk.iv_name, node) + " (is_primary=True, pass=2): %s", + inst_disk.iv_name, node, msg) disks_ok = False - device_info.append((instance.primary_node, inst_disk.iv_name, result.data)) + device_info.append((instance.primary_node, inst_disk.iv_name, + result.payload)) # leave the disks configured for the primary node # this is a workaround that would be fixed better by @@ -2490,7 +2823,7 @@ def _StartInstanceDisks(lu, instance, force): """Start the disks of an instance. """ - disks_ok, dummy = _AssembleInstanceDisks(lu, instance, + disks_ok, _ = _AssembleInstanceDisks(lu, instance, ignore_secondaries=force) if not disks_ok: _ShutdownInstanceDisks(lu, instance) @@ -2565,17 +2898,18 @@ def _ShutdownInstanceDisks(lu, instance, ignore_primary=False): ignored. """ - result = True + all_result = True for disk in instance.disks: for node, top_disk in disk.ComputeNodeTree(instance.primary_node): lu.cfg.SetDiskID(top_disk, node) result = lu.rpc.call_blockdev_shutdown(node, top_disk) - if result.failed or not result.data: - logging.error("Could not shutdown block device %s on node %s", - disk.iv_name, node) + msg = result.RemoteFailMsg() + if msg: + lu.LogWarning("Could not shutdown block device %s on node %s: %s", + disk.iv_name, node, msg) if not ignore_primary or node != instance.primary_node: - result = False - return result + all_result = False + return all_result def _CheckNodeFreeMemory(lu, node, reason, requested, hypervisor_name): @@ -2647,15 +2981,48 @@ class LUStartupInstance(LogicalUnit): assert self.instance is not None, \ "Cannot retrieve locked instance %s" % self.op.instance_name + # extra beparams + self.beparams = getattr(self.op, "beparams", {}) + if self.beparams: + if not isinstance(self.beparams, dict): + raise errors.OpPrereqError("Invalid beparams passed: %s, expected" + " dict" % (type(self.beparams), )) + # fill the beparams dict + utils.ForceDictType(self.beparams, constants.BES_PARAMETER_TYPES) + self.op.beparams = self.beparams + + # extra hvparams + self.hvparams = getattr(self.op, "hvparams", {}) + if self.hvparams: + if not isinstance(self.hvparams, dict): + raise errors.OpPrereqError("Invalid hvparams passed: %s, expected" + " dict" % (type(self.hvparams), )) + + # check hypervisor parameter syntax (locally) + cluster = self.cfg.GetClusterInfo() + utils.ForceDictType(self.hvparams, constants.HVS_PARAMETER_TYPES) + filled_hvp = cluster.FillDict(cluster.hvparams[instance.hypervisor], + instance.hvparams) + filled_hvp.update(self.hvparams) + hv_type = hypervisor.GetHypervisor(instance.hypervisor) + hv_type.CheckParameterSyntax(filled_hvp) + _CheckHVParams(self, instance.all_nodes, instance.hypervisor, filled_hvp) + self.op.hvparams = self.hvparams + _CheckNodeOnline(self, instance.primary_node) bep = self.cfg.GetClusterInfo().FillBE(instance) - # check bridges existance + # check bridges existence _CheckInstanceBridgesExist(self, instance) - _CheckNodeFreeMemory(self, instance.primary_node, - "starting instance %s" % instance.name, - bep[constants.BE_MEMORY], instance.hypervisor) + remote_info = self.rpc.call_instance_info(instance.primary_node, + instance.name, + instance.hypervisor) + remote_info.Raise() + if not remote_info.data: + _CheckNodeFreeMemory(self, instance.primary_node, + "starting instance %s" % instance.name, + bep[constants.BE_MEMORY], instance.hypervisor) def Exec(self, feedback_fn): """Start the instance. @@ -2663,7 +3030,6 @@ class LUStartupInstance(LogicalUnit): """ instance = self.instance force = self.op.force - extra_args = getattr(self.op, "extra_args", "") self.cfg.MarkInstanceUp(instance.name) @@ -2671,7 +3037,8 @@ class LUStartupInstance(LogicalUnit): _StartInstanceDisks(self, instance, force) - result = self.rpc.call_instance_start(node_current, instance, extra_args) + result = self.rpc.call_instance_start(node_current, instance, + self.hvparams, self.beparams) msg = result.RemoteFailMsg() if msg: _ShutdownInstanceDisks(self, instance) @@ -2705,6 +3072,7 @@ class LURebootInstance(LogicalUnit): """ env = { "IGNORE_SECONDARIES": self.op.ignore_secondaries, + "REBOOT_TYPE": self.op.reboot_type, } env.update(_BuildInstanceHookEnvByObject(self, self.instance)) nl = [self.cfg.GetMasterNode()] + list(self.instance.all_nodes) @@ -2722,7 +3090,7 @@ class LURebootInstance(LogicalUnit): _CheckNodeOnline(self, instance.primary_node) - # check bridges existance + # check bridges existence _CheckInstanceBridgesExist(self, instance) def Exec(self, feedback_fn): @@ -2732,22 +3100,27 @@ class LURebootInstance(LogicalUnit): instance = self.instance ignore_secondaries = self.op.ignore_secondaries reboot_type = self.op.reboot_type - extra_args = getattr(self.op, "extra_args", "") node_current = instance.primary_node if reboot_type in [constants.INSTANCE_REBOOT_SOFT, constants.INSTANCE_REBOOT_HARD]: + for disk in instance.disks: + self.cfg.SetDiskID(disk, node_current) result = self.rpc.call_instance_reboot(node_current, instance, - reboot_type, extra_args) - if result.failed or not result.data: - raise errors.OpExecError("Could not reboot instance") + reboot_type) + msg = result.RemoteFailMsg() + if msg: + raise errors.OpExecError("Could not reboot instance: %s" % msg) else: - if not self.rpc.call_instance_shutdown(node_current, instance): - raise errors.OpExecError("could not shutdown instance for full reboot") + result = self.rpc.call_instance_shutdown(node_current, instance) + msg = result.RemoteFailMsg() + if msg: + raise errors.OpExecError("Could not shutdown instance for" + " full reboot: %s" % msg) _ShutdownInstanceDisks(self, instance) _StartInstanceDisks(self, instance, ignore_secondaries) - result = self.rpc.call_instance_start(node_current, instance, extra_args) + result = self.rpc.call_instance_start(node_current, instance, None, None) msg = result.RemoteFailMsg() if msg: _ShutdownInstanceDisks(self, instance) @@ -2798,8 +3171,9 @@ class LUShutdownInstance(LogicalUnit): node_current = instance.primary_node self.cfg.MarkInstanceDown(instance.name) result = self.rpc.call_instance_shutdown(node_current, instance) - if result.failed or not result.data: - self.proc.LogWarning("Could not shutdown instance") + msg = result.RemoteFailMsg() + if msg: + self.proc.LogWarning("Could not shutdown instance: %s" % msg) _ShutdownInstanceDisks(self, instance) @@ -2846,7 +3220,8 @@ class LUReinstallInstance(LogicalUnit): remote_info = self.rpc.call_instance_info(instance.primary_node, instance.name, instance.hypervisor) - if remote_info.failed or remote_info.data: + remote_info.Raise() + if remote_info.data: raise errors.OpPrereqError("Instance '%s' is running on the node %s" % (self.op.instance_name, instance.primary_node)) @@ -3049,12 +3424,14 @@ class LURemoveInstance(LogicalUnit): instance.name, instance.primary_node) result = self.rpc.call_instance_shutdown(instance.primary_node, instance) - if result.failed or not result.data: + msg = result.RemoteFailMsg() + if msg: if self.op.ignore_failures: - feedback_fn("Warning: can't shutdown instance") + feedback_fn("Warning: can't shutdown instance: %s" % msg) else: - raise errors.OpExecError("Could not shutdown instance %s on node %s" % - (instance.name, instance.primary_node)) + raise errors.OpExecError("Could not shutdown instance %s on" + " node %s: %s" % + (instance.name, instance.primary_node, msg)) logging.info("Removing block devices for instance %s", instance.name) @@ -3074,18 +3451,18 @@ class LUQueryInstances(NoHooksLU): """Logical unit for querying instances. """ - _OP_REQP = ["output_fields", "names"] + _OP_REQP = ["output_fields", "names", "use_locking"] REQ_BGL = False _FIELDS_STATIC = utils.FieldSet(*["name", "os", "pnode", "snodes", - "admin_state", "admin_ram", + "admin_state", "disk_template", "ip", "mac", "bridge", "sda_size", "sdb_size", "vcpus", "tags", "network_port", "beparams", - "(disk).(size)/([0-9]+)", - "(disk).(sizes)", - "(nic).(mac|ip|bridge)/([0-9]+)", - "(nic).(macs|ips|bridges)", - "(disk|nic).(count)", + r"(disk)\.(size)/([0-9]+)", + r"(disk)\.(sizes)", "disk_usage", + r"(nic)\.(mac|ip|bridge)/([0-9]+)", + r"(nic)\.(macs|ips|bridges)", + r"(disk|nic)\.(count)", "serial_no", "hypervisor", "hvparams",] + ["hv/%s" % name for name in constants.HVS_PARAMETERS] + @@ -3108,7 +3485,8 @@ class LUQueryInstances(NoHooksLU): else: self.wanted = locking.ALL_SET - self.do_locking = self._FIELDS_STATIC.NonMatching(self.op.output_fields) + self.do_node_query = self._FIELDS_STATIC.NonMatching(self.op.output_fields) + self.do_locking = self.do_node_query and self.op.use_locking if self.do_locking: self.needed_locks[locking.LEVEL_INSTANCE] = self.wanted self.needed_locks[locking.LEVEL_NODE] = [] @@ -3157,7 +3535,7 @@ class LUQueryInstances(NoHooksLU): bad_nodes = [] off_nodes = [] - if self.do_locking: + if self.do_node_query: live_data = {} node_data = self.rpc.call_all_instances_info(nodes, hv_list) for name in nodes: @@ -3224,20 +3602,34 @@ class LUQueryInstances(NoHooksLU): val = live_data[instance.name].get("memory", "?") else: val = "-" + elif field == "vcpus": + val = i_be[constants.BE_VCPUS] elif field == "disk_template": val = instance.disk_template elif field == "ip": - val = instance.nics[0].ip + if instance.nics: + val = instance.nics[0].ip + else: + val = None elif field == "bridge": - val = instance.nics[0].bridge + if instance.nics: + val = instance.nics[0].bridge + else: + val = None elif field == "mac": - val = instance.nics[0].mac + if instance.nics: + val = instance.nics[0].mac + else: + val = None elif field == "sda_size" or field == "sdb_size": idx = ord(field[2]) - ord('a') try: val = instance.FindDisk(idx).size except errors.OpPrereqError: val = None + elif field == "disk_usage": # total disk usage per node + disk_sizes = [{'size': disk.size} for disk in instance.disks] + val = _ComputeDiskSize(instance.disk_template, disk_sizes) elif field == "tags": val = list(instance.GetTags()) elif field == "serial_no": @@ -3295,9 +3687,10 @@ class LUQueryInstances(NoHooksLU): else: assert False, "Unhandled NIC parameter" else: - assert False, "Unhandled variable parameter" + assert False, ("Declared but unhandled variable parameter '%s'" % + field) else: - raise errors.ParameterError(field) + assert False, "Declared but unhandled parameter '%s'" % field iout.append(val) output.append(iout) @@ -3357,12 +3750,18 @@ class LUFailoverInstance(LogicalUnit): target_node = secondary_nodes[0] _CheckNodeOnline(self, target_node) - # check memory requirements on the secondary node - _CheckNodeFreeMemory(self, target_node, "failing over instance %s" % - instance.name, bep[constants.BE_MEMORY], - instance.hypervisor) + _CheckNodeNotDrained(self, target_node) - # check bridge existance + if instance.admin_up: + # check memory requirements on the secondary node + _CheckNodeFreeMemory(self, target_node, "failing over instance %s" % + instance.name, bep[constants.BE_MEMORY], + instance.hypervisor) + else: + self.LogInfo("Not checking memory on the secondary node as" + " instance will not be started") + + # check bridge existence brlist = [nic.bridge for nic in instance.nics] result = self.rpc.call_bridges_exist(target_node, brlist) result.Raise() @@ -3396,15 +3795,17 @@ class LUFailoverInstance(LogicalUnit): instance.name, source_node) result = self.rpc.call_instance_shutdown(source_node, instance) - if result.failed or not result.data: + msg = result.RemoteFailMsg() + if msg: if self.op.ignore_consistency: self.proc.LogWarning("Could not shutdown instance %s on node %s." - " Proceeding" - " anyway. Please make sure node %s is down", - instance.name, source_node, source_node) + " Proceeding anyway. Please make sure node" + " %s is down. Error details: %s", + instance.name, source_node, source_node, msg) else: - raise errors.OpExecError("Could not shutdown instance %s on node %s" % - (instance.name, source_node)) + raise errors.OpExecError("Could not shutdown instance %s on" + " node %s: %s" % + (instance.name, source_node, msg)) feedback_fn("* deactivating the instance's disks on source node") if not _ShutdownInstanceDisks(self, instance, ignore_primary=True): @@ -3420,14 +3821,14 @@ class LUFailoverInstance(LogicalUnit): logging.info("Starting instance %s on node %s", instance.name, target_node) - disks_ok, dummy = _AssembleInstanceDisks(self, instance, + disks_ok, _ = _AssembleInstanceDisks(self, instance, ignore_secondaries=True) if not disks_ok: _ShutdownInstanceDisks(self, instance) raise errors.OpExecError("Can't activate the instance's disks") feedback_fn("* starting the instance on the target node") - result = self.rpc.call_instance_start(target_node, instance, None) + result = self.rpc.call_instance_start(target_node, instance, None, None) msg = result.RemoteFailMsg() if msg: _ShutdownInstanceDisks(self, instance) @@ -3464,6 +3865,8 @@ class LUMigrateInstance(LogicalUnit): """ env = _BuildInstanceHookEnvByObject(self, self.instance) + env["MIGRATE_LIVE"] = self.op.live + env["MIGRATE_CLEANUP"] = self.op.cleanup nl = [self.cfg.GetMasterNode()] + list(self.instance.secondary_nodes) return env, nl, nl @@ -3485,8 +3888,8 @@ class LUMigrateInstance(LogicalUnit): secondary_nodes = instance.secondary_nodes if not secondary_nodes: - raise errors.ProgrammerError("no secondary node but using " - "drbd8 disk template") + raise errors.ConfigurationError("No secondary node but using" + " drbd8 disk template") i_be = self.cfg.GetClusterInfo().FillBE(instance) @@ -3496,7 +3899,7 @@ class LUMigrateInstance(LogicalUnit): instance.name, i_be[constants.BE_MEMORY], instance.hypervisor) - # check bridge existance + # check bridge existence brlist = [nic.bridge for nic in instance.nics] result = self.rpc.call_bridges_exist(target_node, brlist) if result.failed or not result.data: @@ -3505,6 +3908,7 @@ class LUMigrateInstance(LogicalUnit): (brlist, target_node)) if not self.op.cleanup: + _CheckNodeNotDrained(self, target_node) result = self.rpc.call_instance_migratable(instance.primary_node, instance) msg = result.RemoteFailMsg() @@ -3533,7 +3937,7 @@ class LUMigrateInstance(LogicalUnit): if msg: raise errors.OpExecError("Cannot resync disks on node %s: %s" % (node, msg)) - node_done, node_percent = nres.data[1] + node_done, node_percent = nres.payload all_done = all_done and node_done if node_percent is not None: min_percent = min(min_percent, node_percent) @@ -3719,11 +4123,11 @@ class LUMigrateInstance(LogicalUnit): msg = result.RemoteFailMsg() if msg: log_err = ("Failed fetching source migration information from %s: %s" % - (source_node, msg)) + (source_node, msg)) logging.error(log_err) raise errors.OpExecError(log_err) - self.migration_info = migration_info = result.data[1] + self.migration_info = migration_info = result.payload # Then switch the disks to master/master mode self._EnsureSecondary(target_node) @@ -3826,7 +4230,7 @@ def _CreateBlockDev(lu, node, instance, device, force_create, (this will be represented as a LVM tag) @type force_open: boolean @param force_open: this parameter will be passes to the - L{backend.CreateBlockDevice} function where it specifies + L{backend.BlockdevCreate} function where it specifies whether we run on primary or not, and it affects both the child assembly and the device own Open() execution @@ -3861,7 +4265,7 @@ def _CreateSingleBlockDev(lu, node, instance, device, info, force_open): (this will be represented as a LVM tag) @type force_open: boolean @param force_open: this parameter will be passes to the - L{backend.CreateBlockDevice} function where it specifies + L{backend.BlockdevCreate} function where it specifies whether we run on primary or not, and it affects both the child assembly and the device own Open() execution @@ -3875,7 +4279,7 @@ def _CreateSingleBlockDev(lu, node, instance, device, info, force_open): " node %s for instance %s: %s" % (device, node, instance.name, msg)) if device.physical_id is None: - device.physical_id = result.data[1] + device.physical_id = result.payload def _GenerateUniqueNames(lu, exts): @@ -3931,7 +4335,7 @@ def _GenerateDiskTemplate(lu, template_name, if len(secondary_nodes) != 0: raise errors.ProgrammerError("Wrong template configuration") - names = _GenerateUniqueNames(lu, [".disk%d" % i + names = _GenerateUniqueNames(lu, [".disk%d" % (base_index + i) for i in range(disk_count)]) for idx, disk in enumerate(disk_info): disk_index = idx + base_index @@ -3948,7 +4352,7 @@ def _GenerateDiskTemplate(lu, template_name, [primary_node, remote_node] * len(disk_info), instance_name) names = [] - for lv_prefix in _GenerateUniqueNames(lu, [".disk%d" % i + for lv_prefix in _GenerateUniqueNames(lu, [".disk%d" % (base_index + i) for i in range(disk_count)]): names.append(lv_prefix + "_data") names.append(lv_prefix + "_meta") @@ -3970,7 +4374,7 @@ def _GenerateDiskTemplate(lu, template_name, iv_name="disk/%d" % disk_index, logical_id=(file_driver, "%s/disk%d" % (file_storage_dir, - idx)), + disk_index)), mode=disk["mode"]) disks.append(disk_dev) else: @@ -4041,15 +4445,15 @@ def _RemoveDisks(lu, instance): """ logging.info("Removing block devices for instance %s", instance.name) - result = True + all_result = True for device in instance.disks: for node, disk in device.ComputeNodeTree(instance.primary_node): lu.cfg.SetDiskID(disk, node) - result = lu.rpc.call_blockdev_remove(node, disk) - if result.failed or not result.data: - lu.proc.LogWarning("Could not remove block device %s on node %s," - " continuing anyway", device.iv_name, node) - result = False + msg = lu.rpc.call_blockdev_remove(node, disk).RemoteFailMsg() + if msg: + lu.LogWarning("Could not remove block device %s on node %s," + " continuing anyway: %s", device.iv_name, node, msg) + all_result = False if instance.disk_template == constants.DT_FILE: file_storage_dir = os.path.dirname(instance.disks[0].logical_id[1]) @@ -4057,9 +4461,9 @@ def _RemoveDisks(lu, instance): file_storage_dir) if result.failed or not result.data: logging.error("Could not remove directory '%s'", file_storage_dir) - result = False + all_result = False - return result + return all_result def _ComputeDiskSize(disk_template, disks): @@ -4106,13 +4510,10 @@ def _CheckHVParams(lu, nodenames, hvname, hvparams): info = hvinfo[node] if info.offline: continue - info.Raise() - if not info.data or not isinstance(info.data, (tuple, list)): - raise errors.OpPrereqError("Cannot get current information" - " from node '%s' (%s)" % (node, info.data)) - if not info.data[0]: - raise errors.OpPrereqError("Hypervisor parameter validation failed:" - " %s" % info.data[1]) + msg = info.RemoteFailMsg() + if msg: + raise errors.OpPrereqError("Hypervisor parameter validation" + " failed on node %s: %s" % (node, msg)) class LUCreateInstance(LogicalUnit): @@ -4172,14 +4573,15 @@ class LUCreateInstance(LogicalUnit): ",".join(enabled_hvs))) # check hypervisor parameter syntax (locally) - + utils.ForceDictType(self.op.hvparams, constants.HVS_PARAMETER_TYPES) filled_hvp = cluster.FillDict(cluster.hvparams[self.op.hypervisor], self.op.hvparams) hv_type = hypervisor.GetHypervisor(self.op.hypervisor) hv_type.CheckParameterSyntax(filled_hvp) + self.hv_full = filled_hvp # fill and remember the beparams dict - utils.CheckBEParams(self.op.beparams) + utils.ForceDictType(self.op.beparams, constants.BES_PARAMETER_TYPES) self.be_full = cluster.FillDict(cluster.beparams[constants.BEGR_DEFAULT], self.op.beparams) @@ -4218,6 +4620,12 @@ class LUCreateInstance(LogicalUnit): if not utils.IsValidMac(mac.lower()): raise errors.OpPrereqError("Invalid MAC address specified: %s" % mac) + else: + # or validate/reserve the current one + if self.cfg.IsMacInUse(mac): + raise errors.OpPrereqError("MAC address %s already in use" + " in cluster" % mac) + # bridge verification bridge = nic.get("bridge", None) if bridge is None: @@ -4336,23 +4744,27 @@ class LUCreateInstance(LogicalUnit): """ env = { - "INSTANCE_DISK_TEMPLATE": self.op.disk_template, - "INSTANCE_DISK_SIZE": ",".join(str(d["size"]) for d in self.disks), - "INSTANCE_ADD_MODE": self.op.mode, + "ADD_MODE": self.op.mode, } if self.op.mode == constants.INSTANCE_IMPORT: - env["INSTANCE_SRC_NODE"] = self.op.src_node - env["INSTANCE_SRC_PATH"] = self.op.src_path - env["INSTANCE_SRC_IMAGES"] = self.src_images + env["SRC_NODE"] = self.op.src_node + env["SRC_PATH"] = self.op.src_path + env["SRC_IMAGES"] = self.src_images - env.update(_BuildInstanceHookEnv(name=self.op.instance_name, + env.update(_BuildInstanceHookEnv( + name=self.op.instance_name, primary_node=self.op.pnode, secondary_nodes=self.secondaries, - status=self.instance_status, + status=self.op.start, os_type=self.op.os_type, memory=self.be_full[constants.BE_MEMORY], vcpus=self.be_full[constants.BE_VCPUS], nics=[(n.ip, n.bridge, n.mac) for n in self.nics], + disk_template=self.op.disk_template, + disks=[(d["size"], d["mode"]) for d in self.disks], + bep=self.be_full, + hvp=self.hv_full, + hypervisor_name=self.op.hypervisor, )) nl = ([self.cfg.GetMasterNode(), self.op.pnode] + @@ -4369,7 +4781,6 @@ class LUCreateInstance(LogicalUnit): raise errors.OpPrereqError("Cluster does not support lvm-based" " instances") - if self.op.mode == constants.INSTANCE_IMPORT: src_node = self.op.src_node src_path = self.op.src_path @@ -4435,6 +4846,7 @@ class LUCreateInstance(LogicalUnit): nic_mac_ini = 'nic%d_mac' % idx nic.mac = export_info.get(constants.INISECT_INS, nic_mac_ini) + # ENDIF: self.op.mode == constants.INSTANCE_IMPORT # ip ping checks (we use the same ip that was resolved in ExpandNames) if self.op.start and not self.op.ip_check: raise errors.OpPrereqError("Cannot ignore IP address conflicts when" @@ -4445,6 +4857,18 @@ class LUCreateInstance(LogicalUnit): raise errors.OpPrereqError("IP %s of instance %s already in use" % (self.check_ip, self.op.instance_name)) + #### mac address generation + # By generating here the mac address both the allocator and the hooks get + # the real final mac address rather than the 'auto' or 'generate' value. + # There is a race condition between the generation and the instance object + # creation, which means that we know the mac is valid now, but we're not + # sure it will be when we actually add the instance. If things go bad + # adding the instance will abort because of a duplicate mac, and the + # creation job will fail. + for nic in self.nics: + if nic.mac in (constants.VALUE_AUTO, constants.VALUE_GENERATE): + nic.mac = self.cfg.GenerateMAC() + #### allocator run if self.op.iallocator is not None: @@ -4459,6 +4883,9 @@ class LUCreateInstance(LogicalUnit): if pnode.offline: raise errors.OpPrereqError("Cannot use offline primary node '%s'" % pnode.name) + if pnode.drained: + raise errors.OpPrereqError("Cannot use drained primary node '%s'" % + pnode.name) self.secondaries = [] @@ -4470,8 +4897,9 @@ class LUCreateInstance(LogicalUnit): if self.op.snode == pnode.name: raise errors.OpPrereqError("The secondary node cannot be" " the primary node.") - self.secondaries.append(self.op.snode) _CheckNodeOnline(self, self.op.snode) + _CheckNodeNotDrained(self, self.op.snode) + self.secondaries.append(self.op.snode) nodenames = [pnode.name] + self.secondaries @@ -4503,7 +4931,7 @@ class LUCreateInstance(LogicalUnit): # os verification result = self.rpc.call_os_get(pnode.name, self.op.os_type) result.Raise() - if not isinstance(result.data, objects.OS): + if not isinstance(result.data, objects.OS) or not result.data: raise errors.OpPrereqError("OS '%s' not in supported os list for" " primary node" % self.op.os_type) @@ -4523,8 +4951,6 @@ class LUCreateInstance(LogicalUnit): self.be_full[constants.BE_MEMORY], self.op.hypervisor) - self.instance_status = self.op.start - def Exec(self, feedback_fn): """Create and add the instance to the cluster. @@ -4532,10 +4958,6 @@ class LUCreateInstance(LogicalUnit): instance = self.op.instance_name pnode_name = self.pnode.name - for nic in self.nics: - if nic.mac in (constants.VALUE_AUTO, constants.VALUE_GENERATE): - nic.mac = self.cfg.GenerateMAC() - ht_kind = self.op.hypervisor if ht_kind in constants.HTS_REQ_PORT: network_port = self.cfg.AllocatePort() @@ -4570,7 +4992,7 @@ class LUCreateInstance(LogicalUnit): primary_node=pnode_name, nics=self.nics, disks=disks, disk_template=self.op.disk_template, - admin_up=self.instance_status, + admin_up=False, network_port=network_port, beparams=self.op.beparams, hvparams=self.op.hvparams, @@ -4656,9 +5078,11 @@ class LUCreateInstance(LogicalUnit): % self.op.mode) if self.op.start: + iobj.admin_up = True + self.cfg.Update(iobj) logging.info("Starting instance %s on node %s", instance, pnode_name) feedback_fn("* starting instance...") - result = self.rpc.call_instance_start(pnode_name, iobj, None) + result = self.rpc.call_instance_start(pnode_name, iobj, None, None) msg = result.RemoteFailMsg() if msg: raise errors.OpExecError("Could not start instance: %s" % msg) @@ -4868,6 +5292,7 @@ class LUReplaceDisks(LogicalUnit): n1 = self.new_node = remote_node n2 = self.oth_node = instance.primary_node self.tgt_node = self.sec_node + _CheckNodeNotDrained(self, remote_node) else: raise errors.ProgrammerError("Unhandled disk replace mode") @@ -4930,9 +5355,13 @@ class LUReplaceDisks(LogicalUnit): for node in tgt_node, oth_node: info("checking disk/%d on %s" % (idx, node)) cfg.SetDiskID(dev, node) - if not self.rpc.call_blockdev_find(node, dev): - raise errors.OpExecError("Can't find disk/%d on node %s" % - (idx, node)) + result = self.rpc.call_blockdev_find(node, dev) + msg = result.RemoteFailMsg() + if not msg and not result.payload: + msg = "disk not found" + if msg: + raise errors.OpExecError("Can't find disk/%d on node %s: %s" % + (idx, node, msg)) # Step: check other node consistency self.proc.LogStep(2, steps_total, "check peer consistency") @@ -4995,8 +5424,9 @@ class LUReplaceDisks(LogicalUnit): # build the rename list based on what LVs exist on the node rlist = [] for to_ren in old_lvs: - find_res = self.rpc.call_blockdev_find(tgt_node, to_ren) - if not find_res.failed and find_res.data is not None: # device exists + result = self.rpc.call_blockdev_find(tgt_node, to_ren) + if not result.RemoteFailMsg() and result.payload: + # device exists rlist.append((to_ren, ren_fn(to_ren, temp_suffix))) info("renaming the old LVs on the target node") @@ -5025,10 +5455,10 @@ class LUReplaceDisks(LogicalUnit): result = self.rpc.call_blockdev_addchildren(tgt_node, dev, new_lvs) if result.failed or not result.data: for new_lv in new_lvs: - result = self.rpc.call_blockdev_remove(tgt_node, new_lv) - if result.failed or not result.data: - warning("Can't rollback device %s", hint="manually cleanup unused" - " logical volumes") + msg = self.rpc.call_blockdev_remove(tgt_node, new_lv).RemoteFailMsg() + if msg: + warning("Can't rollback device %s: %s", dev, msg, + hint="cleanup manually the unused logical volumes") raise errors.OpExecError("Can't add local storage to drbd") dev.children = new_lvs @@ -5046,7 +5476,13 @@ class LUReplaceDisks(LogicalUnit): for name, (dev, old_lvs, new_lvs) in iv_names.iteritems(): cfg.SetDiskID(dev, instance.primary_node) result = self.rpc.call_blockdev_find(instance.primary_node, dev) - if result.failed or result.data[5]: + msg = result.RemoteFailMsg() + if not msg and not result.payload: + msg = "disk not found" + if msg: + raise errors.OpExecError("Can't find DRBD device %s: %s" % + (name, msg)) + if result.payload[5]: raise errors.OpExecError("DRBD device %s is degraded!" % name) # Step: remove old storage @@ -5055,9 +5491,10 @@ class LUReplaceDisks(LogicalUnit): info("remove logical volumes for %s" % name) for lv in old_lvs: cfg.SetDiskID(lv, tgt_node) - result = self.rpc.call_blockdev_remove(tgt_node, lv) - if result.failed or not result.data: - warning("Can't remove old LV", hint="manually remove unused LVs") + msg = self.rpc.call_blockdev_remove(tgt_node, lv).RemoteFailMsg() + if msg: + warning("Can't remove old LV: %s" % msg, + hint="manually remove unused LVs") continue def _ExecD8Secondary(self, feedback_fn): @@ -5110,10 +5547,12 @@ class LUReplaceDisks(LogicalUnit): info("checking disk/%d on %s" % (idx, pri_node)) cfg.SetDiskID(dev, pri_node) result = self.rpc.call_blockdev_find(pri_node, dev) - result.Raise() - if not result.data: - raise errors.OpExecError("Can't find disk/%d on node %s" % - (idx, pri_node)) + msg = result.RemoteFailMsg() + if not msg and not result.payload: + msg = "disk not found" + if msg: + raise errors.OpExecError("Can't find disk/%d on node %s: %s" % + (idx, pri_node, msg)) # Step: check other node consistency self.proc.LogStep(2, steps_total, "check peer consistency") @@ -5144,7 +5583,6 @@ class LUReplaceDisks(LogicalUnit): logging.debug("Allocated minors %s" % (minors,)) self.proc.LogStep(4, steps_total, "changing drbd configuration") for idx, (dev, new_minor) in enumerate(zip(instance.disks, minors)): - size = dev.size info("activating a new drbd on %s for disk/%d" % (new_node, idx)) # create new devices on new_node; note that we create two IDs: # one without port, so the drbd will be activated without @@ -5164,11 +5602,12 @@ class LUReplaceDisks(LogicalUnit): new_net_id) new_drbd = objects.Disk(dev_type=constants.LD_DRBD8, logical_id=new_alone_id, - children=dev.children) + children=dev.children, + size=dev.size) try: _CreateSingleBlockDev(self, new_node, instance, new_drbd, _GetInstanceInfoText(instance), False) - except errors.BlockDeviceError: + except errors.GenericError: self.cfg.ReleaseDRBDMinors(instance.name) raise @@ -5176,9 +5615,10 @@ class LUReplaceDisks(LogicalUnit): # we have new devices, shutdown the drbd on the old secondary info("shutting down drbd for disk/%d on old node" % idx) cfg.SetDiskID(dev, old_node) - result = self.rpc.call_blockdev_shutdown(old_node, dev) - if result.failed or not result.data: - warning("Failed to shutdown drbd for disk/%d on old node" % idx, + msg = self.rpc.call_blockdev_shutdown(old_node, dev).RemoteFailMsg() + if msg: + warning("Failed to shutdown drbd for disk/%d on old node: %s" % + (idx, msg), hint="Please cleanup this device manually as soon as possible") info("detaching primary drbds from the network (=> standalone)") @@ -5222,8 +5662,13 @@ class LUReplaceDisks(LogicalUnit): for idx, (dev, old_lvs, _) in iv_names.iteritems(): cfg.SetDiskID(dev, pri_node) result = self.rpc.call_blockdev_find(pri_node, dev) - result.Raise() - if result.data[5]: + msg = result.RemoteFailMsg() + if not msg and not result.payload: + msg = "disk not found" + if msg: + raise errors.OpExecError("Can't find DRBD device disk/%d: %s" % + (idx, msg)) + if result.payload[5]: raise errors.OpExecError("DRBD device disk/%d is degraded!" % idx) self.proc.LogStep(6, steps_total, "removing old storage") @@ -5231,9 +5676,9 @@ class LUReplaceDisks(LogicalUnit): info("remove logical volumes for disk/%d" % idx) for lv in old_lvs: cfg.SetDiskID(lv, old_node) - result = self.rpc.call_blockdev_remove(old_node, lv) - if result.failed or not result.data: - warning("Can't remove LV on old secondary", + msg = self.rpc.call_blockdev_remove(old_node, lv).RemoteFailMsg() + if msg: + warning("Can't remove LV on old secondary: %s", msg, hint="Cleanup stale volumes by hand") def Exec(self, feedback_fn): @@ -5344,13 +5789,10 @@ class LUGrowDisk(LogicalUnit): for node in instance.all_nodes: self.cfg.SetDiskID(disk, node) result = self.rpc.call_blockdev_grow(node, disk, self.op.amount) - result.Raise() - if (not result.data or not isinstance(result.data, (list, tuple)) or - len(result.data) != 2): - raise errors.OpExecError("Grow request failed to node %s" % node) - elif not result.data[0]: + msg = result.RemoteFailMsg() + if msg: raise errors.OpExecError("Grow request failed to node %s: %s" % - (node, result.data[1])) + (node, msg)) disk.RecordGrow(self.op.amount) self.cfg.Update(instance) if self.op.wait_for_sync: @@ -5414,8 +5856,14 @@ class LUQueryInstanceData(NoHooksLU): if not static: self.cfg.SetDiskID(dev, instance.primary_node) dev_pstatus = self.rpc.call_blockdev_find(instance.primary_node, dev) - dev_pstatus.Raise() - dev_pstatus = dev_pstatus.data + if dev_pstatus.offline: + dev_pstatus = None + else: + msg = dev_pstatus.RemoteFailMsg() + if msg: + raise errors.OpExecError("Can't compute disk status for %s: %s" % + (instance.name, msg)) + dev_pstatus = dev_pstatus.payload else: dev_pstatus = None @@ -5429,8 +5877,14 @@ class LUQueryInstanceData(NoHooksLU): if snode and not static: self.cfg.SetDiskID(dev, snode) dev_sstatus = self.rpc.call_blockdev_find(snode, dev) - dev_sstatus.Raise() - dev_sstatus = dev_sstatus.data + if dev_sstatus.offline: + dev_sstatus = None + else: + msg = dev_sstatus.RemoteFailMsg() + if msg: + raise errors.OpExecError("Can't compute disk status for %s: %s" % + (instance.name, msg)) + dev_sstatus = dev_sstatus.payload else: dev_sstatus = None @@ -5449,6 +5903,7 @@ class LUQueryInstanceData(NoHooksLU): "sstatus": dev_sstatus, "children": dev_children, "mode": dev.mode, + "size": dev.size, } return data @@ -5525,8 +5980,6 @@ class LUSetInstanceParams(LogicalUnit): self.op.hvparams or self.op.beparams): raise errors.OpPrereqError("No changes submitted") - utils.CheckBEParams(self.op.beparams) - # Disk validation disk_addremove = 0 for disk_op, disk_dict in self.op.disks: @@ -5576,24 +6029,29 @@ class LUSetInstanceParams(LogicalUnit): # nic_dict should be a dict nic_ip = nic_dict.get('ip', None) if nic_ip is not None: - if nic_ip.lower() == "none": + if nic_ip.lower() == constants.VALUE_NONE: nic_dict['ip'] = None else: if not utils.IsValidIP(nic_ip): raise errors.OpPrereqError("Invalid IP address '%s'" % nic_ip) - # we can only check None bridges and assign the default one - nic_bridge = nic_dict.get('bridge', None) - if nic_bridge is None: - nic_dict['bridge'] = self.cfg.GetDefBridge() - # but we can validate MACs - nic_mac = nic_dict.get('mac', None) - if nic_mac is not None: - if self.cfg.IsMacInUse(nic_mac): - raise errors.OpPrereqError("MAC address %s already in use" - " in cluster" % nic_mac) + + if nic_op == constants.DDM_ADD: + nic_bridge = nic_dict.get('bridge', None) + if nic_bridge is None: + nic_dict['bridge'] = self.cfg.GetDefBridge() + nic_mac = nic_dict.get('mac', None) + if nic_mac is None: + nic_dict['mac'] = constants.VALUE_AUTO + + if 'mac' in nic_dict: + nic_mac = nic_dict['mac'] if nic_mac not in (constants.VALUE_AUTO, constants.VALUE_GENERATE): if not utils.IsValidMac(nic_mac): raise errors.OpPrereqError("Invalid MAC address %s" % nic_mac) + if nic_op != constants.DDM_ADD and nic_mac == constants.VALUE_AUTO: + raise errors.OpPrereqError("'auto' is not a valid MAC address when" + " modifying an existing nic") + if nic_addremove > 1: raise errors.OpPrereqError("Only one NIC add or remove operation" " supported at a time") @@ -5618,7 +6076,37 @@ class LUSetInstanceParams(LogicalUnit): args['memory'] = self.be_new[constants.BE_MEMORY] if constants.BE_VCPUS in self.be_new: args['vcpus'] = self.be_new[constants.BE_VCPUS] - # FIXME: readd disk/nic changes + # TODO: export disk changes. Note: _BuildInstanceHookEnv* don't export disk + # information at all. + if self.op.nics: + args['nics'] = [] + nic_override = dict(self.op.nics) + for idx, nic in enumerate(self.instance.nics): + if idx in nic_override: + this_nic_override = nic_override[idx] + else: + this_nic_override = {} + if 'ip' in this_nic_override: + ip = this_nic_override['ip'] + else: + ip = nic.ip + if 'bridge' in this_nic_override: + bridge = this_nic_override['bridge'] + else: + bridge = nic.bridge + if 'mac' in this_nic_override: + mac = this_nic_override['mac'] + else: + mac = nic.mac + args['nics'].append((ip, bridge, mac)) + if constants.DDM_ADD in nic_override: + ip = nic_override[constants.DDM_ADD].get('ip', None) + bridge = nic_override[constants.DDM_ADD]['bridge'] + mac = nic_override[constants.DDM_ADD]['mac'] + args['nics'].append((ip, bridge, mac)) + elif constants.DDM_REMOVE in nic_override: + del args['nics'][-1] + env = _BuildInstanceHookEnvByObject(self, self.instance, override=args) nl = [self.cfg.GetMasterNode()] + list(self.instance.all_nodes) return env, nl, nl @@ -5629,7 +6117,7 @@ class LUSetInstanceParams(LogicalUnit): This only checks the instance list against the existing names. """ - force = self.force = self.op.force + self.force = self.op.force # checking the new params on the primary/secondary nodes @@ -5648,11 +6136,10 @@ class LUSetInstanceParams(LogicalUnit): del i_hvdict[key] except KeyError: pass - elif val == constants.VALUE_NONE: - i_hvdict[key] = None else: i_hvdict[key] = val cluster = self.cfg.GetClusterInfo() + utils.ForceDictType(i_hvdict, constants.HVS_PARAMETER_TYPES) hv_new = cluster.FillDict(cluster.hvparams[instance.hypervisor], i_hvdict) # local check @@ -5676,6 +6163,7 @@ class LUSetInstanceParams(LogicalUnit): else: i_bedict[key] = val cluster = self.cfg.GetClusterInfo() + utils.ForceDictType(i_bedict, constants.BES_PARAMETER_TYPES) be_new = cluster.FillDict(cluster.beparams[constants.BEGR_DEFAULT], i_bedict) self.be_new = be_new # the new actual values @@ -5699,7 +6187,7 @@ class LUSetInstanceParams(LogicalUnit): self.warn.append("Can't get info from primary node %s" % pnode) else: if not instance_info.failed and instance_info.data: - current_mem = instance_info.data['memory'] + current_mem = int(instance_info.data['memory']) else: # Assume instance not running # (there is a slight race condition here, but it's not very probable, @@ -5734,8 +6222,10 @@ class LUSetInstanceParams(LogicalUnit): raise errors.OpPrereqError("Invalid NIC index %s, valid values" " are 0 to %d" % (nic_op, len(instance.nics))) - nic_bridge = nic_dict.get('bridge', None) - if nic_bridge is not None: + if 'bridge' in nic_dict: + nic_bridge = nic_dict['bridge'] + if nic_bridge is None: + raise errors.OpPrereqError('Cannot set the nic bridge to None') if not self.rpc.call_bridges_exist(pnode, [nic_bridge]): msg = ("Bridge '%s' doesn't exist on one of" " the instance nodes" % nic_bridge) @@ -5743,6 +6233,18 @@ class LUSetInstanceParams(LogicalUnit): self.warn.append(msg) else: raise errors.OpPrereqError(msg) + if 'mac' in nic_dict: + nic_mac = nic_dict['mac'] + if nic_mac is None: + raise errors.OpPrereqError('Cannot set the nic mac to None') + elif nic_mac in (constants.VALUE_AUTO, constants.VALUE_GENERATE): + # otherwise generate the mac + nic_dict['mac'] = self.cfg.GenerateMAC() + else: + # or validate/reserve the current one + if self.cfg.IsMacInUse(nic_mac): + raise errors.OpPrereqError("MAC address %s already in use" + " in cluster" % nic_mac) # DISK processing if self.op.disks and instance.disk_template == constants.DT_DISKLESS: @@ -5795,10 +6297,10 @@ class LUSetInstanceParams(LogicalUnit): device_idx = len(instance.disks) for node, disk in device.ComputeNodeTree(instance.primary_node): self.cfg.SetDiskID(disk, node) - rpc_result = self.rpc.call_blockdev_remove(node, disk) - if rpc_result.failed or not rpc_result.data: - self.proc.LogWarning("Could not remove disk/%d on node %s," - " continuing anyway", device_idx, node) + msg = self.rpc.call_blockdev_remove(node, disk).RemoteFailMsg() + if msg: + self.LogWarning("Could not remove disk/%d on node %s: %s," + " continuing anyway", device_idx, node, msg) result.append(("disk/%d" % device_idx, "remove")) elif disk_op == constants.DDM_ADD: # add a new disk @@ -5845,15 +6347,11 @@ class LUSetInstanceParams(LogicalUnit): del instance.nics[-1] result.append(("nic.%d" % len(instance.nics), "remove")) elif nic_op == constants.DDM_ADD: - # add a new nic - if 'mac' not in nic_dict: - mac = constants.VALUE_GENERATE - else: - mac = nic_dict['mac'] - if mac in (constants.VALUE_AUTO, constants.VALUE_GENERATE): - mac = self.cfg.GenerateMAC() + # mac and bridge should be set, by now + mac = nic_dict['mac'] + bridge = nic_dict['bridge'] new_nic = objects.NIC(mac=mac, ip=nic_dict.get('ip', None), - bridge=nic_dict.get('bridge', None)) + bridge=bridge) instance.nics.append(new_nic) result.append(("nic.%d" % (len(instance.nics) - 1), "add:mac=%s,ip=%s,bridge=%s" % @@ -5867,7 +6365,7 @@ class LUSetInstanceParams(LogicalUnit): # hvparams changes if self.op.hvparams: - instance.hvparams = self.hv_new + instance.hvparams = self.hv_inst for key, val in self.op.hvparams.iteritems(): result.append(("hv/%s" % key, val)) @@ -5942,7 +6440,7 @@ class LUExportInstance(LogicalUnit): # remove it from its current node. In the future we could fix this by: # - making a tasklet to search (share-lock all), then create the new one, # then one to remove, after - # - removing the removal operation altoghether + # - removing the removal operation altogether self.needed_locks[locking.LEVEL_NODE] = locking.ALL_SET def DeclareLocks(self, level): @@ -5983,6 +6481,7 @@ class LUExportInstance(LogicalUnit): # This is wrong node name, not a non-locked node raise errors.OpPrereqError("Wrong node name %s" % self.op.target_node) _CheckNodeOnline(self, self.dst_node.name) + _CheckNodeNotDrained(self, self.dst_node.name) # instance disk type verification for disk in self.instance.disks: @@ -6000,10 +6499,11 @@ class LUExportInstance(LogicalUnit): if self.op.shutdown: # shutdown the instance, but not the disks result = self.rpc.call_instance_shutdown(src_node, instance) - result.Raise() - if not result.data: - raise errors.OpExecError("Could not shutdown instance %s on node %s" % - (instance.name, src_node)) + msg = result.RemoteFailMsg() + if msg: + raise errors.OpExecError("Could not shutdown instance %s on" + " node %s: %s" % + (instance.name, src_node, msg)) vgname = self.cfg.GetVGName() @@ -6014,13 +6514,15 @@ class LUExportInstance(LogicalUnit): for disk in instance.disks: self.cfg.SetDiskID(disk, src_node) + # per-disk results + dresults = [] try: - for disk in instance.disks: + for idx, disk in enumerate(instance.disks): # new_dev_name will be a snapshot of an lvm leaf of the one we passed new_dev_name = self.rpc.call_blockdev_snapshot(src_node, disk) if new_dev_name.failed or not new_dev_name.data: - self.LogWarning("Could not snapshot block device %s on node %s", - disk.logical_id[1], src_node) + self.LogWarning("Could not snapshot disk/%d on node %s", + idx, src_node) snap_disks.append(False) else: new_dev = objects.Disk(dev_type=constants.LD_LV, size=disk.size, @@ -6031,7 +6533,7 @@ class LUExportInstance(LogicalUnit): finally: if self.op.shutdown and instance.admin_up: - result = self.rpc.call_instance_start(src_node, instance, None) + result = self.rpc.call_instance_start(src_node, instance, None, None) msg = result.RemoteFailMsg() if msg: _ShutdownInstanceDisks(self, instance) @@ -6045,18 +6547,24 @@ class LUExportInstance(LogicalUnit): result = self.rpc.call_snapshot_export(src_node, dev, dst_node.name, instance, cluster_name, idx) if result.failed or not result.data: - self.LogWarning("Could not export block device %s from node %s to" - " node %s", dev.logical_id[1], src_node, - dst_node.name) - result = self.rpc.call_blockdev_remove(src_node, dev) - if result.failed or not result.data: - self.LogWarning("Could not remove snapshot block device %s from node" - " %s", dev.logical_id[1], src_node) + self.LogWarning("Could not export disk/%d from node %s to" + " node %s", idx, src_node, dst_node.name) + dresults.append(False) + else: + dresults.append(True) + msg = self.rpc.call_blockdev_remove(src_node, dev).RemoteFailMsg() + if msg: + self.LogWarning("Could not remove snapshot for disk/%d from node" + " %s: %s", idx, src_node, msg) + else: + dresults.append(False) result = self.rpc.call_finalize_export(dst_node.name, instance, snap_disks) + fin_resu = True if result.failed or not result.data: self.LogWarning("Could not finalize export for instance %s on node %s", instance.name, dst_node.name) + fin_resu = False nodelist = self.cfg.GetNodeList() nodelist.remove(dst_node.name) @@ -6073,6 +6581,7 @@ class LUExportInstance(LogicalUnit): if not self.rpc.call_export_remove(node, instance.name): self.LogWarning("Could not remove older export for instance %s" " on node %s", instance.name, node) + return fin_resu, dresults class LURemoveExport(NoHooksLU): @@ -6403,7 +6912,7 @@ class IAllocator(object): cluster_info = cfg.GetClusterInfo() # cluster data data = { - "version": 1, + "version": constants.IALLOCATOR_VERSION, "cluster_name": cfg.GetClusterName(), "cluster_tags": list(cluster_info.GetTags()), "enabled_hypervisors": list(cluster_info.enabled_hypervisors), @@ -6433,10 +6942,11 @@ class IAllocator(object): "primary_ip": ninfo.primary_ip, "secondary_ip": ninfo.secondary_ip, "offline": ninfo.offline, + "drained": ninfo.drained, "master_candidate": ninfo.master_candidate, } - if not ninfo.offline: + if not (ninfo.offline or ninfo.drained): nresult.Raise() if not isinstance(nresult.data, dict): raise errors.OpExecError("Can't get data for node %s" % nname) @@ -6499,6 +7009,8 @@ class IAllocator(object): "disk_template": iinfo.disk_template, "hypervisor": iinfo.hypervisor, } + pir["disk_space_total"] = _ComputeDiskSize(iinfo.disk_template, + pir["disks"]) instance_data[iinfo.name] = pir data["instances"] = instance_data @@ -6591,7 +7103,6 @@ class IAllocator(object): """ if call_fn is None: call_fn = self.lu.rpc.call_iallocator_runner - data = self.in_text result = call_fn(self.lu.cfg.GetMasterNode(), name, self.in_text) result.Raise()