X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/53c776b5b9204d16d0f1ae15c8e2bc9805d88e45..c57f169e84ca5fbcc25af1f4db2014742e6dd45e:/lib/backend.py diff --git a/lib/backend.py b/lib/backend.py index 75c4bdb..b08c937 100644 --- a/lib/backend.py +++ b/lib/backend.py @@ -260,8 +260,9 @@ def AddNode(dsa, dsapub, rsa, rsapub, sshkey, sshpub): priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.GANETI_RUNAS, mkdir=True) except errors.OpExecError, err: - logging.exception("Error while processing user ssh files") - return False + msg = "Error while processing user ssh files" + logging.exception(msg) + return (False, "%s: %s" % (msg, err)) for name, content in [(priv_key, sshkey), (pub_key, sshpub)]: utils.WriteFile(name, data=content, mode=0600) @@ -270,7 +271,7 @@ def AddNode(dsa, dsapub, rsa, rsapub, sshkey, sshpub): utils.RunCmd([constants.SSH_INITD_SCRIPT, "restart"]) - return True + return (True, "Node added successfully") def LeaveCluster(): @@ -426,12 +427,21 @@ def VerifyNode(what, cluster_name): result[constants.NV_VGLIST] = ListVolumeGroups() if constants.NV_VERSION in what: - result[constants.NV_VERSION] = constants.PROTOCOL_VERSION + result[constants.NV_VERSION] = (constants.PROTOCOL_VERSION, + constants.RELEASE_VERSION) if constants.NV_HVINFO in what: hyper = hypervisor.GetHypervisor(what[constants.NV_HVINFO]) result[constants.NV_HVINFO] = hyper.GetNodeInfo() + if constants.NV_DRBDLIST in what: + try: + used_minors = bdev.DRBD8.GetUsedDevs().keys() + except errors.BlockDeviceError, err: + logging.warning("Can't get used minors list", exc_info=True) + used_minors = str(err) + result[constants.NV_DRBDLIST] = used_minors + return result @@ -565,7 +575,6 @@ def GetInstanceList(hypervisor_list): results.extend(names) except errors.HypervisorError, err: logging.exception("Error enumerating instances for hypevisor %s", hname) - # FIXME: should we somehow not propagate this to the master? raise return results @@ -651,15 +660,20 @@ def GetAllInstancesInfo(hypervisor_list): 'state': state, 'time': times, } - if name in output and output[name] != value: - raise errors.HypervisorError("Instance %s running duplicate" - " with different parameters" % name) + if name in output: + # we only check static parameters, like memory and vcpus, + # and not state and time which can change between the + # invocations of the different hypervisors + for key in 'memory', 'vcpus': + if value[key] != output[name][key]: + raise errors.HypervisorError("Instance %s is running twice" + " with different parameters" % name) output[name] = value return output -def AddOSToInstance(instance): +def InstanceOsAdd(instance): """Add an OS to an instance. @type instance: L{objects.Instance} @@ -668,7 +682,15 @@ def AddOSToInstance(instance): @return: the success of the operation """ - inst_os = OSFromDisk(instance.os) + try: + inst_os = OSFromDisk(instance.os) + except errors.InvalidOS, err: + os_name, os_dir, os_err = err.args + if os_dir is None: + return (False, "Can't find OS '%s': %s" % (os_name, os_err)) + else: + return (False, "Error parsing OS '%s' in directory %s: %s" % + (os_name, os_dir, os_err)) create_env = OSEnvironment(instance) @@ -681,9 +703,12 @@ def AddOSToInstance(instance): logging.error("os create command '%s' returned error: %s, logfile: %s," " output: %s", result.cmd, result.fail_reason, logfile, result.output) - return False + lines = [utils.SafeEncode(val) + for val in utils.TailFile(logfile, lines=20)] + return (False, "OS create script failed (%s), last lines in the" + " log file:\n%s" % (result.fail_reason, "\n".join(lines))) - return True + return (True, "Successfully installed") def RunRenameInstance(instance, old_name): @@ -712,9 +737,12 @@ def RunRenameInstance(instance, old_name): if result.failed: logging.error("os create command '%s' returned error: %s output: %s", result.cmd, result.fail_reason, result.output) - return False + lines = [utils.SafeEncode(val) + for val in utils.TailFile(logfile, lines=20)] + return (False, "OS rename script failed (%s), last lines in the" + " log file:\n%s" % (result.fail_reason, "\n".join(lines))) - return True + return (True, "Rename successful") def _GetVGInfo(vg_name): @@ -833,7 +861,7 @@ def _GatherAndLinkBlockDevs(instance): return block_devices -def StartInstance(instance, extra_args): +def StartInstance(instance): """Start an instance. @type instance: L{objects.Instance} @@ -845,24 +873,24 @@ def StartInstance(instance, extra_args): running_instances = GetInstanceList([instance.hypervisor]) if instance.name in running_instances: - return True + return (True, "Already running") try: block_devices = _GatherAndLinkBlockDevs(instance) hyper = hypervisor.GetHypervisor(instance.hypervisor) - hyper.StartInstance(instance, block_devices, extra_args) + hyper.StartInstance(instance, block_devices) except errors.BlockDeviceError, err: logging.exception("Failed to start instance") - return False + return (False, "Block device error: %s" % str(err)) except errors.HypervisorError, err: logging.exception("Failed to start instance") _RemoveBlockDevLinks(instance.name, instance.disks) - return False + return (False, "Hypervisor error: %s" % str(err)) - return True + return (True, "Instance started successfully") -def ShutdownInstance(instance): +def InstanceShutdown(instance): """Shut an instance down. @note: this functions uses polling with a hardcoded timeout. @@ -877,14 +905,15 @@ def ShutdownInstance(instance): running_instances = GetInstanceList([hv_name]) if instance.name not in running_instances: - return True + return (True, "Instance already stopped") hyper = hypervisor.GetHypervisor(hv_name) try: hyper.StopInstance(instance) except errors.HypervisorError, err: - logging.error("Failed to stop instance") - return False + msg = "Failed to stop instance %s: %s" % (instance.name, err) + logging.error(msg) + return (False, msg) # test every 10secs for 2min @@ -895,26 +924,29 @@ def ShutdownInstance(instance): time.sleep(10) else: # the shutdown did not succeed - logging.error("shutdown of '%s' unsuccessful, using destroy", instance) + logging.error("Shutdown of '%s' unsuccessful, using destroy", + instance.name) try: hyper.StopInstance(instance, force=True) except errors.HypervisorError, err: - logging.exception("Failed to stop instance") - return False + msg = "Failed to force stop instance %s: %s" % (instance.name, err) + logging.error(msg) + return (False, msg) time.sleep(1) if instance.name in GetInstanceList([hv_name]): - logging.error("could not shutdown instance '%s' even by destroy", - instance.name) - return False + msg = ("Could not shutdown instance %s even by destroy" % + instance.name) + logging.error(msg) + return (False, msg) _RemoveBlockDevLinks(instance.name, instance.disks) - return True + return (True, "Instance has been shutdown successfully") -def RebootInstance(instance, reboot_type, extra_args): +def InstanceReboot(instance, reboot_type): """Reboot an instance. @type instance: L{objects.Instance} @@ -936,27 +968,91 @@ def RebootInstance(instance, reboot_type, extra_args): running_instances = GetInstanceList([instance.hypervisor]) if instance.name not in running_instances: - logging.error("Cannot reboot instance that is not running") - return False + msg = "Cannot reboot instance %s that is not running" % instance.name + logging.error(msg) + return (False, msg) hyper = hypervisor.GetHypervisor(instance.hypervisor) if reboot_type == constants.INSTANCE_REBOOT_SOFT: try: hyper.RebootInstance(instance) except errors.HypervisorError, err: - logging.exception("Failed to soft reboot instance") - return False + msg = "Failed to soft reboot instance %s: %s" % (instance.name, err) + logging.error(msg) + return (False, msg) elif reboot_type == constants.INSTANCE_REBOOT_HARD: try: - ShutdownInstance(instance) - StartInstance(instance, extra_args) + stop_result = InstanceShutdown(instance) + if not stop_result[0]: + return stop_result + return StartInstance(instance) except errors.HypervisorError, err: - logging.exception("Failed to hard reboot instance") - return False + msg = "Failed to hard reboot instance %s: %s" % (instance.name, err) + logging.error(msg) + return (False, msg) else: - raise errors.ParameterError("reboot_type invalid") + return (False, "Invalid reboot_type received: %s" % (reboot_type,)) - return True + return (True, "Reboot successful") + + +def MigrationInfo(instance): + """Gather information about an instance to be migrated. + + @type instance: L{objects.Instance} + @param instance: the instance definition + + """ + hyper = hypervisor.GetHypervisor(instance.hypervisor) + try: + info = hyper.MigrationInfo(instance) + except errors.HypervisorError, err: + msg = "Failed to fetch migration information" + logging.exception(msg) + return (False, '%s: %s' % (msg, err)) + return (True, info) + + +def AcceptInstance(instance, info, target): + """Prepare the node to accept an instance. + + @type instance: L{objects.Instance} + @param instance: the instance definition + @type info: string/data (opaque) + @param info: migration information, from the source node + @type target: string + @param target: target host (usually ip), on this node + + """ + hyper = hypervisor.GetHypervisor(instance.hypervisor) + try: + hyper.AcceptInstance(instance, info, target) + except errors.HypervisorError, err: + msg = "Failed to accept instance" + logging.exception(msg) + return (False, '%s: %s' % (msg, err)) + return (True, "Accept successfull") + + +def FinalizeMigration(instance, info, success): + """Finalize any preparation to accept an instance. + + @type instance: L{objects.Instance} + @param instance: the instance definition + @type info: string/data (opaque) + @param info: migration information, from the source node + @type success: boolean + @param success: whether the migration was a success or a failure + + """ + hyper = hypervisor.GetHypervisor(instance.hypervisor) + try: + hyper.FinalizeMigration(instance, info, success) + except errors.HypervisorError, err: + msg = "Failed to finalize migration" + logging.exception(msg) + return (False, '%s: %s' % (msg, err)) + return (True, "Migration Finalized") def MigrateInstance(instance, target, live): @@ -986,7 +1082,7 @@ def MigrateInstance(instance, target, live): return (True, "Migration successfull") -def CreateBlockDevice(disk, size, owner, on_primary, info): +def BlockdevCreate(disk, size, owner, on_primary, info): """Creates a block device for an instance. @type disk: L{objects.Disk} @@ -1010,44 +1106,55 @@ def CreateBlockDevice(disk, size, owner, on_primary, info): clist = [] if disk.children: for child in disk.children: - crdev = _RecursiveAssembleBD(child, owner, on_primary) + try: + crdev = _RecursiveAssembleBD(child, owner, on_primary) + except errors.BlockDeviceError, err: + errmsg = "Can't assemble device %s: %s" % (child, err) + logging.error(errmsg) + return False, errmsg if on_primary or disk.AssembleOnSecondary(): # we need the children open in case the device itself has to # be assembled - crdev.Open() + try: + crdev.Open() + except errors.BlockDeviceError, err: + errmsg = "Can't make child '%s' read-write: %s" % (child, err) + logging.error(errmsg) + return False, errmsg clist.append(crdev) + try: - device = bdev.FindDevice(disk.dev_type, disk.physical_id, clist) - if device is not None: - logging.info("removing existing device %s", disk) - device.Remove() + device = bdev.Create(disk.dev_type, disk.physical_id, clist, disk.size) except errors.BlockDeviceError, err: - pass + return False, "Can't create block device: %s" % str(err) - device = bdev.Create(disk.dev_type, disk.physical_id, - clist, size) - if device is None: - raise ValueError("Can't create child device for %s, %s" % - (disk, size)) if on_primary or disk.AssembleOnSecondary(): - if not device.Assemble(): - errorstring = "Can't assemble device after creation" - logging.error(errorstring) - raise errors.BlockDeviceError("%s, very unusual event - check the node" - " daemon logs" % errorstring) + try: + device.Assemble() + except errors.BlockDeviceError, err: + errmsg = ("Can't assemble device after creation, very" + " unusual event: %s" % str(err)) + logging.error(errmsg) + return False, errmsg device.SetSyncSpeed(constants.SYNC_SPEED) if on_primary or disk.OpenOnSecondary(): - device.Open(force=True) + try: + device.Open(force=True) + except errors.BlockDeviceError, err: + errmsg = ("Can't make device r/w after creation, very" + " unusual event: %s" % str(err)) + logging.error(errmsg) + return False, errmsg DevCacheManager.UpdateCache(device.dev_path, owner, on_primary, disk.iv_name) device.SetInfo(info) physical_id = device.unique_id - return physical_id + return True, physical_id -def RemoveBlockDevice(disk): +def BlockdevRemove(disk): """Remove a block device. @note: This is intended to be called recursively. @@ -1058,6 +1165,8 @@ def RemoveBlockDevice(disk): @return: the success of the operation """ + msgs = [] + result = True try: rdev = _RecursiveFindBD(disk) except errors.BlockDeviceError, err: @@ -1066,15 +1175,22 @@ def RemoveBlockDevice(disk): rdev = None if rdev is not None: r_path = rdev.dev_path - result = rdev.Remove() + try: + rdev.Remove() + except errors.BlockDeviceError, err: + msgs.append(str(err)) + result = False if result: DevCacheManager.RemoveCache(r_path) - else: - result = True + if disk.children: for child in disk.children: - result = result and RemoveBlockDevice(child) - return result + c_status, c_msg = BlockdevRemove(child) + result = result and c_status + if c_msg: # not an empty message + msgs.append(c_msg) + + return (result, "; ".join(msgs)) def _RecursiveAssembleBD(disk, owner, as_primary): @@ -1113,11 +1229,12 @@ def _RecursiveAssembleBD(disk, owner, as_primary): if children.count(None) >= mcn: raise cdev = None - logging.debug("Error in child activation: %s", str(err)) + logging.error("Error in child activation (but continuing): %s", + str(err)) children.append(cdev) if as_primary or disk.AssembleOnSecondary(): - r_dev = bdev.Assemble(disk.dev_type, disk.physical_id, children) + r_dev = bdev.Assemble(disk.dev_type, disk.physical_id, children, disk.size) r_dev.SetSyncSpeed(constants.SYNC_SPEED) result = r_dev if as_primary or disk.OpenOnSecondary(): @@ -1130,7 +1247,7 @@ def _RecursiveAssembleBD(disk, owner, as_primary): return result -def AssembleBlockDevice(disk, owner, as_primary): +def BlockdevAssemble(disk, owner, as_primary): """Activate a block device for an instance. This is a wrapper over _RecursiveAssembleBD. @@ -1140,13 +1257,19 @@ def AssembleBlockDevice(disk, owner, as_primary): C{True} for secondary nodes """ - result = _RecursiveAssembleBD(disk, owner, as_primary) - if isinstance(result, bdev.BlockDev): - result = result.dev_path - return result + status = True + result = "no error information" + try: + result = _RecursiveAssembleBD(disk, owner, as_primary) + if isinstance(result, bdev.BlockDev): + result = result.dev_path + except errors.BlockDeviceError, err: + result = "Error while assembling disk: %s" % str(err) + status = False + return (status, result) -def ShutdownBlockDevice(disk): +def BlockdevShutdown(disk): """Shut down a block device. First, if the device is assembled (Attach() is successfull), then @@ -1164,21 +1287,29 @@ def ShutdownBlockDevice(disk): @return: the success of the operation """ + msgs = [] + result = True r_dev = _RecursiveFindBD(disk) if r_dev is not None: r_path = r_dev.dev_path - result = r_dev.Shutdown() - if result: + try: + r_dev.Shutdown() DevCacheManager.RemoveCache(r_path) - else: - result = True + except errors.BlockDeviceError, err: + msgs.append(str(err)) + result = False + if disk.children: for child in disk.children: - result = result and ShutdownBlockDevice(child) - return result + c_status, c_msg = BlockdevShutdown(child) + result = result and c_status + if c_msg: # not an empty message + msgs.append(c_msg) + + return (result, "; ".join(msgs)) -def MirrorAddChildren(parent_cdev, new_cdevs): +def BlockdevAddchildren(parent_cdev, new_cdevs): """Extend a mirrored block device. @type parent_cdev: L{objects.Disk} @@ -1202,7 +1333,7 @@ def MirrorAddChildren(parent_cdev, new_cdevs): return True -def MirrorRemoveChildren(parent_cdev, new_cdevs): +def BlockdevRemovechildren(parent_cdev, new_cdevs): """Shrink a mirrored block device. @type parent_cdev: L{objects.Disk} @@ -1234,7 +1365,7 @@ def MirrorRemoveChildren(parent_cdev, new_cdevs): return True -def GetMirrorStatus(disks): +def BlockdevGetmirrorstatus(disks): """Get the mirroring status of a list of devices. @type disks: list of L{objects.Disk} @@ -1273,10 +1404,10 @@ def _RecursiveFindBD(disk): for chdisk in disk.children: children.append(_RecursiveFindBD(chdisk)) - return bdev.FindDevice(disk.dev_type, disk.physical_id, children) + return bdev.FindDevice(disk.dev_type, disk.physical_id, children, disk.size) -def FindBlockDevice(disk): +def BlockdevFind(disk): """Check if a device is activated. If it is, return informations about the real device. @@ -1289,10 +1420,13 @@ def FindBlockDevice(disk): estimated_time, is_degraded) """ - rbd = _RecursiveFindBD(disk) + try: + rbd = _RecursiveFindBD(disk) + except errors.BlockDeviceError, err: + return (False, str(err)) if rbd is None: - return rbd - return (rbd.dev_path, rbd.major, rbd.minor) + rbd.GetSyncStatus() + return (True, None) + return (True, (rbd.dev_path, rbd.major, rbd.minor) + rbd.GetSyncStatus()) def UploadFile(file_name, data, mode, uid, gid, atime, mtime): @@ -1526,6 +1660,7 @@ def OSEnvironment(instance, debug=0): result = {} result['OS_API_VERSION'] = '%d' % constants.OS_API_VERSION result['INSTANCE_NAME'] = instance.name + result['INSTANCE_OS'] = instance.os result['HYPERVISOR'] = instance.hypervisor result['DISK_COUNT'] = '%d' % len(instance.disks) result['NIC_COUNT'] = '%d' % len(instance.nics) @@ -1537,8 +1672,7 @@ def OSEnvironment(instance, debug=0): str(disk)) real_disk.Open() result['DISK_%d_PATH' % idx] = real_disk.dev_path - # FIXME: When disks will have read-only mode, populate this - result['DISK_%d_ACCESS' % idx] = 'W' + result['DISK_%d_ACCESS' % idx] = disk.mode if constants.HV_DISK_TYPE in instance.hvparams: result['DISK_%d_FRONTEND_TYPE' % idx] = \ instance.hvparams[constants.HV_DISK_TYPE] @@ -1556,9 +1690,13 @@ def OSEnvironment(instance, debug=0): result['NIC_%d_FRONTEND_TYPE' % idx] = \ instance.hvparams[constants.HV_NIC_TYPE] + for source, kind in [(instance.beparams, "BE"), (instance.hvparams, "HV")]: + for key, value in source.items(): + result["INSTANCE_%s_%s" % (kind, key)] = str(value) + return result -def GrowBlockDevice(disk, amount): +def BlockdevGrow(disk, amount): """Grow a stack of block devices. This function is called recursively, with the childrens being the @@ -1584,7 +1722,7 @@ def GrowBlockDevice(disk, amount): return True, None -def SnapshotBlockDevice(disk): +def BlockdevSnapshot(disk): """Create a snapshot copy of a block device. This function is called recursively, and the snapshot is actually created @@ -1599,13 +1737,13 @@ def SnapshotBlockDevice(disk): if disk.children: if len(disk.children) == 1: # only one child, let's recurse on it - return SnapshotBlockDevice(disk.children[0]) + return BlockdevSnapshot(disk.children[0]) else: # more than one child, choose one that matches for child in disk.children: if child.size == disk.size: # return implies breaking the loop - return SnapshotBlockDevice(child) + return BlockdevSnapshot(child) elif disk.dev_type == constants.LD_LV: r_dev = _RecursiveFindBD(disk) if r_dev is not None: @@ -1855,7 +1993,7 @@ def RemoveExport(export): return True -def RenameBlockDevices(devlist): +def BlockdevRename(devlist): """Rename a list of block devices. @type devlist: list of tuples @@ -2097,7 +2235,7 @@ def JobQueueSetDrainFlag(drain_flag): return True -def CloseBlockDevices(instance_name, disks): +def BlockdevClose(instance_name, disks): """Closes the given block devices. This means they will be switched to secondary mode (in case of @@ -2254,7 +2392,7 @@ def DrbdAttachNet(nodes_ip, disks, instance_name, multimaster): # standalone, even though this should not happen with the # new staged way of changing disk configs try: - rd.ReAttachNet(multimaster) + rd.AttachNet(multimaster) except errors.BlockDeviceError, err: return (False, "Can't change network configuration: %s" % str(err)) if all_connected: @@ -2266,7 +2404,10 @@ def DrbdAttachNet(nodes_ip, disks, instance_name, multimaster): if multimaster: # change to primary mode for rd in bdevs: - rd.Open() + try: + rd.Open() + except errors.BlockDeviceError, err: + return (False, "Can't change to primary mode: %s" % str(err)) if multimaster: msg = "multi-master and primary" else: @@ -2365,7 +2506,7 @@ class HooksRunner(object): #logging.exception("Error while closing fd %s", fd) pass - return result == 0, output + return result == 0, utils.SafeEncode(output.strip()) def RunHooks(self, hpath, phase, env): """Run the scripts in the hooks directory.