X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/f96e3c4f756dfbe4cd63f77d5a5d06a7ae09be2b..ba55d062da8dfb89a37afc2f13f2e689d0094829:/lib/backend.py diff --git a/lib/backend.py b/lib/backend.py index c57eeaf..08b65aa 100644 --- a/lib/backend.py +++ b/lib/backend.py @@ -46,6 +46,35 @@ from ganeti import objects from ganeti import ssconf +class RPCFail(Exception): + """Class denoting RPC failure. + + Its argument is the error message. + + """ + +def _Fail(msg, *args, **kwargs): + """Log an error and the raise an RPCFail exception. + + This exception is then handled specially in the ganeti daemon and + turned into a 'failed' return type. As such, this function is a + useful shortcut for logging the error and returning it to the master + daemon. + + @type msg: string + @param msg: the text of the exception + @raise RPCFail + + """ + if args: + msg = msg % args + if "exc" in kwargs and kwargs["exc"]: + logging.exception(msg) + else: + logging.error(msg) + raise RPCFail(msg) + + def _GetConfig(): """Simple wrapper to return a SimpleStore. @@ -260,8 +289,7 @@ def AddNode(dsa, dsapub, rsa, rsapub, sshkey, sshpub): priv_key, pub_key, auth_keys = ssh.GetUserFiles(constants.GANETI_RUNAS, mkdir=True) except errors.OpExecError, err: - logging.exception("Error while processing user ssh files") - return False + _Fail("Error while processing user ssh files: %s", err, exc=True) for name, content in [(priv_key, sshkey), (pub_key, sshpub)]: utils.WriteFile(name, data=content, mode=0600) @@ -270,7 +298,7 @@ def AddNode(dsa, dsapub, rsa, rsapub, sshkey, sshpub): utils.RunCmd([constants.SSH_INITD_SCRIPT, "restart"]) - return True + return (True, "Node added successfully") def LeaveCluster(): @@ -426,12 +454,21 @@ def VerifyNode(what, cluster_name): result[constants.NV_VGLIST] = ListVolumeGroups() if constants.NV_VERSION in what: - result[constants.NV_VERSION] = constants.PROTOCOL_VERSION + result[constants.NV_VERSION] = (constants.PROTOCOL_VERSION, + constants.RELEASE_VERSION) if constants.NV_HVINFO in what: hyper = hypervisor.GetHypervisor(what[constants.NV_HVINFO]) result[constants.NV_HVINFO] = hyper.GetNodeInfo() + if constants.NV_DRBDLIST in what: + try: + used_minors = bdev.DRBD8.GetUsedDevs().keys() + except errors.BlockDeviceError, err: + logging.warning("Can't get used minors list", exc_info=True) + used_minors = str(err) + result[constants.NV_DRBDLIST] = used_minors + return result @@ -565,7 +602,6 @@ def GetInstanceList(hypervisor_list): results.extend(names) except errors.HypervisorError, err: logging.exception("Error enumerating instances for hypevisor %s", hname) - # FIXME: should we somehow not propagate this to the master? raise return results @@ -651,26 +687,43 @@ def GetAllInstancesInfo(hypervisor_list): 'state': state, 'time': times, } - if name in output and output[name] != value: - raise errors.HypervisorError("Instance %s running duplicate" - " with different parameters" % name) + if name in output: + # we only check static parameters, like memory and vcpus, + # and not state and time which can change between the + # invocations of the different hypervisors + for key in 'memory', 'vcpus': + if value[key] != output[name][key]: + raise errors.HypervisorError("Instance %s is running twice" + " with different parameters" % name) output[name] = value return output -def AddOSToInstance(instance): +def InstanceOsAdd(instance, reinstall): """Add an OS to an instance. @type instance: L{objects.Instance} @param instance: Instance whose OS is to be installed + @type reinstall: boolean + @param reinstall: whether this is an instance reinstall @rtype: boolean @return: the success of the operation """ - inst_os = OSFromDisk(instance.os) + try: + inst_os = OSFromDisk(instance.os) + except errors.InvalidOS, err: + os_name, os_dir, os_err = err.args + if os_dir is None: + return (False, "Can't find OS '%s': %s" % (os_name, os_err)) + else: + return (False, "Error parsing OS '%s' in directory %s: %s" % + (os_name, os_dir, os_err)) create_env = OSEnvironment(instance) + if reinstall: + create_env['INSTANCE_REINSTALL'] = "1" logfile = "%s/add-%s-%s-%d.log" % (constants.LOG_OS_DIR, instance.os, instance.name, int(time.time())) @@ -681,9 +734,12 @@ def AddOSToInstance(instance): logging.error("os create command '%s' returned error: %s, logfile: %s," " output: %s", result.cmd, result.fail_reason, logfile, result.output) - return False + lines = [utils.SafeEncode(val) + for val in utils.TailFile(logfile, lines=20)] + return (False, "OS create script failed (%s), last lines in the" + " log file:\n%s" % (result.fail_reason, "\n".join(lines))) - return True + return (True, "Successfully installed") def RunRenameInstance(instance, old_name): @@ -712,9 +768,12 @@ def RunRenameInstance(instance, old_name): if result.failed: logging.error("os create command '%s' returned error: %s output: %s", result.cmd, result.fail_reason, result.output) - return False + lines = [utils.SafeEncode(val) + for val in utils.TailFile(logfile, lines=20)] + return (False, "OS rename script failed (%s), last lines in the" + " log file:\n%s" % (result.fail_reason, "\n".join(lines))) - return True + return (True, "Rename successful") def _GetVGInfo(vg_name): @@ -833,7 +892,7 @@ def _GatherAndLinkBlockDevs(instance): return block_devices -def StartInstance(instance, extra_args): +def StartInstance(instance): """Start an instance. @type instance: L{objects.Instance} @@ -845,24 +904,22 @@ def StartInstance(instance, extra_args): running_instances = GetInstanceList([instance.hypervisor]) if instance.name in running_instances: - return True + return (True, "Already running") try: block_devices = _GatherAndLinkBlockDevs(instance) hyper = hypervisor.GetHypervisor(instance.hypervisor) - hyper.StartInstance(instance, block_devices, extra_args) + hyper.StartInstance(instance, block_devices) except errors.BlockDeviceError, err: - logging.exception("Failed to start instance") - return False + _Fail("Block device error: %s", err, exc=True) except errors.HypervisorError, err: - logging.exception("Failed to start instance") _RemoveBlockDevLinks(instance.name, instance.disks) - return False + _Fail("Hypervisor error: %s", err, exc=True) - return True + return (True, "Instance started successfully") -def ShutdownInstance(instance): +def InstanceShutdown(instance): """Shut an instance down. @note: this functions uses polling with a hardcoded timeout. @@ -877,14 +934,13 @@ def ShutdownInstance(instance): running_instances = GetInstanceList([hv_name]) if instance.name not in running_instances: - return True + return (True, "Instance already stopped") hyper = hypervisor.GetHypervisor(hv_name) try: hyper.StopInstance(instance) except errors.HypervisorError, err: - logging.error("Failed to stop instance") - return False + _Fail("Failed to stop instance %s: %s", instance.name, err) # test every 10secs for 2min @@ -895,26 +951,24 @@ def ShutdownInstance(instance): time.sleep(10) else: # the shutdown did not succeed - logging.error("shutdown of '%s' unsuccessful, using destroy", instance) + logging.error("Shutdown of '%s' unsuccessful, using destroy", + instance.name) try: hyper.StopInstance(instance, force=True) except errors.HypervisorError, err: - logging.exception("Failed to stop instance") - return False + _Fail("Failed to force stop instance %s: %s", instance.name, err) time.sleep(1) if instance.name in GetInstanceList([hv_name]): - logging.error("could not shutdown instance '%s' even by destroy", - instance.name) - return False + _Fail("Could not shutdown instance %s even by destroy", instance.name) _RemoveBlockDevLinks(instance.name, instance.disks) - return True + return (True, "Instance has been shutdown successfully") -def RebootInstance(instance, reboot_type, extra_args): +def InstanceReboot(instance, reboot_type): """Reboot an instance. @type instance: L{objects.Instance} @@ -936,27 +990,79 @@ def RebootInstance(instance, reboot_type, extra_args): running_instances = GetInstanceList([instance.hypervisor]) if instance.name not in running_instances: - logging.error("Cannot reboot instance that is not running") - return False + _Fail("Cannot reboot instance %s that is not running", instance.name) hyper = hypervisor.GetHypervisor(instance.hypervisor) if reboot_type == constants.INSTANCE_REBOOT_SOFT: try: hyper.RebootInstance(instance) except errors.HypervisorError, err: - logging.exception("Failed to soft reboot instance") - return False + _Fail("Failed to soft reboot instance %s: %s", instance.name, err) elif reboot_type == constants.INSTANCE_REBOOT_HARD: try: - ShutdownInstance(instance) - StartInstance(instance, extra_args) + stop_result = InstanceShutdown(instance) + if not stop_result[0]: + return stop_result + return StartInstance(instance) except errors.HypervisorError, err: - logging.exception("Failed to hard reboot instance") - return False + _Fail("Failed to hard reboot instance %s: %s", instance.name, err) else: - raise errors.ParameterError("reboot_type invalid") + _Fail("Invalid reboot_type received: %s", reboot_type) - return True + return (True, "Reboot successful") + + +def MigrationInfo(instance): + """Gather information about an instance to be migrated. + + @type instance: L{objects.Instance} + @param instance: the instance definition + + """ + hyper = hypervisor.GetHypervisor(instance.hypervisor) + try: + info = hyper.MigrationInfo(instance) + except errors.HypervisorError, err: + _Fail("Failed to fetch migration information: %s", err, exc=True) + return (True, info) + + +def AcceptInstance(instance, info, target): + """Prepare the node to accept an instance. + + @type instance: L{objects.Instance} + @param instance: the instance definition + @type info: string/data (opaque) + @param info: migration information, from the source node + @type target: string + @param target: target host (usually ip), on this node + + """ + hyper = hypervisor.GetHypervisor(instance.hypervisor) + try: + hyper.AcceptInstance(instance, info, target) + except errors.HypervisorError, err: + _Fail("Failed to accept instance: %s", err, exc=True) + return (True, "Accept successfull") + + +def FinalizeMigration(instance, info, success): + """Finalize any preparation to accept an instance. + + @type instance: L{objects.Instance} + @param instance: the instance definition + @type info: string/data (opaque) + @param info: migration information, from the source node + @type success: boolean + @param success: whether the migration was a success or a failure + + """ + hyper = hypervisor.GetHypervisor(instance.hypervisor) + try: + hyper.FinalizeMigration(instance, info, success) + except errors.HypervisorError, err: + _Fail("Failed to finalize migration: %s", err, exc=True) + return (True, "Migration Finalized") def MigrateInstance(instance, target, live): @@ -975,18 +1081,16 @@ def MigrateInstance(instance, target, live): - msg is a string with details in case of failure """ - hyper = hypervisor.GetHypervisor(instance.hypervisor_name) + hyper = hypervisor.GetHypervisor(instance.hypervisor) try: hyper.MigrateInstance(instance.name, target, live) except errors.HypervisorError, err: - msg = "Failed to migrate instance: %s" % str(err) - logging.error(msg) - return (False, msg) + _Fail("Failed to migrate instance: %s", err, exc=True) return (True, "Migration successfull") -def CreateBlockDevice(disk, size, owner, on_primary, info): +def BlockdevCreate(disk, size, owner, on_primary, info): """Creates a block device for an instance. @type disk: L{objects.Disk} @@ -1010,44 +1114,45 @@ def CreateBlockDevice(disk, size, owner, on_primary, info): clist = [] if disk.children: for child in disk.children: - crdev = _RecursiveAssembleBD(child, owner, on_primary) + try: + crdev = _RecursiveAssembleBD(child, owner, on_primary) + except errors.BlockDeviceError, err: + _Fail("Can't assemble device %s: %s", child, err) if on_primary or disk.AssembleOnSecondary(): # we need the children open in case the device itself has to # be assembled - crdev.Open() + try: + crdev.Open() + except errors.BlockDeviceError, err: + _Fail("Can't make child '%s' read-write: %s", child, err) clist.append(crdev) + try: - device = bdev.FindDevice(disk.dev_type, disk.physical_id, clist) - if device is not None: - logging.info("removing existing device %s", disk) - device.Remove() + device = bdev.Create(disk.dev_type, disk.physical_id, clist, size) except errors.BlockDeviceError, err: - pass + _Fail("Can't create block device: %s", err) - device = bdev.Create(disk.dev_type, disk.physical_id, - clist, size) - if device is None: - raise ValueError("Can't create child device for %s, %s" % - (disk, size)) if on_primary or disk.AssembleOnSecondary(): - if not device.Assemble(): - errorstring = "Can't assemble device after creation" - logging.error(errorstring) - raise errors.BlockDeviceError("%s, very unusual event - check the node" - " daemon logs" % errorstring) + try: + device.Assemble() + except errors.BlockDeviceError, err: + _Fail("Can't assemble device after creation, unusual event: %s", err) device.SetSyncSpeed(constants.SYNC_SPEED) if on_primary or disk.OpenOnSecondary(): - device.Open(force=True) + try: + device.Open(force=True) + except errors.BlockDeviceError, err: + _Fail("Can't make device r/w after creation, unusual event: %s", err) DevCacheManager.UpdateCache(device.dev_path, owner, on_primary, disk.iv_name) device.SetInfo(info) physical_id = device.unique_id - return physical_id + return True, physical_id -def RemoveBlockDevice(disk): +def BlockdevRemove(disk): """Remove a block device. @note: This is intended to be called recursively. @@ -1058,6 +1163,8 @@ def RemoveBlockDevice(disk): @return: the success of the operation """ + msgs = [] + result = True try: rdev = _RecursiveFindBD(disk) except errors.BlockDeviceError, err: @@ -1066,15 +1173,22 @@ def RemoveBlockDevice(disk): rdev = None if rdev is not None: r_path = rdev.dev_path - result = rdev.Remove() + try: + rdev.Remove() + except errors.BlockDeviceError, err: + msgs.append(str(err)) + result = False if result: DevCacheManager.RemoveCache(r_path) - else: - result = True + if disk.children: for child in disk.children: - result = result and RemoveBlockDevice(child) - return result + c_status, c_msg = BlockdevRemove(child) + result = result and c_status + if c_msg: # not an empty message + msgs.append(c_msg) + + return (result, "; ".join(msgs)) def _RecursiveAssembleBD(disk, owner, as_primary): @@ -1113,7 +1227,8 @@ def _RecursiveAssembleBD(disk, owner, as_primary): if children.count(None) >= mcn: raise cdev = None - logging.debug("Error in child activation: %s", str(err)) + logging.error("Error in child activation (but continuing): %s", + str(err)) children.append(cdev) if as_primary or disk.AssembleOnSecondary(): @@ -1130,7 +1245,7 @@ def _RecursiveAssembleBD(disk, owner, as_primary): return result -def AssembleBlockDevice(disk, owner, as_primary): +def BlockdevAssemble(disk, owner, as_primary): """Activate a block device for an instance. This is a wrapper over _RecursiveAssembleBD. @@ -1140,13 +1255,19 @@ def AssembleBlockDevice(disk, owner, as_primary): C{True} for secondary nodes """ - result = _RecursiveAssembleBD(disk, owner, as_primary) - if isinstance(result, bdev.BlockDev): - result = result.dev_path - return result + status = True + result = "no error information" + try: + result = _RecursiveAssembleBD(disk, owner, as_primary) + if isinstance(result, bdev.BlockDev): + result = result.dev_path + except errors.BlockDeviceError, err: + result = "Error while assembling disk: %s" % str(err) + status = False + return (status, result) -def ShutdownBlockDevice(disk): +def BlockdevShutdown(disk): """Shut down a block device. First, if the device is assembled (Attach() is successfull), then @@ -1164,21 +1285,29 @@ def ShutdownBlockDevice(disk): @return: the success of the operation """ + msgs = [] + result = True r_dev = _RecursiveFindBD(disk) if r_dev is not None: r_path = r_dev.dev_path - result = r_dev.Shutdown() - if result: + try: + r_dev.Shutdown() DevCacheManager.RemoveCache(r_path) - else: - result = True + except errors.BlockDeviceError, err: + msgs.append(str(err)) + result = False + if disk.children: for child in disk.children: - result = result and ShutdownBlockDevice(child) - return result + c_status, c_msg = BlockdevShutdown(child) + result = result and c_status + if c_msg: # not an empty message + msgs.append(c_msg) + return (result, "; ".join(msgs)) -def MirrorAddChildren(parent_cdev, new_cdevs): + +def BlockdevAddchildren(parent_cdev, new_cdevs): """Extend a mirrored block device. @type parent_cdev: L{objects.Disk} @@ -1191,18 +1320,15 @@ def MirrorAddChildren(parent_cdev, new_cdevs): """ parent_bdev = _RecursiveFindBD(parent_cdev) if parent_bdev is None: - logging.error("Can't find parent device") - return False + _Fail("Can't find parent device '%s' in add children", parent_cdev) new_bdevs = [_RecursiveFindBD(disk) for disk in new_cdevs] if new_bdevs.count(None) > 0: - logging.error("Can't find new device(s) to add: %s:%s", - new_bdevs, new_cdevs) - return False + _Fail("Can't find new device(s) to add: %s:%s", new_bdevs, new_cdevs) parent_bdev.AddChildren(new_bdevs) - return True + return (True, None) -def MirrorRemoveChildren(parent_cdev, new_cdevs): +def BlockdevRemovechildren(parent_cdev, new_cdevs): """Shrink a mirrored block device. @type parent_cdev: L{objects.Disk} @@ -1215,26 +1341,23 @@ def MirrorRemoveChildren(parent_cdev, new_cdevs): """ parent_bdev = _RecursiveFindBD(parent_cdev) if parent_bdev is None: - logging.error("Can't find parent in remove children: %s", parent_cdev) - return False + _Fail("Can't find parent device '%s' in remove children", parent_cdev) devs = [] for disk in new_cdevs: rpath = disk.StaticDevPath() if rpath is None: bd = _RecursiveFindBD(disk) if bd is None: - logging.error("Can't find dynamic device %s while removing children", - disk) - return False + _Fail("Can't find device %s while removing children", disk) else: devs.append(bd.dev_path) else: devs.append(rpath) parent_bdev.RemoveChildren(devs) - return True + return (True, None) -def GetMirrorStatus(disks): +def BlockdevGetmirrorstatus(disks): """Get the mirroring status of a list of devices. @type disks: list of L{objects.Disk} @@ -1251,9 +1374,9 @@ def GetMirrorStatus(disks): for dsk in disks: rbd = _RecursiveFindBD(dsk) if rbd is None: - raise errors.BlockDeviceError("Can't find device %s" % str(dsk)) + _Fail("Can't find device %s", dsk) stats.append(rbd.CombinedSyncStatus()) - return stats + return True, stats def _RecursiveFindBD(disk): @@ -1276,7 +1399,7 @@ def _RecursiveFindBD(disk): return bdev.FindDevice(disk.dev_type, disk.physical_id, children) -def FindBlockDevice(disk): +def BlockdevFind(disk): """Check if a device is activated. If it is, return informations about the real device. @@ -1289,10 +1412,13 @@ def FindBlockDevice(disk): estimated_time, is_degraded) """ - rbd = _RecursiveFindBD(disk) + try: + rbd = _RecursiveFindBD(disk) + except errors.BlockDeviceError, err: + _Fail("Failed to find device: %s", err, exc=True) if rbd is None: - return rbd - return (rbd.dev_path, rbd.major, rbd.minor) + rbd.GetSyncStatus() + return (True, None) + return (True, (rbd.dev_path, rbd.major, rbd.minor) + rbd.GetSyncStatus()) def UploadFile(file_name, data, mode, uid, gid, atime, mtime): @@ -1321,27 +1447,30 @@ def UploadFile(file_name, data, mode, uid, gid, atime, mtime): """ if not os.path.isabs(file_name): - logging.error("Filename passed to UploadFile is not absolute: '%s'", - file_name) - return False + _Fail("Filename passed to UploadFile is not absolute: '%s'", file_name) - allowed_files = [ + allowed_files = set([ constants.CLUSTER_CONF_FILE, constants.ETC_HOSTS, constants.SSH_KNOWN_HOSTS_FILE, constants.VNC_PASSWORD_FILE, - ] + constants.RAPI_CERT_FILE, + constants.RAPI_USERS_FILE, + ]) + + for hv_name in constants.HYPER_TYPES: + hv_class = hypervisor.GetHypervisor(hv_name) + allowed_files.update(hv_class.GetAncillaryFiles()) if file_name not in allowed_files: - logging.error("Filename passed to UploadFile not in allowed" - " upload targets: '%s'", file_name) - return False + _Fail("Filename passed to UploadFile not in allowed upload targets: '%s'", + file_name) raw_data = _Decompress(data) utils.WriteFile(file_name, data=raw_data, mode=mode, uid=uid, gid=gid, atime=atime, mtime=mtime) - return True + return (True, "success") def WriteSsconfFiles(values): @@ -1526,6 +1655,7 @@ def OSEnvironment(instance, debug=0): result = {} result['OS_API_VERSION'] = '%d' % constants.OS_API_VERSION result['INSTANCE_NAME'] = instance.name + result['INSTANCE_OS'] = instance.os result['HYPERVISOR'] = instance.hypervisor result['DISK_COUNT'] = '%d' % len(instance.disks) result['NIC_COUNT'] = '%d' % len(instance.nics) @@ -1537,8 +1667,7 @@ def OSEnvironment(instance, debug=0): str(disk)) real_disk.Open() result['DISK_%d_PATH' % idx] = real_disk.dev_path - # FIXME: When disks will have read-only mode, populate this - result['DISK_%d_ACCESS' % idx] = 'W' + result['DISK_%d_ACCESS' % idx] = disk.mode if constants.HV_DISK_TYPE in instance.hvparams: result['DISK_%d_FRONTEND_TYPE' % idx] = \ instance.hvparams[constants.HV_DISK_TYPE] @@ -1551,14 +1680,18 @@ def OSEnvironment(instance, debug=0): result['NIC_%d_MAC' % idx] = nic.mac if nic.ip: result['NIC_%d_IP' % idx] = nic.ip - result['NIC_%d_BRIDGE' % idx] = nic.bridge + result['NIC_%d_MODE' % idx] = nic.nicparams[constants.NIC_MODE] + if nic.nicparams[constants.NIC_MODE] == constants.NIC_MODE_BRIDGED: + result['NIC_%d_BRIDGE' % idx] = nic.nicparams[constants.NIC_LINK] + if nic.nicparams[constants.NIC_LINK]: + result['NIC_%d_LINK' % idx] = nic.nicparams[constants.NIC_LINK] if constants.HV_NIC_TYPE in instance.hvparams: result['NIC_%d_FRONTEND_TYPE' % idx] = \ instance.hvparams[constants.HV_NIC_TYPE] return result -def GrowBlockDevice(disk, amount): +def BlockdevGrow(disk, amount): """Grow a stack of block devices. This function is called recursively, with the childrens being the @@ -1579,12 +1712,12 @@ def GrowBlockDevice(disk, amount): try: r_dev.Grow(amount) except errors.BlockDeviceError, err: - return False, str(err) + _Fail("Failed to grow block device: %s", err, exc=True) return True, None -def SnapshotBlockDevice(disk): +def BlockdevSnapshot(disk): """Create a snapshot copy of a block device. This function is called recursively, and the snapshot is actually created @@ -1599,24 +1732,23 @@ def SnapshotBlockDevice(disk): if disk.children: if len(disk.children) == 1: # only one child, let's recurse on it - return SnapshotBlockDevice(disk.children[0]) + return BlockdevSnapshot(disk.children[0]) else: # more than one child, choose one that matches for child in disk.children: if child.size == disk.size: # return implies breaking the loop - return SnapshotBlockDevice(child) + return BlockdevSnapshot(child) elif disk.dev_type == constants.LD_LV: r_dev = _RecursiveFindBD(disk) if r_dev is not None: # let's stay on the safe side and ask for the full size, for now - return r_dev.Snapshot(disk.size) + return True, r_dev.Snapshot(disk.size) else: - return None + _Fail("Cannot find block device %s", disk) else: - raise errors.ProgrammerError("Cannot snapshot non-lvm block device" - " '%s' of type '%s'" % - (disk.unique_id, disk.dev_type)) + _Fail("Cannot snapshot non-lvm block device '%s' of type '%s'", + disk.unique_id, disk.dev_type) def ExportSnapshot(disk, dest_node, instance, cluster_name, idx): @@ -1648,8 +1780,8 @@ def ExportSnapshot(disk, dest_node, instance, cluster_name, idx): os.mkdir(constants.LOG_OS_DIR, 0750) real_disk = _RecursiveFindBD(disk) if real_disk is None: - raise errors.BlockDeviceError("Block device '%s' is not set up" % - str(disk)) + _Fail("Block device '%s' is not set up", disk) + real_disk.Open() export_env['EXPORT_DEVICE'] = real_disk.dev_path @@ -1678,11 +1810,10 @@ def ExportSnapshot(disk, dest_node, instance, cluster_name, idx): result = utils.RunCmd(command, env=export_env) if result.failed: - logging.error("os snapshot export command '%s' returned error: %s" - " output: %s", command, result.fail_reason, result.output) - return False + _Fail("OS snapshot export command '%s' returned error: %s" + " output: %s", command, result.fail_reason, result.output) - return True + return (True, None) def FinalizeExport(instance, snap_disks): @@ -1855,7 +1986,7 @@ def RemoveExport(export): return True -def RenameBlockDevices(devlist): +def BlockdevRename(devlist): """Rename a list of block devices. @type devlist: list of tuples @@ -1868,10 +1999,12 @@ def RenameBlockDevices(devlist): @return: True if all renames succeeded, False otherwise """ + msgs = [] result = True for disk, unique_id in devlist: dev = _RecursiveFindBD(disk) if dev is None: + msgs.append("Can't find device %s in rename" % str(disk)) result = False continue try: @@ -1886,9 +2019,11 @@ def RenameBlockDevices(devlist): # cache? for now, we only lose lvm data when we rename, which # is less critical than DRBD or MD except errors.BlockDeviceError, err: + msgs.append("Can't rename device '%s' to '%s': %s" % + (dev, unique_id, err)) logging.exception("Can't rename device '%s' to '%s'", dev, unique_id) result = False - return result + return (result, "; ".join(msgs)) def _TransformFileStorageDir(file_storage_dir): @@ -2097,7 +2232,7 @@ def JobQueueSetDrainFlag(drain_flag): return True -def CloseBlockDevices(instance_name, disks): +def BlockdevClose(instance_name, disks): """Closes the given block devices. This means they will be switched to secondary mode (in case of @@ -2118,7 +2253,7 @@ def CloseBlockDevices(instance_name, disks): for cf in disks: rd = _RecursiveFindBD(cf) if rd is None: - return (False, "Can't find device %s" % cf) + _Fail("Can't find device %s", cf) bdevs.append(rd) msg = [] @@ -2177,6 +2312,147 @@ def DemoteFromMC(): return (True, "Done") +def _FindDisks(nodes_ip, disks): + """Sets the physical ID on disks and returns the block devices. + + """ + # set the correct physical ID + my_name = utils.HostInfo().name + for cf in disks: + cf.SetPhysicalID(my_name, nodes_ip) + + bdevs = [] + + for cf in disks: + rd = _RecursiveFindBD(cf) + if rd is None: + return (False, "Can't find device %s" % cf) + bdevs.append(rd) + return (True, bdevs) + + +def DrbdDisconnectNet(nodes_ip, disks): + """Disconnects the network on a list of drbd devices. + + """ + status, bdevs = _FindDisks(nodes_ip, disks) + if not status: + return status, bdevs + + # disconnect disks + for rd in bdevs: + try: + rd.DisconnectNet() + except errors.BlockDeviceError, err: + _Fail("Can't change network configuration to standalone mode: %s", + err, exc=True) + return (True, "All disks are now disconnected") + + +def DrbdAttachNet(nodes_ip, disks, instance_name, multimaster): + """Attaches the network on a list of drbd devices. + + """ + status, bdevs = _FindDisks(nodes_ip, disks) + if not status: + return status, bdevs + + if multimaster: + for idx, rd in enumerate(bdevs): + try: + _SymlinkBlockDev(instance_name, rd.dev_path, idx) + except EnvironmentError, err: + _Fail("Can't create symlink: %s", err) + # reconnect disks, switch to new master configuration and if + # needed primary mode + for rd in bdevs: + try: + rd.AttachNet(multimaster) + except errors.BlockDeviceError, err: + _Fail("Can't change network configuration: %s", err) + # wait until the disks are connected; we need to retry the re-attach + # if the device becomes standalone, as this might happen if the one + # node disconnects and reconnects in a different mode before the + # other node reconnects; in this case, one or both of the nodes will + # decide it has wrong configuration and switch to standalone + RECONNECT_TIMEOUT = 2 * 60 + sleep_time = 0.100 # start with 100 miliseconds + timeout_limit = time.time() + RECONNECT_TIMEOUT + while time.time() < timeout_limit: + all_connected = True + for rd in bdevs: + stats = rd.GetProcStatus() + if not (stats.is_connected or stats.is_in_resync): + all_connected = False + if stats.is_standalone: + # peer had different config info and this node became + # standalone, even though this should not happen with the + # new staged way of changing disk configs + try: + rd.ReAttachNet(multimaster) + except errors.BlockDeviceError, err: + _Fail("Can't change network configuration: %s", err) + if all_connected: + break + time.sleep(sleep_time) + sleep_time = min(5, sleep_time * 1.5) + if not all_connected: + return (False, "Timeout in disk reconnecting") + if multimaster: + # change to primary mode + for rd in bdevs: + try: + rd.Open() + except errors.BlockDeviceError, err: + _Fail("Can't change to primary mode: %s", err) + if multimaster: + msg = "multi-master and primary" + else: + msg = "single-master" + return (True, "Disks are now configured as %s" % msg) + + +def DrbdWaitSync(nodes_ip, disks): + """Wait until DRBDs have synchronized. + + """ + status, bdevs = _FindDisks(nodes_ip, disks) + if not status: + return status, bdevs + + min_resync = 100 + alldone = True + failure = False + for rd in bdevs: + stats = rd.GetProcStatus() + if not (stats.is_connected or stats.is_in_resync): + failure = True + break + alldone = alldone and (not stats.is_in_resync) + if stats.sync_percent is not None: + min_resync = min(min_resync, stats.sync_percent) + return (not failure, (alldone, min_resync)) + + +def PowercycleNode(hypervisor_type): + """Hard-powercycle the node. + + Because we need to return first, and schedule the powercycle in the + background, we won't be able to report failures nicely. + + """ + hyper = hypervisor.GetHypervisor(hypervisor_type) + try: + pid = os.fork() + except OSError, err: + # if we can't fork, we'll pretend that we're in the child process + pid = 0 + if pid > 0: + return (True, "Reboot scheduled in 5 seconds") + time.sleep(5) + hyper.PowercycleNode() + + class HooksRunner(object): """Hook runner. @@ -2246,7 +2522,7 @@ class HooksRunner(object): #logging.exception("Error while closing fd %s", fd) pass - return result == 0, output + return result == 0, utils.SafeEncode(output.strip()) def RunHooks(self, hpath, phase, env): """Run the scripts in the hooks directory.