X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/5a47ad2092e66b2f4f8c3de00f21da736bd7fd03..29df1f02bbc588ace14f4a40e346aa502b686462:/lib/bdev.py diff --git a/lib/bdev.py b/lib/bdev.py index 7d717c9..c7d2a83 100644 --- a/lib/bdev.py +++ b/lib/bdev.py @@ -24,6 +24,7 @@ import re import time import errno +import pyparsing as pyp from ganeti import utils from ganeti import logger @@ -93,7 +94,6 @@ class BlockDev(object): STATUS_ONLINE: "online", } - def __init__(self, unique_id, children): self._children = children self.dev_path = None @@ -101,7 +101,6 @@ class BlockDev(object): self.major = None self.minor = None - def Assemble(self): """Assemble the device from its components. @@ -123,28 +122,31 @@ class BlockDev(object): status = status and child.Assemble() if not status: break - status = status and child.Open() + + try: + child.Open() + except errors.BlockDeviceError: + for child in self._children: + child.Shutdown() + raise if not status: for child in self._children: child.Shutdown() return status - def Attach(self): """Find a device which matches our config and attach to it. """ raise NotImplementedError - def Close(self): """Notifies that the device will no longer be used for I/O. """ raise NotImplementedError - @classmethod def Create(cls, unique_id, children, size): """Create the device. @@ -159,7 +161,6 @@ class BlockDev(object): """ raise NotImplementedError - def Remove(self): """Remove this device. @@ -170,6 +171,13 @@ class BlockDev(object): """ raise NotImplementedError + def Rename(self, new_id): + """Rename this device. + + This may or may not make sense for a given device type. + + """ + raise NotImplementedError def GetStatus(self): """Return the status of the device. @@ -177,7 +185,6 @@ class BlockDev(object): """ raise NotImplementedError - def Open(self, force=False): """Make the device ready for use. @@ -190,7 +197,6 @@ class BlockDev(object): """ raise NotImplementedError - def Shutdown(self): """Shut down the device, freeing its children. @@ -201,7 +207,6 @@ class BlockDev(object): """ raise NotImplementedError - def SetSyncSpeed(self, speed): """Adjust the sync speed of the mirror. @@ -214,7 +219,6 @@ class BlockDev(object): result = result and child.SetSyncSpeed(speed) return result - def GetSyncStatus(self): """Returns the sync status of the device. @@ -222,17 +226,23 @@ class BlockDev(object): status of the mirror. Returns: - (sync_percent, estimated_time, is_degraded) + (sync_percent, estimated_time, is_degraded, ldisk) + + If sync_percent is None, it means the device is not syncing. - If sync_percent is None, it means all is ok If estimated_time is None, it means we can't estimate - the time needed, otherwise it's the time left in seconds + the time needed, otherwise it's the time left in seconds. + If is_degraded is True, it means the device is missing redundancy. This is usually a sign that something went wrong in the device setup, if sync_percent is None. + The ldisk parameter represents the degradation of the local + data. This is only valid for some devices, the rest will always + return False (not degraded). + """ - return None, None, False + return None, None, False, False def CombinedSyncStatus(self): @@ -243,10 +253,10 @@ class BlockDev(object): children. """ - min_percent, max_time, is_degraded = self.GetSyncStatus() + min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus() if self._children: for child in self._children: - c_percent, c_time, c_degraded = child.GetSyncStatus() + c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus() if min_percent is None: min_percent = c_percent elif c_percent is not None: @@ -256,7 +266,8 @@ class BlockDev(object): elif c_time is not None: max_time = max(max_time, c_time) is_degraded = is_degraded or c_degraded - return min_percent, max_time, is_degraded + ldisk = ldisk or c_ldisk + return min_percent, max_time, is_degraded, ldisk def SetInfo(self, text): @@ -292,7 +303,6 @@ class LogicalVolume(BlockDev): self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name) self.Attach() - @classmethod def Create(cls, unique_id, children, size): """Create a new logical volume. @@ -367,19 +377,36 @@ class LogicalVolume(BlockDev): return not result.failed + def Rename(self, new_id): + """Rename this logical volume. + + """ + if not isinstance(new_id, (tuple, list)) or len(new_id) != 2: + raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id) + new_vg, new_name = new_id + if new_vg != self._vg_name: + raise errors.ProgrammerError("Can't move a logical volume across" + " volume groups (from %s to to %s)" % + (self._vg_name, new_vg)) + result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name]) + if result.failed: + raise errors.BlockDeviceError("Failed to rename the logical volume: %s" % + result.output) + self._lv_name = new_name + self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name) def Attach(self): """Attach to an existing LV. This method will try to see if an existing and active LV exists - which matches the our name. If so, its major/minor will be + which matches our name. If so, its major/minor will be recorded. """ result = utils.RunCmd(["lvdisplay", self.dev_path]) if result.failed: - logger.Error("Can't find LV %s: %s" % - (self.dev_path, result.fail_reason)) + logger.Error("Can't find LV %s: %s, %s" % + (self.dev_path, result.fail_reason, result.output)) return False match = re.compile("^ *Block device *([0-9]+):([0-9]+).*$") for line in result.stdout.splitlines(): @@ -390,16 +417,18 @@ class LogicalVolume(BlockDev): return True return False - def Assemble(self): """Assemble the device. - This is a no-op for the LV device type. Eventually, we could - lvchange -ay here if we see that the LV is not active. + We alway run `lvchange -ay` on the LV to ensure it's active before + use, as there were cases when xenvg was not active after boot + (also possibly after disk issues). """ - return True - + result = utils.RunCmd(["lvchange", "-ay", self.dev_path]) + if result.failed: + logger.Error("Can't activate lv %s: %s" % (self.dev_path, result.output)) + return not result.failed def Shutdown(self): """Shutdown the device. @@ -410,7 +439,6 @@ class LogicalVolume(BlockDev): """ return True - def GetStatus(self): """Return the status of the device. @@ -439,6 +467,39 @@ class LogicalVolume(BlockDev): return retval + def GetSyncStatus(self): + """Returns the sync status of the device. + + If this device is a mirroring device, this function returns the + status of the mirror. + + Returns: + (sync_percent, estimated_time, is_degraded, ldisk) + + For logical volumes, sync_percent and estimated_time are always + None (no recovery in progress, as we don't handle the mirrored LV + case). The is_degraded parameter is the inverse of the ldisk + parameter. + + For the ldisk parameter, we check if the logical volume has the + 'virtual' type, which means it's not backed by existing storage + anymore (read from it return I/O error). This happens after a + physical disk failure and subsequent 'vgreduce --removemissing' on + the volume group. + + """ + result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path]) + if result.failed: + logger.Error("Can't display lv: %s" % result.fail_reason) + return None, None, True, True + out = result.stdout.strip() + # format: type/permissions/alloc/fixed_minor/state/open + if len(out) != 6: + logger.Debug("Error in lvs output: attrs=%s, len != 6" % out) + return None, None, True, True + ldisk = out[0] == 'v' # virtual volume, i.e. doesn't have + # backing storage + return None, None, ldisk, ldisk def Open(self, force=False): """Make the device ready for I/O. @@ -446,8 +507,7 @@ class LogicalVolume(BlockDev): This is a no-op for the LV device type. """ - return True - + pass def Close(self): """Notifies that the device will no longer be used for I/O. @@ -455,8 +515,7 @@ class LogicalVolume(BlockDev): This is a no-op for the LV device type. """ - return True - + pass def Snapshot(self, size): """Create a snapshot copy of an lvm block device. @@ -487,7 +546,6 @@ class LogicalVolume(BlockDev): return snap_name - def SetInfo(self, text): """Update metadata with info text. @@ -517,7 +575,6 @@ class MDRaid1(BlockDev): self.major = 9 self.Attach() - def Attach(self): """Find an array which matches our config and attach to it. @@ -533,7 +590,6 @@ class MDRaid1(BlockDev): return (minor is not None) - @staticmethod def _GetUsedDevs(): """Compute the list of in-use MD devices. @@ -556,7 +612,6 @@ class MDRaid1(BlockDev): return used_md - @staticmethod def _GetDevInfo(minor): """Get info about a MD device. @@ -579,7 +634,6 @@ class MDRaid1(BlockDev): retval["state"] = kv[1].split(", ") return retval - @staticmethod def _FindUnusedMinor(): """Compute an unused MD minor. @@ -598,7 +652,6 @@ class MDRaid1(BlockDev): raise errors.BlockDeviceError("Can't find a free MD minor") return i - @classmethod def _FindMDByUUID(cls, uuid): """Find the minor of an MD array with a given UUID. @@ -611,7 +664,6 @@ class MDRaid1(BlockDev): return minor return None - @staticmethod def _ZeroSuperblock(dev_path): """Zero the possible locations for an MD superblock. @@ -689,7 +741,6 @@ class MDRaid1(BlockDev): return None return MDRaid1(info["uuid"], children) - def Remove(self): """Stub remove function for MD RAID 1 arrays. @@ -699,61 +750,78 @@ class MDRaid1(BlockDev): #TODO: maybe zero superblock on child devices? return self.Shutdown() + def Rename(self, new_id): + """Rename a device. + + This is not supported for md raid1 devices. - def AddChild(self, device): - """Add a new member to the md raid1. + """ + raise errors.ProgrammerError("Can't rename a md raid1 device") + + def AddChildren(self, devices): + """Add new member(s) to the md raid1. """ if self.minor is None and not self.Attach(): raise errors.BlockDeviceError("Can't attach to device") - if device.dev_path is None: - raise errors.BlockDeviceError("New child is not initialised") - result = utils.RunCmd(["mdadm", "-a", self.dev_path, device.dev_path]) + + args = ["mdadm", "-a", self.dev_path] + for dev in devices: + if dev.dev_path is None: + raise errors.BlockDeviceError("Child '%s' is not initialised" % dev) + dev.Open() + args.append(dev.dev_path) + result = utils.RunCmd(args) if result.failed: raise errors.BlockDeviceError("Failed to add new device to array: %s" % result.output) - new_len = len(self._children) + 1 + new_len = len(self._children) + len(devices) result = utils.RunCmd(["mdadm", "--grow", self.dev_path, "-n", new_len]) if result.failed: raise errors.BlockDeviceError("Can't grow md array: %s" % result.output) - self._children.append(device) - + self._children.extend(devices) - def RemoveChild(self, dev_path): - """Remove member from the md raid1. + def RemoveChildren(self, devices): + """Remove member(s) from the md raid1. """ if self.minor is None and not self.Attach(): raise errors.BlockDeviceError("Can't attach to device") - if len(self._children) == 1: - raise errors.BlockDeviceError("Can't reduce member when only one" - " child left") - for device in self._children: - if device.dev_path == dev_path: - break - else: - raise errors.BlockDeviceError("Can't find child with this path") - new_len = len(self._children) - 1 - result = utils.RunCmd(["mdadm", "-f", self.dev_path, dev_path]) + new_len = len(self._children) - len(devices) + if new_len < 1: + raise errors.BlockDeviceError("Can't reduce to less than one child") + args = ["mdadm", "-f", self.dev_path] + orig_devs = [] + for dev in devices: + args.append(dev) + for c in self._children: + if c.dev_path == dev: + orig_devs.append(c) + break + else: + raise errors.BlockDeviceError("Can't find device '%s' for removal" % + dev) + result = utils.RunCmd(args) if result.failed: - raise errors.BlockDeviceError("Failed to mark device as failed: %s" % + raise errors.BlockDeviceError("Failed to mark device(s) as failed: %s" % result.output) # it seems here we need a short delay for MD to update its # superblocks time.sleep(0.5) - result = utils.RunCmd(["mdadm", "-r", self.dev_path, dev_path]) + args[1] = "-r" + result = utils.RunCmd(args) if result.failed: - raise errors.BlockDeviceError("Failed to remove device from array:" - " %s" % result.output) + raise errors.BlockDeviceError("Failed to remove device(s) from array:" + " %s" % result.output) result = utils.RunCmd(["mdadm", "--grow", "--force", self.dev_path, "-n", new_len]) if result.failed: raise errors.BlockDeviceError("Can't shrink md array: %s" % result.output) - self._children.remove(device) - + for dev in orig_devs: + self._children.remove(dev) def GetStatus(self): """Return the status of the device. @@ -766,7 +834,6 @@ class MDRaid1(BlockDev): retval = self.STATUS_ONLINE return retval - def _SetFromMinor(self, minor): """Set our parameters based on the given minor. @@ -776,7 +843,6 @@ class MDRaid1(BlockDev): self.minor = minor self.dev_path = "/dev/md%d" % minor - def Assemble(self): """Assemble the MD device. @@ -807,7 +873,6 @@ class MDRaid1(BlockDev): self.minor = free_minor return not result.failed - def Shutdown(self): """Tear down the MD array. @@ -827,7 +892,6 @@ class MDRaid1(BlockDev): self.dev_path = None return True - def SetSyncSpeed(self, kbytes): """Set the maximum sync speed for the MD array. @@ -848,16 +912,17 @@ class MDRaid1(BlockDev): f.close() return result - def GetSyncStatus(self): """Returns the sync status of the device. Returns: - (sync_percent, estimated_time) + (sync_percent, estimated_time, is_degraded, ldisk) If sync_percent is None, it means all is ok If estimated_time is None, it means we can't esimate - the time needed, otherwise it's the time left in seconds + the time needed, otherwise it's the time left in seconds. + + The ldisk parameter is always true for MD devices. """ if self.minor is None and not self.Attach(): @@ -871,12 +936,12 @@ class MDRaid1(BlockDev): sync_status = f.readline().strip() f.close() if sync_status == "idle": - return None, None, not is_clean + return None, None, not is_clean, False f = file(sys_path + "sync_completed") sync_completed = f.readline().strip().split(" / ") f.close() if len(sync_completed) != 2: - return 0, None, not is_clean + return 0, None, not is_clean, False sync_done, sync_total = [float(i) for i in sync_completed] sync_percent = 100.0*sync_done/sync_total f = file(sys_path + "sync_speed") @@ -885,8 +950,7 @@ class MDRaid1(BlockDev): time_est = None else: time_est = (sync_total - sync_done) / 2 / sync_speed_k - return sync_percent, time_est, not is_clean - + return sync_percent, time_est, not is_clean, False def Open(self, force=False): """Make the device ready for I/O. @@ -895,8 +959,7 @@ class MDRaid1(BlockDev): the 2.6.18's new array_state thing. """ - return True - + pass def Close(self): """Notifies that the device will no longer be used for I/O. @@ -905,7 +968,7 @@ class MDRaid1(BlockDev): `Open()`. """ - return True + pass class BaseDRBD(BlockDev): @@ -916,7 +979,8 @@ class BaseDRBD(BlockDev): """ _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)" - r" \(api:(\d+)/proto:(\d+)\)") + r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)") + _DRBD_MAJOR = 147 _ST_UNCONFIGURED = "Unconfigured" _ST_WFCONNECTION = "WFConnection" @@ -966,7 +1030,13 @@ class BaseDRBD(BlockDev): def _GetVersion(cls): """Return the DRBD version. - This will return a list [k_major, k_minor, k_point, api, proto]. + This will return a dict with keys: + k_major, + k_minor, + k_point, + api, + proto, + proto2 (only on drbd > 8.2.X) """ proc_data = cls._GetProcData() @@ -975,7 +1045,18 @@ class BaseDRBD(BlockDev): if not version: raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" % first_line) - return [int(val) for val in version.groups()] + + values = version.groups() + retval = {'k_major': int(values[0]), + 'k_minor': int(values[1]), + 'k_point': int(values[2]), + 'api': int(values[3]), + 'proto': int(values[4]), + } + if values[5] is not None: + retval['proto2'] = values[5] + + return retval @staticmethod def _DevPath(minor): @@ -1017,6 +1098,40 @@ class BaseDRBD(BlockDev): self.minor = minor self.dev_path = self._DevPath(minor) + @staticmethod + def _CheckMetaSize(meta_device): + """Check if the given meta device looks like a valid one. + + This currently only check the size, which must be around + 128MiB. + + """ + result = utils.RunCmd(["blockdev", "--getsize", meta_device]) + if result.failed: + logger.Error("Failed to get device size: %s" % result.fail_reason) + return False + try: + sectors = int(result.stdout) + except ValueError: + logger.Error("Invalid output from blockdev: '%s'" % result.stdout) + return False + bytes = sectors * 512 + if bytes < 128 * 1024 * 1024: # less than 128MiB + logger.Error("Meta device too small (%.2fMib)" % (bytes / 1024 / 1024)) + return False + if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM + logger.Error("Meta device too big (%.2fMiB)" % (bytes / 1024 / 1024)) + return False + return True + + def Rename(self, new_id): + """Rename a device. + + This is not supported for drbd devices. + + """ + raise errors.ProgrammerError("Can't rename a drbd device") + class DRBDev(BaseDRBD): """DRBD block device. @@ -1034,12 +1149,12 @@ class DRBDev(BaseDRBD): def __init__(self, unique_id, children): super(DRBDev, self).__init__(unique_id, children) self.major = self._DRBD_MAJOR - [kmaj, kmin, kfix, api, proto] = self._GetVersion() - if kmaj != 0 and kmin != 7: + version = self._GetVersion() + if version['k_major'] != 0 and version['k_minor'] != 7: raise errors.BlockDeviceError("Mismatch in DRBD kernel version and" " requested ganeti usage: kernel is" - " %s.%s, ganeti wants 0.7" % (kmaj, kmin)) - + " %s.%s, ganeti wants 0.7" % + (version['k_major'], version['k_minor'])) if len(children) != 2: raise ValueError("Invalid configuration data %s" % str(children)) if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4: @@ -1113,7 +1228,6 @@ class DRBDev(BaseDRBD): continue return data - def _MatchesLocal(self, info): """Test if our local config matches with an existing device. @@ -1141,7 +1255,6 @@ class DRBDev(BaseDRBD): info["meta_index"] == -1) return retval - def _MatchesNet(self, info): """Test if our network config matches with an existing device. @@ -1167,34 +1280,6 @@ class DRBDev(BaseDRBD): info["remote_addr"] == (self._rhost, self._rport)) return retval - - @staticmethod - def _IsValidMeta(meta_device): - """Check if the given meta device looks like a valid one. - - This currently only check the size, which must be around - 128MiB. - - """ - result = utils.RunCmd(["blockdev", "--getsize", meta_device]) - if result.failed: - logger.Error("Failed to get device size: %s" % result.fail_reason) - return False - try: - sectors = int(result.stdout) - except ValueError: - logger.Error("Invalid output from blockdev: '%s'" % result.stdout) - return False - bytes = sectors * 512 - if bytes < 128*1024*1024: # less than 128MiB - logger.Error("Meta device too small (%.2fMib)" % (bytes/1024/1024)) - return False - if bytes > (128+32)*1024*1024: # account for an extra (big) PE on LVM - logger.Error("Meta device too big (%.2fMiB)" % (bytes/1024/1024)) - return False - return True - - @classmethod def _AssembleLocal(cls, minor, backend, meta): """Configure the local part of a DRBD device. @@ -1203,7 +1288,7 @@ class DRBDev(BaseDRBD): device. And it must be done only once. """ - if not cls._IsValidMeta(meta): + if not cls._CheckMetaSize(meta): return False result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk", backend, meta, "0", "-e", "detach"]) @@ -1211,7 +1296,6 @@ class DRBDev(BaseDRBD): logger.Error("Can't attach local disk: %s" % result.output) return not result.failed - @classmethod def _ShutdownLocal(cls, minor): """Detach from the local device. @@ -1225,7 +1309,6 @@ class DRBDev(BaseDRBD): logger.Error("Can't detach local device: %s" % result.output) return not result.failed - @staticmethod def _ShutdownAll(minor): """Deactivate the device. @@ -1238,7 +1321,6 @@ class DRBDev(BaseDRBD): logger.Error("Can't shutdown drbd device: %s" % result.output) return not result.failed - @classmethod def _AssembleNet(cls, minor, net_info, protocol): """Configure the network part of the device. @@ -1278,7 +1360,6 @@ class DRBDev(BaseDRBD): return False return True - @classmethod def _ShutdownNet(cls, minor): """Disconnect from the remote peer. @@ -1287,10 +1368,10 @@ class DRBDev(BaseDRBD): """ result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"]) - logger.Error("Can't shutdown network: %s" % result.output) + if result.failed: + logger.Error("Can't shutdown network: %s" % result.output) return not result.failed - def Assemble(self): """Assemble the drbd. @@ -1341,7 +1422,6 @@ class DRBDev(BaseDRBD): self._SetFromMinor(minor) return True - def Shutdown(self): """Shutdown the DRBD device. @@ -1355,7 +1435,6 @@ class DRBDev(BaseDRBD): self.dev_path = None return True - def Attach(self): """Find a DRBD device which matches our config and attach to it. @@ -1383,7 +1462,6 @@ class DRBDev(BaseDRBD): self._SetFromMinor(minor) return minor is not None - def Open(self, force=False): """Make the local state primary. @@ -1401,10 +1479,9 @@ class DRBDev(BaseDRBD): cmd.append("--do-what-I-say") result = utils.RunCmd(cmd) if result.failed: - logger.Error("Can't make drbd device primary: %s" % result.output) - return False - return True - + msg = ("Can't make drbd device primary: %s" % result.output) + logger.Error(msg) + raise errors.BlockDeviceError(msg) def Close(self): """Make the local state secondary. @@ -1417,9 +1494,10 @@ class DRBDev(BaseDRBD): raise errors.BlockDeviceError("Can't find device") result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"]) if result.failed: - logger.Error("Can't switch drbd device to secondary: %s" % result.output) - raise errors.BlockDeviceError("Can't switch drbd device to secondary") - + msg = ("Can't switch drbd device to" + " secondary: %s" % result.output) + logger.Error(msg) + raise errors.BlockDeviceError(msg) def SetSyncSpeed(self, kbytes): """Set the speed of the DRBD syncer. @@ -1435,16 +1513,18 @@ class DRBDev(BaseDRBD): logger.Error("Can't change syncer rate: %s " % result.fail_reason) return not result.failed and children_result - def GetSyncStatus(self): """Returns the sync status of the device. Returns: - (sync_percent, estimated_time) + (sync_percent, estimated_time, is_degraded, ldisk) If sync_percent is None, it means all is ok If estimated_time is None, it means we can't esimate - the time needed, otherwise it's the time left in seconds + the time needed, otherwise it's the time left in seconds. + + The ldisk parameter will be returned as True, since the DRBD7 + devices have not been converted. """ if self.minor is None and not self.Attach(): @@ -1471,10 +1551,7 @@ class DRBDev(BaseDRBD): self.minor) client_state = match.group(1) is_degraded = client_state != "Connected" - return sync_percent, est_time, is_degraded - - - + return sync_percent, est_time, is_degraded, False def GetStatus(self): """Compute the status of the DRBD device @@ -1504,7 +1581,6 @@ class DRBDev(BaseDRBD): return result - @staticmethod def _ZeroDevice(device): """Zero a device. @@ -1521,7 +1597,6 @@ class DRBDev(BaseDRBD): if err.errno != errno.ENOSPC: raise - @classmethod def Create(cls, unique_id, children, size): """Create a new DRBD device. @@ -1536,14 +1611,13 @@ class DRBDev(BaseDRBD): meta.Assemble() if not meta.Attach(): raise errors.BlockDeviceError("Can't attach to meta device") - if not cls._IsValidMeta(meta.dev_path): + if not cls._CheckMetaSize(meta.dev_path): raise errors.BlockDeviceError("Invalid meta device") logger.Info("Started zeroing device %s" % meta.dev_path) cls._ZeroDevice(meta.dev_path) logger.Info("Done zeroing device %s" % meta.dev_path) return cls(unique_id, children) - def Remove(self): """Stub remove for DRBD devices. @@ -1551,44 +1625,712 @@ class DRBDev(BaseDRBD): return self.Shutdown() -DEV_MAP = { - constants.LD_LV: LogicalVolume, - constants.LD_MD_R1: MDRaid1, - constants.LD_DRBD7: DRBDev, - } - +class DRBD8(BaseDRBD): + """DRBD v8.x block device. -def FindDevice(dev_type, unique_id, children): - """Search for an existing, assembled device. + This implements the local host part of the DRBD device, i.e. it + doesn't do anything to the supposed peer. If you need a fully + connected DRBD pair, you need to use this class on both hosts. - This will succeed only if the device exists and is assembled, but it - does not do any actions in order to activate the device. + The unique_id for the drbd device is the (local_ip, local_port, + remote_ip, remote_port) tuple, and it must have two children: the + data device and the meta_device. The meta device is checked for + valid size and is zeroed on create. """ - if dev_type not in DEV_MAP: - raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type) - device = DEV_MAP[dev_type](unique_id, children) - if not device.Attach(): - return None - return device + _MAX_MINORS = 255 + _PARSE_SHOW = None + def __init__(self, unique_id, children): + if children and children.count(None) > 0: + children = [] + super(DRBD8, self).__init__(unique_id, children) + self.major = self._DRBD_MAJOR + version = self._GetVersion() + if version['k_major'] != 8 : + raise errors.BlockDeviceError("Mismatch in DRBD kernel version and" + " requested ganeti usage: kernel is" + " %s.%s, ganeti wants 8.x" % + (version['k_major'], version['k_minor'])) -def AttachOrAssemble(dev_type, unique_id, children): - """Try to attach or assemble an existing device. + if len(children) not in (0, 2): + raise ValueError("Invalid configuration data %s" % str(children)) + if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4: + raise ValueError("Invalid configuration data %s" % str(unique_id)) + self._lhost, self._lport, self._rhost, self._rport = unique_id + self.Attach() - This will attach to an existing assembled device or will assemble - the device, as needed, to bring it fully up. + @classmethod + def _InitMeta(cls, minor, dev_path): + """Initialize a meta device. - """ - if dev_type not in DEV_MAP: - raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type) - device = DEV_MAP[dev_type](unique_id, children) - if not device.Attach(): - device.Assemble() - if not device.Attach(): - raise errors.BlockDeviceError("Can't find a valid block device for" - " %s/%s/%s" % - (dev_type, unique_id, children)) + This will not work if the given minor is in use. + + """ + result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor), + "v08", dev_path, "0", "create-md"]) + if result.failed: + raise errors.BlockDeviceError("Can't initialize meta device: %s" % + result.output) + + @classmethod + def _FindUnusedMinor(cls): + """Find an unused DRBD device. + + This is specific to 8.x as the minors are allocated dynamically, + so non-existing numbers up to a max minor count are actually free. + + """ + data = cls._GetProcData() + + unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$") + used_line = re.compile("^ *([0-9]+): cs:") + highest = None + for line in data: + match = unused_line.match(line) + if match: + return int(match.group(1)) + match = used_line.match(line) + if match: + minor = int(match.group(1)) + highest = max(highest, minor) + if highest is None: # there are no minors in use at all + return 0 + if highest >= cls._MAX_MINORS: + logger.Error("Error: no free drbd minors!") + raise errors.BlockDeviceError("Can't find a free DRBD minor") + return highest + 1 + + @classmethod + def _IsValidMeta(cls, meta_device): + """Check if the given meta device looks like a valid one. + + """ + minor = cls._FindUnusedMinor() + minor_path = cls._DevPath(minor) + result = utils.RunCmd(["drbdmeta", minor_path, + "v08", meta_device, "0", + "dstate"]) + if result.failed: + logger.Error("Invalid meta device %s: %s" % (meta_device, result.output)) + return False + return True + + @classmethod + def _GetShowParser(cls): + """Return a parser for `drbd show` output. + + This will either create or return an already-create parser for the + output of the command `drbd show`. + + """ + if cls._PARSE_SHOW is not None: + return cls._PARSE_SHOW + + # pyparsing setup + lbrace = pyp.Literal("{").suppress() + rbrace = pyp.Literal("}").suppress() + semi = pyp.Literal(";").suppress() + # this also converts the value to an int + number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0])) + + comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine) + defa = pyp.Literal("_is_default").suppress() + dbl_quote = pyp.Literal('"').suppress() + + keyword = pyp.Word(pyp.alphanums + '-') + + # value types + value = pyp.Word(pyp.alphanums + '_-/.:') + quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote + addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() + + number) + # meta device, extended syntax + meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() + + number + pyp.Word(']').suppress()) + + # a statement + stmt = (~rbrace + keyword + ~lbrace + + (addr_port ^ value ^ quoted ^ meta_value) + + pyp.Optional(defa) + semi + + pyp.Optional(pyp.restOfLine).suppress()) + + # an entire section + section_name = pyp.Word(pyp.alphas + '_') + section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace + + bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt)) + bnf.ignore(comment) + + cls._PARSE_SHOW = bnf + + return bnf + + @classmethod + def _GetShowData(cls, minor): + """Return the `drbdsetup show` data for a minor. + + """ + result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"]) + if result.failed: + logger.Error("Can't display the drbd config: %s" % result.fail_reason) + return None + return result.stdout + + @classmethod + def _GetDevInfo(cls, out): + """Parse details about a given DRBD minor. + + This return, if available, the local backing device (as a path) + and the local and remote (ip, port) information from a string + containing the output of the `drbdsetup show` command as returned + by _GetShowData. + + """ + data = {} + if not out: + return data + + bnf = cls._GetShowParser() + # run pyparse + + try: + results = bnf.parseString(out) + except pyp.ParseException, err: + raise errors.BlockDeviceError("Can't parse drbdsetup show output: %s" % + str(err)) + + # and massage the results into our desired format + for section in results: + sname = section[0] + if sname == "_this_host": + for lst in section[1:]: + if lst[0] == "disk": + data["local_dev"] = lst[1] + elif lst[0] == "meta-disk": + data["meta_dev"] = lst[1] + data["meta_index"] = lst[2] + elif lst[0] == "address": + data["local_addr"] = tuple(lst[1:]) + elif sname == "_remote_host": + for lst in section[1:]: + if lst[0] == "address": + data["remote_addr"] = tuple(lst[1:]) + return data + + def _MatchesLocal(self, info): + """Test if our local config matches with an existing device. + + The parameter should be as returned from `_GetDevInfo()`. This + method tests if our local backing device is the same as the one in + the info parameter, in effect testing if we look like the given + device. + + """ + if self._children: + backend, meta = self._children + else: + backend = meta = None + + if backend is not None: + retval = ("local_dev" in info and info["local_dev"] == backend.dev_path) + else: + retval = ("local_dev" not in info) + + if meta is not None: + retval = retval and ("meta_dev" in info and + info["meta_dev"] == meta.dev_path) + retval = retval and ("meta_index" in info and + info["meta_index"] == 0) + else: + retval = retval and ("meta_dev" not in info and + "meta_index" not in info) + return retval + + def _MatchesNet(self, info): + """Test if our network config matches with an existing device. + + The parameter should be as returned from `_GetDevInfo()`. This + method tests if our network configuration is the same as the one + in the info parameter, in effect testing if we look like the given + device. + + """ + if (((self._lhost is None and not ("local_addr" in info)) and + (self._rhost is None and not ("remote_addr" in info)))): + return True + + if self._lhost is None: + return False + + if not ("local_addr" in info and + "remote_addr" in info): + return False + + retval = (info["local_addr"] == (self._lhost, self._lport)) + retval = (retval and + info["remote_addr"] == (self._rhost, self._rport)) + return retval + + @classmethod + def _AssembleLocal(cls, minor, backend, meta): + """Configure the local part of a DRBD device. + + This is the first thing that must be done on an unconfigured DRBD + device. And it must be done only once. + + """ + if not cls._IsValidMeta(meta): + return False + result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk", + backend, meta, "0", "-e", "detach", + "--create-device"]) + if result.failed: + logger.Error("Can't attach local disk: %s" % result.output) + return not result.failed + + @classmethod + def _AssembleNet(cls, minor, net_info, protocol, + dual_pri=False, hmac=None, secret=None): + """Configure the network part of the device. + + """ + lhost, lport, rhost, rport = net_info + if None in net_info: + # we don't want network connection and actually want to make + # sure its shutdown + return cls._ShutdownNet(minor) + + args = ["drbdsetup", cls._DevPath(minor), "net", + "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol, + "-A", "discard-zero-changes", + "-B", "consensus", + ] + if dual_pri: + args.append("-m") + if hmac and secret: + args.extend(["-a", hmac, "-x", secret]) + result = utils.RunCmd(args) + if result.failed: + logger.Error("Can't setup network for dbrd device: %s" % + result.fail_reason) + return False + + timeout = time.time() + 10 + ok = False + while time.time() < timeout: + info = cls._GetDevInfo(cls._GetShowData(minor)) + if not "local_addr" in info or not "remote_addr" in info: + time.sleep(1) + continue + if (info["local_addr"] != (lhost, lport) or + info["remote_addr"] != (rhost, rport)): + time.sleep(1) + continue + ok = True + break + if not ok: + logger.Error("Timeout while configuring network") + return False + return True + + def AddChildren(self, devices): + """Add a disk to the DRBD device. + + """ + if self.minor is None: + raise errors.BlockDeviceError("Can't attach to dbrd8 during AddChildren") + if len(devices) != 2: + raise errors.BlockDeviceError("Need two devices for AddChildren") + info = self._GetDevInfo(self._GetShowData(self.minor)) + if "local_dev" in info: + raise errors.BlockDeviceError("DRBD8 already attached to a local disk") + backend, meta = devices + if backend.dev_path is None or meta.dev_path is None: + raise errors.BlockDeviceError("Children not ready during AddChildren") + backend.Open() + meta.Open() + if not self._CheckMetaSize(meta.dev_path): + raise errors.BlockDeviceError("Invalid meta device size") + self._InitMeta(self._FindUnusedMinor(), meta.dev_path) + if not self._IsValidMeta(meta.dev_path): + raise errors.BlockDeviceError("Cannot initalize meta device") + + if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path): + raise errors.BlockDeviceError("Can't attach to local storage") + self._children = devices + + def RemoveChildren(self, devices): + """Detach the drbd device from local storage. + + """ + if self.minor is None: + raise errors.BlockDeviceError("Can't attach to drbd8 during" + " RemoveChildren") + # early return if we don't actually have backing storage + info = self._GetDevInfo(self._GetShowData(self.minor)) + if "local_dev" not in info: + return + if len(self._children) != 2: + raise errors.BlockDeviceError("We don't have two children: %s" % + self._children) + if self._children.count(None) == 2: # we don't actually have children :) + logger.Error("Requested detach while detached") + return + if len(devices) != 2: + raise errors.BlockDeviceError("We need two children in RemoveChildren") + for child, dev in zip(self._children, devices): + if dev != child.dev_path: + raise errors.BlockDeviceError("Mismatch in local storage" + " (%s != %s) in RemoveChildren" % + (dev, child.dev_path)) + + if not self._ShutdownLocal(self.minor): + raise errors.BlockDeviceError("Can't detach from local storage") + self._children = [] + + def SetSyncSpeed(self, kbytes): + """Set the speed of the DRBD syncer. + + """ + children_result = super(DRBD8, self).SetSyncSpeed(kbytes) + if self.minor is None: + logger.Info("Instance not attached to a device") + return False + result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" % + kbytes]) + if result.failed: + logger.Error("Can't change syncer rate: %s " % result.fail_reason) + return not result.failed and children_result + + def GetSyncStatus(self): + """Returns the sync status of the device. + + Returns: + (sync_percent, estimated_time, is_degraded) + + If sync_percent is None, it means all is ok + If estimated_time is None, it means we can't esimate + the time needed, otherwise it's the time left in seconds. + + + We set the is_degraded parameter to True on two conditions: + network not connected or local disk missing. + + We compute the ldisk parameter based on wheter we have a local + disk or not. + + """ + if self.minor is None and not self.Attach(): + raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus") + proc_info = self._MassageProcData(self._GetProcData()) + if self.minor not in proc_info: + raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" % + self.minor) + line = proc_info[self.minor] + match = re.match("^.*sync'ed: *([0-9.]+)%.*" + " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line) + if match: + sync_percent = float(match.group(1)) + hours = int(match.group(2)) + minutes = int(match.group(3)) + seconds = int(match.group(4)) + est_time = hours * 3600 + minutes * 60 + seconds + else: + sync_percent = None + est_time = None + match = re.match("^ *\d+: cs:(\w+).*ds:(\w+)/(\w+).*$", line) + if not match: + raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" % + self.minor) + client_state = match.group(1) + local_disk_state = match.group(2) + ldisk = local_disk_state != "UpToDate" + is_degraded = client_state != "Connected" + return sync_percent, est_time, is_degraded or ldisk, ldisk + + def GetStatus(self): + """Compute the status of the DRBD device + + Note that DRBD devices don't have the STATUS_EXISTING state. + + """ + if self.minor is None and not self.Attach(): + return self.STATUS_UNKNOWN + + data = self._GetProcData() + match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" % + self.minor) + for line in data: + mresult = match.match(line) + if mresult: + break + else: + logger.Error("Can't find myself!") + return self.STATUS_UNKNOWN + + state = mresult.group(2) + if state == "Primary": + result = self.STATUS_ONLINE + else: + result = self.STATUS_STANDBY + + return result + + def Open(self, force=False): + """Make the local state primary. + + If the 'force' parameter is given, the '--do-what-I-say' parameter + is given. Since this is a pottentialy dangerous operation, the + force flag should be only given after creation, when it actually + has to be given. + + """ + if self.minor is None and not self.Attach(): + logger.Error("DRBD cannot attach to a device during open") + return False + cmd = ["drbdsetup", self.dev_path, "primary"] + if force: + cmd.append("-o") + result = utils.RunCmd(cmd) + if result.failed: + msg = ("Can't make drbd device primary: %s" % result.output) + logger.Error(msg) + raise errors.BlockDeviceError(msg) + + def Close(self): + """Make the local state secondary. + + This will, of course, fail if the device is in use. + + """ + if self.minor is None and not self.Attach(): + logger.Info("Instance not attached to a device") + raise errors.BlockDeviceError("Can't find device") + result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"]) + if result.failed: + msg = ("Can't switch drbd device to" + " secondary: %s" % result.output) + logger.Error(msg) + raise errors.BlockDeviceError(msg) + + def Attach(self): + """Find a DRBD device which matches our config and attach to it. + + In case of partially attached (local device matches but no network + setup), we perform the network attach. If successful, we re-test + the attach if can return success. + + """ + for minor in self._GetUsedDevs(): + info = self._GetDevInfo(self._GetShowData(minor)) + match_l = self._MatchesLocal(info) + match_r = self._MatchesNet(info) + if match_l and match_r: + break + if match_l and not match_r and "local_addr" not in info: + res_r = self._AssembleNet(minor, + (self._lhost, self._lport, + self._rhost, self._rport), + "C") + if res_r: + if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))): + break + # the weakest case: we find something that is only net attached + # even though we were passed some children at init time + if match_r and "local_dev" not in info: + break + if match_l and not match_r and "local_addr" in info: + # strange case - the device network part points to somewhere + # else, even though its local storage is ours; as we own the + # drbd space, we try to disconnect from the remote peer and + # reconnect to our correct one + if not self._ShutdownNet(minor): + raise errors.BlockDeviceError("Device has correct local storage," + " wrong remote peer and is unable to" + " disconnect in order to attach to" + " the correct peer") + # note: _AssembleNet also handles the case when we don't want + # local storage (i.e. one or more of the _[lr](host|port) is + # None) + if (self._AssembleNet(minor, (self._lhost, self._lport, + self._rhost, self._rport), "C") and + self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))): + break + + else: + minor = None + + self._SetFromMinor(minor) + return minor is not None + + def Assemble(self): + """Assemble the drbd. + + Method: + - if we have a local backing device, we bind to it by: + - checking the list of used drbd devices + - check if the local minor use of any of them is our own device + - if yes, abort? + - if not, bind + - if we have a local/remote net info: + - redo the local backing device step for the remote device + - check if any drbd device is using the local port, + if yes abort + - check if any remote drbd device is using the remote + port, if yes abort (for now) + - bind our net port + - bind the remote net port + + """ + self.Attach() + if self.minor is not None: + logger.Info("Already assembled") + return True + + result = super(DRBD8, self).Assemble() + if not result: + return result + + minor = self._FindUnusedMinor() + need_localdev_teardown = False + if self._children and self._children[0] and self._children[1]: + result = self._AssembleLocal(minor, self._children[0].dev_path, + self._children[1].dev_path) + if not result: + return False + need_localdev_teardown = True + if self._lhost and self._lport and self._rhost and self._rport: + result = self._AssembleNet(minor, + (self._lhost, self._lport, + self._rhost, self._rport), + "C") + if not result: + if need_localdev_teardown: + # we will ignore failures from this + logger.Error("net setup failed, tearing down local device") + self._ShutdownAll(minor) + return False + self._SetFromMinor(minor) + return True + + @classmethod + def _ShutdownLocal(cls, minor): + """Detach from the local device. + + I/Os will continue to be served from the remote device. If we + don't have a remote device, this operation will fail. + + """ + result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"]) + if result.failed: + logger.Error("Can't detach local device: %s" % result.output) + return not result.failed + + @classmethod + def _ShutdownNet(cls, minor): + """Disconnect from the remote peer. + + This fails if we don't have a local device. + + """ + result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"]) + if result.failed: + logger.Error("Can't shutdown network: %s" % result.output) + return not result.failed + + @classmethod + def _ShutdownAll(cls, minor): + """Deactivate the device. + + This will, of course, fail if the device is in use. + + """ + result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"]) + if result.failed: + logger.Error("Can't shutdown drbd device: %s" % result.output) + return not result.failed + + def Shutdown(self): + """Shutdown the DRBD device. + + """ + if self.minor is None and not self.Attach(): + logger.Info("DRBD device not attached to a device during Shutdown") + return True + if not self._ShutdownAll(self.minor): + return False + self.minor = None + self.dev_path = None + return True + + def Remove(self): + """Stub remove for DRBD devices. + + """ + return self.Shutdown() + + @classmethod + def Create(cls, unique_id, children, size): + """Create a new DRBD8 device. + + Since DRBD devices are not created per se, just assembled, this + function only initializes the metadata. + + """ + if len(children) != 2: + raise errors.ProgrammerError("Invalid setup for the drbd device") + meta = children[1] + meta.Assemble() + if not meta.Attach(): + raise errors.BlockDeviceError("Can't attach to meta device") + if not cls._CheckMetaSize(meta.dev_path): + raise errors.BlockDeviceError("Invalid meta device size") + cls._InitMeta(cls._FindUnusedMinor(), meta.dev_path) + if not cls._IsValidMeta(meta.dev_path): + raise errors.BlockDeviceError("Cannot initalize meta device") + return cls(unique_id, children) + + +DEV_MAP = { + constants.LD_LV: LogicalVolume, + constants.LD_MD_R1: MDRaid1, + constants.LD_DRBD7: DRBDev, + constants.LD_DRBD8: DRBD8, + } + + +def FindDevice(dev_type, unique_id, children): + """Search for an existing, assembled device. + + This will succeed only if the device exists and is assembled, but it + does not do any actions in order to activate the device. + + """ + if dev_type not in DEV_MAP: + raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type) + device = DEV_MAP[dev_type](unique_id, children) + if not device.Attach(): + return None + return device + + +def AttachOrAssemble(dev_type, unique_id, children): + """Try to attach or assemble an existing device. + + This will attach to an existing assembled device or will assemble + the device, as needed, to bring it fully up. + + """ + if dev_type not in DEV_MAP: + raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type) + device = DEV_MAP[dev_type](unique_id, children) + if not device.Attach(): + device.Assemble() + if not device.Attach(): + raise errors.BlockDeviceError("Can't find a valid block device for" + " %s/%s/%s" % + (dev_type, unique_id, children)) return device