4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
28 from ganeti import utils
29 from ganeti import logger
30 from ganeti import errors
31 from ganeti import constants
34 class BlockDev(object):
35 """Block device abstract class.
37 A block device can be in the following states:
38 - not existing on the system, and by `Create()` it goes into:
39 - existing but not setup/not active, and by `Assemble()` goes into:
40 - active read-write and by `Open()` it goes into
41 - online (=used, or ready for use)
43 A device can also be online but read-only, however we are not using
44 the readonly state (MD and LV have it, if needed in the future)
45 and we are usually looking at this like at a stack, so it's easier
46 to conceptualise the transition from not-existing to online and back
49 The many different states of the device are due to the fact that we
50 need to cover many device types:
51 - logical volumes are created, lvchange -a y $lv, and used
52 - md arrays are created or assembled and used
53 - drbd devices are attached to a local disk/remote peer and made primary
55 The status of the device can be examined by `GetStatus()`, which
56 returns a numerical value, depending on the position in the
57 transition stack of the device.
59 A block device is identified by three items:
60 - the /dev path of the device (dynamic)
61 - a unique ID of the device (static)
62 - it's major/minor pair (dynamic)
64 Not all devices implement both the first two as distinct items. LVM
65 logical volumes have their unique ID (the pair volume group, logical
66 volume name) in a 1-to-1 relation to the dev path. For MD devices,
67 the /dev path is dynamic and the unique ID is the UUID generated at
68 array creation plus the slave list. For DRBD devices, the /dev path
69 is again dynamic and the unique id is the pair (host1, dev1),
72 You can get to a device in two ways:
73 - creating the (real) device, which returns you
74 an attached instance (lvcreate, mdadm --create)
75 - attaching of a python instance to an existing (real) device
77 The second point, the attachement to a device, is different
78 depending on whether the device is assembled or not. At init() time,
79 we search for a device with the same unique_id as us. If found,
80 good. It also means that the device is already assembled. If not,
81 after assembly we'll have our correct major/minor.
90 STATUS_UNKNOWN: "unknown",
91 STATUS_EXISTING: "existing",
92 STATUS_STANDBY: "ready for use",
93 STATUS_ONLINE: "online",
97 def __init__(self, unique_id, children):
98 self._children = children
100 self.unique_id = unique_id
106 """Assemble the device from its components.
108 If this is a plain block device (e.g. LVM) than assemble does
109 nothing, as the LVM has no children and we don't put logical
112 One guarantee is that after the device has been assembled, it
113 knows its major/minor numbers. This allows other devices (usually
114 parents) to probe correctly for their children.
118 for child in self._children:
119 if not isinstance(child, BlockDev):
120 raise TypeError("Invalid child passed of type '%s'" % type(child))
123 status = status and child.Assemble()
126 status = status and child.Open()
129 for child in self._children:
135 """Find a device which matches our config and attach to it.
138 raise NotImplementedError
142 """Notifies that the device will no longer be used for I/O.
145 raise NotImplementedError
149 def Create(cls, unique_id, children, size):
150 """Create the device.
152 If the device cannot be created, it will return None
153 instead. Error messages go to the logging system.
155 Note that for some devices, the unique_id is used, and for other,
156 the children. The idea is that these two, taken together, are
157 enough for both creation and assembly (later).
160 raise NotImplementedError
164 """Remove this device.
166 This makes sense only for some of the device types: LV and to a
167 lesser degree, md devices. Also note that if the device can't
168 attach, the removal can't be completed.
171 raise NotImplementedError
175 """Return the status of the device.
178 raise NotImplementedError
181 def Open(self, force=False):
182 """Make the device ready for use.
184 This makes the device ready for I/O. For now, just the DRBD
187 The force parameter signifies that if the device has any kind of
188 --force thing, it should be used, we know what we are doing.
191 raise NotImplementedError
195 """Shut down the device, freeing its children.
197 This undoes the `Assemble()` work, except for the child
198 assembling; as such, the children on the device are still
199 assembled after this call.
202 raise NotImplementedError
205 def SetSyncSpeed(self, speed):
206 """Adjust the sync speed of the mirror.
208 In case this is not a mirroring device, this is no-op.
213 for child in self._children:
214 result = result and child.SetSyncSpeed(speed)
218 def GetSyncStatus(self):
219 """Returns the sync status of the device.
221 If this device is a mirroring device, this function returns the
222 status of the mirror.
225 (sync_percent, estimated_time, is_degraded)
227 If sync_percent is None, it means all is ok
228 If estimated_time is None, it means we can't estimate
229 the time needed, otherwise it's the time left in seconds
230 If is_degraded is True, it means the device is missing
231 redundancy. This is usually a sign that something went wrong in
232 the device setup, if sync_percent is None.
235 return None, None, False
238 def CombinedSyncStatus(self):
239 """Calculate the mirror status recursively for our children.
241 The return value is the same as for `GetSyncStatus()` except the
242 minimum percent and maximum time are calculated across our
246 min_percent, max_time, is_degraded = self.GetSyncStatus()
248 for child in self._children:
249 c_percent, c_time, c_degraded = child.GetSyncStatus()
250 if min_percent is None:
251 min_percent = c_percent
252 elif c_percent is not None:
253 min_percent = min(min_percent, c_percent)
256 elif c_time is not None:
257 max_time = max(max_time, c_time)
258 is_degraded = is_degraded or c_degraded
259 return min_percent, max_time, is_degraded
262 def SetInfo(self, text):
263 """Update metadata with info text.
265 Only supported for some device types.
268 for child in self._children:
273 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
274 (self.__class__, self.unique_id, self._children,
275 self.major, self.minor, self.dev_path))
278 class LogicalVolume(BlockDev):
279 """Logical Volume block device.
282 def __init__(self, unique_id, children):
283 """Attaches to a LV device.
285 The unique_id is a tuple (vg_name, lv_name)
288 super(LogicalVolume, self).__init__(unique_id, children)
289 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
290 raise ValueError("Invalid configuration data %s" % str(unique_id))
291 self._vg_name, self._lv_name = unique_id
292 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
297 def Create(cls, unique_id, children, size):
298 """Create a new logical volume.
301 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
302 raise ValueError("Invalid configuration data %s" % str(unique_id))
303 vg_name, lv_name = unique_id
304 pvs_info = cls.GetPVInfo(vg_name)
306 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
311 pvlist = [ pv[1] for pv in pvs_info ]
312 free_size = sum([ pv[0] for pv in pvs_info ])
314 # The size constraint should have been checked from the master before
315 # calling the create function.
317 raise errors.BlockDeviceError("Not enough free space: required %s,"
318 " available %s" % (size, free_size))
319 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
322 raise errors.BlockDeviceError(result.fail_reason)
323 return LogicalVolume(unique_id, children)
326 def GetPVInfo(vg_name):
327 """Get the free space info for PVs in a volume group.
330 vg_name: the volume group name
333 list of (free_space, name) with free_space in mebibytes
336 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
337 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
339 result = utils.RunCmd(command)
341 logger.Error("Can't get the PV information: %s" % result.fail_reason)
344 for line in result.stdout.splitlines():
345 fields = line.strip().split(':')
347 logger.Error("Can't parse pvs output: line '%s'" % line)
349 # skip over pvs from another vg or ones which are not allocatable
350 if fields[1] != vg_name or fields[3][0] != 'a':
352 data.append((float(fields[2]), fields[0]))
357 """Remove this logical volume.
360 if not self.minor and not self.Attach():
361 # the LV does not exist
363 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
364 (self._vg_name, self._lv_name)])
366 logger.Error("Can't lvremove: %s" % result.fail_reason)
368 return not result.failed
372 """Attach to an existing LV.
374 This method will try to see if an existing and active LV exists
375 which matches the our name. If so, its major/minor will be
379 result = utils.RunCmd(["lvdisplay", self.dev_path])
381 logger.Error("Can't find LV %s: %s" %
382 (self.dev_path, result.fail_reason))
384 match = re.compile("^ *Block device *([0-9]+):([0-9]+).*$")
385 for line in result.stdout.splitlines():
386 match_result = match.match(line)
388 self.major = int(match_result.group(1))
389 self.minor = int(match_result.group(2))
395 """Assemble the device.
397 This is a no-op for the LV device type. Eventually, we could
398 lvchange -ay here if we see that the LV is not active.
405 """Shutdown the device.
407 This is a no-op for the LV device type, as we don't deactivate the
415 """Return the status of the device.
417 Logical volumes will can be in all four states, although we don't
418 deactivate (lvchange -an) them when shutdown, so STATUS_EXISTING
419 should not be seen for our devices.
422 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
424 logger.Error("Can't display lv: %s" % result.fail_reason)
425 return self.STATUS_UNKNOWN
426 out = result.stdout.strip()
427 # format: type/permissions/alloc/fixed_minor/state/open
429 return self.STATUS_UNKNOWN
430 #writable = (out[1] == "w")
431 active = (out[4] == "a")
432 online = (out[5] == "o")
434 retval = self.STATUS_ONLINE
436 retval = self.STATUS_STANDBY
438 retval = self.STATUS_EXISTING
443 def Open(self, force=False):
444 """Make the device ready for I/O.
446 This is a no-op for the LV device type.
453 """Notifies that the device will no longer be used for I/O.
455 This is a no-op for the LV device type.
461 def Snapshot(self, size):
462 """Create a snapshot copy of an lvm block device.
465 snap_name = self._lv_name + ".snap"
467 # remove existing snapshot if found
468 snap = LogicalVolume((self._vg_name, snap_name), None)
471 pvs_info = self.GetPVInfo(self._vg_name)
473 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
477 free_size, pv_name = pvs_info[0]
479 raise errors.BlockDeviceError("Not enough free space: required %s,"
480 " available %s" % (size, free_size))
482 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
483 "-n%s" % snap_name, self.dev_path])
485 raise errors.BlockDeviceError("command: %s error: %s" %
486 (result.cmd, result.fail_reason))
491 def SetInfo(self, text):
492 """Update metadata with info text.
495 BlockDev.SetInfo(self, text)
497 # Replace invalid characters
498 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
499 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
501 # Only up to 128 characters are allowed
504 result = utils.RunCmd(["lvchange", "--addtag", text,
507 raise errors.BlockDeviceError("Command: %s error: %s" %
508 (result.cmd, result.fail_reason))
511 class MDRaid1(BlockDev):
512 """raid1 device implemented via md.
515 def __init__(self, unique_id, children):
516 super(MDRaid1, self).__init__(unique_id, children)
522 """Find an array which matches our config and attach to it.
524 This tries to find a MD array which has the same UUID as our own.
527 minor = self._FindMDByUUID(self.unique_id)
528 if minor is not None:
529 self._SetFromMinor(minor)
534 return (minor is not None)
539 """Compute the list of in-use MD devices.
541 It doesn't matter if the used device have other raid level, just
542 that they are in use.
545 mdstat = open("/proc/mdstat", "r")
546 data = mdstat.readlines()
550 valid_line = re.compile("^md([0-9]+) : .*$")
552 match = valid_line.match(line)
554 md_no = int(match.group(1))
555 used_md[md_no] = line
561 def _GetDevInfo(minor):
562 """Get info about a MD device.
564 Currently only uuid is returned.
567 result = utils.RunCmd(["mdadm", "-D", "/dev/md%d" % minor])
569 logger.Error("Can't display md: %s" % result.fail_reason)
572 for line in result.stdout.splitlines():
574 kv = line.split(" : ", 1)
577 retval["uuid"] = kv[1].split()[0]
578 elif kv[0] == "State":
579 retval["state"] = kv[1].split(", ")
584 def _FindUnusedMinor():
585 """Compute an unused MD minor.
587 This code assumes that there are 256 minors only.
590 used_md = MDRaid1._GetUsedDevs()
597 logger.Error("Critical: Out of md minor numbers.")
598 raise errors.BlockDeviceError("Can't find a free MD minor")
603 def _FindMDByUUID(cls, uuid):
604 """Find the minor of an MD array with a given UUID.
607 md_list = cls._GetUsedDevs()
608 for minor in md_list:
609 info = cls._GetDevInfo(minor)
610 if info and info["uuid"] == uuid:
616 def _ZeroSuperblock(dev_path):
617 """Zero the possible locations for an MD superblock.
619 The zero-ing can't be done via ``mdadm --zero-superblock`` as that
620 fails in versions 2.x with the same error code as non-writable
623 The superblocks are located at (negative values are relative to
624 the end of the block device):
625 - -128k to end for version 0.90 superblock
626 - -8k to -12k for version 1.0 superblock (included in the above)
627 - 0k to 4k for version 1.1 superblock
628 - 4k to 8k for version 1.2 superblock
630 To cover all situations, the zero-ing will be:
634 As such, the minimum device size must be 128k, otherwise we'll get
637 Note that this function depends on the fact that one can open,
638 read and write block devices normally.
641 overwrite_size = 128 * 1024
642 empty_buf = '\0' * overwrite_size
643 fd = open(dev_path, "r+")
649 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
650 fd.seek(-overwrite_size, 2)
654 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
659 def Create(cls, unique_id, children, size):
660 """Create a new MD raid1 array.
663 if not isinstance(children, (tuple, list)):
664 raise ValueError("Invalid setup data for MDRaid1 dev: %s" %
667 if not isinstance(i, BlockDev):
668 raise ValueError("Invalid member in MDRaid1 dev: %s" % type(i))
671 cls._ZeroSuperblock(i.dev_path)
672 except EnvironmentError, err:
673 logger.Error("Can't zero superblock for %s: %s" %
674 (i.dev_path, str(err)))
676 minor = cls._FindUnusedMinor()
677 result = utils.RunCmd(["mdadm", "--create", "/dev/md%d" % minor,
678 "--auto=yes", "--force", "-l1",
679 "-n%d" % len(children)] +
680 [dev.dev_path for dev in children])
683 logger.Error("Can't create md: %s: %s" % (result.fail_reason,
686 info = cls._GetDevInfo(minor)
687 if not info or not "uuid" in info:
688 logger.Error("Wrong information returned from mdadm -D: %s" % str(info))
690 return MDRaid1(info["uuid"], children)
694 """Stub remove function for MD RAID 1 arrays.
696 We don't remove the superblock right now. Mark a to do.
699 #TODO: maybe zero superblock on child devices?
700 return self.Shutdown()
703 def AddChild(self, device):
704 """Add a new member to the md raid1.
707 if self.minor is None and not self.Attach():
708 raise errors.BlockDeviceError("Can't attach to device")
709 if device.dev_path is None:
710 raise errors.BlockDeviceError("New child is not initialised")
711 result = utils.RunCmd(["mdadm", "-a", self.dev_path, device.dev_path])
713 raise errors.BlockDeviceError("Failed to add new device to array: %s" %
715 new_len = len(self._children) + 1
716 result = utils.RunCmd(["mdadm", "--grow", self.dev_path, "-n", new_len])
718 raise errors.BlockDeviceError("Can't grow md array: %s" %
720 self._children.append(device)
723 def RemoveChild(self, dev_path):
724 """Remove member from the md raid1.
727 if self.minor is None and not self.Attach():
728 raise errors.BlockDeviceError("Can't attach to device")
729 if len(self._children) == 1:
730 raise errors.BlockDeviceError("Can't reduce member when only one"
732 for device in self._children:
733 if device.dev_path == dev_path:
736 raise errors.BlockDeviceError("Can't find child with this path")
737 new_len = len(self._children) - 1
738 result = utils.RunCmd(["mdadm", "-f", self.dev_path, dev_path])
740 raise errors.BlockDeviceError("Failed to mark device as failed: %s" %
743 # it seems here we need a short delay for MD to update its
746 result = utils.RunCmd(["mdadm", "-r", self.dev_path, dev_path])
748 raise errors.BlockDeviceError("Failed to remove device from array:"
749 " %s" % result.output)
750 result = utils.RunCmd(["mdadm", "--grow", "--force", self.dev_path,
753 raise errors.BlockDeviceError("Can't shrink md array: %s" %
755 self._children.remove(device)
759 """Return the status of the device.
763 if self.minor is None:
764 retval = self.STATUS_UNKNOWN
766 retval = self.STATUS_ONLINE
770 def _SetFromMinor(self, minor):
771 """Set our parameters based on the given minor.
773 This sets our minor variable and our dev_path.
777 self.dev_path = "/dev/md%d" % minor
781 """Assemble the MD device.
783 At this point we should have:
784 - list of children devices
788 result = super(MDRaid1, self).Assemble()
791 md_list = self._GetUsedDevs()
792 for minor in md_list:
793 info = self._GetDevInfo(minor)
794 if info and info["uuid"] == self.unique_id:
795 self._SetFromMinor(minor)
796 logger.Info("MD array %s already started" % str(self))
798 free_minor = self._FindUnusedMinor()
799 result = utils.RunCmd(["mdadm", "-A", "--auto=yes", "--uuid",
800 self.unique_id, "/dev/md%d" % free_minor] +
801 [bdev.dev_path for bdev in self._children])
803 logger.Error("Can't assemble MD array: %s: %s" %
804 (result.fail_reason, result.output))
807 self.minor = free_minor
808 return not result.failed
812 """Tear down the MD array.
814 This does a 'mdadm --stop' so after this command, the array is no
818 if self.minor is None and not self.Attach():
819 logger.Info("MD object not attached to a device")
822 result = utils.RunCmd(["mdadm", "--stop", "/dev/md%d" % self.minor])
824 logger.Error("Can't stop MD array: %s" % result.fail_reason)
831 def SetSyncSpeed(self, kbytes):
832 """Set the maximum sync speed for the MD array.
835 result = super(MDRaid1, self).SetSyncSpeed(kbytes)
836 if self.minor is None:
837 logger.Error("MD array not attached to a device")
839 f = open("/sys/block/md%d/md/sync_speed_max" % self.minor, "w")
841 f.write("%d" % kbytes)
844 f = open("/sys/block/md%d/md/sync_speed_min" % self.minor, "w")
846 f.write("%d" % (kbytes/2))
852 def GetSyncStatus(self):
853 """Returns the sync status of the device.
856 (sync_percent, estimated_time)
858 If sync_percent is None, it means all is ok
859 If estimated_time is None, it means we can't esimate
860 the time needed, otherwise it's the time left in seconds
863 if self.minor is None and not self.Attach():
864 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
865 dev_info = self._GetDevInfo(self.minor)
866 is_clean = ("state" in dev_info and
867 len(dev_info["state"]) == 1 and
868 dev_info["state"][0] in ("clean", "active"))
869 sys_path = "/sys/block/md%s/md/" % self.minor
870 f = file(sys_path + "sync_action")
871 sync_status = f.readline().strip()
873 if sync_status == "idle":
874 return None, None, not is_clean
875 f = file(sys_path + "sync_completed")
876 sync_completed = f.readline().strip().split(" / ")
878 if len(sync_completed) != 2:
879 return 0, None, not is_clean
880 sync_done, sync_total = [float(i) for i in sync_completed]
881 sync_percent = 100.0*sync_done/sync_total
882 f = file(sys_path + "sync_speed")
883 sync_speed_k = int(f.readline().strip())
884 if sync_speed_k == 0:
887 time_est = (sync_total - sync_done) / 2 / sync_speed_k
888 return sync_percent, time_est, not is_clean
891 def Open(self, force=False):
892 """Make the device ready for I/O.
894 This is a no-op for the MDRaid1 device type, although we could use
895 the 2.6.18's new array_state thing.
902 """Notifies that the device will no longer be used for I/O.
904 This is a no-op for the MDRaid1 device type, but see comment for
911 class BaseDRBD(BlockDev):
914 This class contains a few bits of common functionality between the
915 0.7 and 8.x versions of DRBD.
918 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
919 r" \(api:(\d+)/proto:(\d+)\)")
921 _ST_UNCONFIGURED = "Unconfigured"
922 _ST_WFCONNECTION = "WFConnection"
923 _ST_CONNECTED = "Connected"
927 """Return data from /proc/drbd.
930 stat = open("/proc/drbd", "r")
932 data = stat.read().splitlines()
936 raise errors.BlockDeviceError("Can't read any data from /proc/drbd")
940 def _GetVersion(cls):
941 """Return the DRBD version.
943 This will return a list [k_major, k_minor, k_point, api, proto].
946 proc_data = cls._GetProcData()
947 first_line = proc_data[0].strip()
948 version = cls._VERSION_RE.match(first_line)
950 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
952 return [int(val) for val in version.groups()]
956 """Return the path to a drbd device for a given minor.
959 return "/dev/drbd%d" % minor
962 def _GetUsedDevs(cls):
963 """Compute the list of used DRBD devices.
966 data = cls._GetProcData()
969 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
971 match = valid_line.match(line)
974 minor = int(match.group(1))
975 state = match.group(2)
976 if state == cls._ST_UNCONFIGURED:
978 used_devs[minor] = state, line
983 class DRBDev(BaseDRBD):
984 """DRBD block device.
986 This implements the local host part of the DRBD device, i.e. it
987 doesn't do anything to the supposed peer. If you need a fully
988 connected DRBD pair, you need to use this class on both hosts.
990 The unique_id for the drbd device is the (local_ip, local_port,
991 remote_ip, remote_port) tuple, and it must have two children: the
992 data device and the meta_device. The meta device is checked for
993 valid size and is zeroed on create.
996 def __init__(self, unique_id, children):
997 super(DRBDev, self).__init__(unique_id, children)
998 self.major = self._DRBD_MAJOR
999 [kmaj, kmin, kfix, api, proto] = self._GetVersion()
1000 if kmaj != 0 and kmin != 7:
1001 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
1002 " requested ganeti usage: kernel is"
1003 " %s.%s, ganeti wants 0.7" % (kmaj, kmin))
1005 if len(children) != 2:
1006 raise ValueError("Invalid configuration data %s" % str(children))
1007 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
1008 raise ValueError("Invalid configuration data %s" % str(unique_id))
1009 self._lhost, self._lport, self._rhost, self._rport = unique_id
1013 def _FindUnusedMinor(cls):
1014 """Find an unused DRBD device.
1017 data = cls._GetProcData()
1019 valid_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
1021 match = valid_line.match(line)
1023 return int(match.group(1))
1024 logger.Error("Error: no free drbd minors!")
1025 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1028 def _GetDevInfo(cls, minor):
1029 """Get details about a given DRBD minor.
1031 This return, if available, the local backing device in (major,
1032 minor) formant and the local and remote (ip, port) information.
1036 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1038 logger.Error("Can't display the drbd config: %s" % result.fail_reason)
1041 if out == "Not configured\n":
1043 for line in out.splitlines():
1044 if "local_dev" not in data:
1045 match = re.match("^Lower device: ([0-9]+):([0-9]+) .*$", line)
1047 data["local_dev"] = (int(match.group(1)), int(match.group(2)))
1049 if "meta_dev" not in data:
1050 match = re.match("^Meta device: (([0-9]+):([0-9]+)|internal).*$", line)
1052 if match.group(2) is not None and match.group(3) is not None:
1053 # matched on the major/minor
1054 data["meta_dev"] = (int(match.group(2)), int(match.group(3)))
1056 # matched on the "internal" string
1057 data["meta_dev"] = match.group(1)
1058 # in this case, no meta_index is in the output
1059 data["meta_index"] = -1
1061 if "meta_index" not in data:
1062 match = re.match("^Meta index: ([0-9]+).*$", line)
1064 data["meta_index"] = int(match.group(1))
1066 if "local_addr" not in data:
1067 match = re.match("^Local address: ([0-9.]+):([0-9]+)$", line)
1069 data["local_addr"] = (match.group(1), int(match.group(2)))
1071 if "remote_addr" not in data:
1072 match = re.match("^Remote address: ([0-9.]+):([0-9]+)$", line)
1074 data["remote_addr"] = (match.group(1), int(match.group(2)))
1079 def _MatchesLocal(self, info):
1080 """Test if our local config matches with an existing device.
1082 The parameter should be as returned from `_GetDevInfo()`. This
1083 method tests if our local backing device is the same as the one in
1084 the info parameter, in effect testing if we look like the given
1088 if not ("local_dev" in info and "meta_dev" in info and
1089 "meta_index" in info):
1092 backend = self._children[0]
1093 if backend is not None:
1094 retval = (info["local_dev"] == (backend.major, backend.minor))
1096 retval = (info["local_dev"] == (0, 0))
1097 meta = self._children[1]
1098 if meta is not None:
1099 retval = retval and (info["meta_dev"] == (meta.major, meta.minor))
1100 retval = retval and (info["meta_index"] == 0)
1102 retval = retval and (info["meta_dev"] == "internal" and
1103 info["meta_index"] == -1)
1107 def _MatchesNet(self, info):
1108 """Test if our network config matches with an existing device.
1110 The parameter should be as returned from `_GetDevInfo()`. This
1111 method tests if our network configuration is the same as the one
1112 in the info parameter, in effect testing if we look like the given
1116 if (((self._lhost is None and not ("local_addr" in info)) and
1117 (self._rhost is None and not ("remote_addr" in info)))):
1120 if self._lhost is None:
1123 if not ("local_addr" in info and
1124 "remote_addr" in info):
1127 retval = (info["local_addr"] == (self._lhost, self._lport))
1128 retval = (retval and
1129 info["remote_addr"] == (self._rhost, self._rport))
1134 def _IsValidMeta(meta_device):
1135 """Check if the given meta device looks like a valid one.
1137 This currently only check the size, which must be around
1141 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1143 logger.Error("Failed to get device size: %s" % result.fail_reason)
1146 sectors = int(result.stdout)
1148 logger.Error("Invalid output from blockdev: '%s'" % result.stdout)
1150 bytes = sectors * 512
1151 if bytes < 128*1024*1024: # less than 128MiB
1152 logger.Error("Meta device too small (%.2fMib)" % (bytes/1024/1024))
1154 if bytes > (128+32)*1024*1024: # account for an extra (big) PE on LVM
1155 logger.Error("Meta device too big (%.2fMiB)" % (bytes/1024/1024))
1161 def _AssembleLocal(cls, minor, backend, meta):
1162 """Configure the local part of a DRBD device.
1164 This is the first thing that must be done on an unconfigured DRBD
1165 device. And it must be done only once.
1168 if not cls._IsValidMeta(meta):
1170 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1171 backend, meta, "0", "-e", "detach"])
1173 logger.Error("Can't attach local disk: %s" % result.output)
1174 return not result.failed
1178 def _ShutdownLocal(cls, minor):
1179 """Detach from the local device.
1181 I/Os will continue to be served from the remote device. If we
1182 don't have a remote device, this operation will fail.
1185 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1187 logger.Error("Can't detach local device: %s" % result.output)
1188 return not result.failed
1192 def _ShutdownAll(minor):
1193 """Deactivate the device.
1195 This will, of course, fail if the device is in use.
1198 result = utils.RunCmd(["drbdsetup", DRBDev._DevPath(minor), "down"])
1200 logger.Error("Can't shutdown drbd device: %s" % result.output)
1201 return not result.failed
1205 def _AssembleNet(cls, minor, net_info, protocol):
1206 """Configure the network part of the device.
1208 This operation can be, in theory, done multiple times, but there
1209 have been cases (in lab testing) in which the network part of the
1210 device had become stuck and couldn't be shut down because activity
1211 from the new peer (also stuck) triggered a timer re-init and
1212 needed remote peer interface shutdown in order to clear. So please
1213 don't change online the net config.
1216 lhost, lport, rhost, rport = net_info
1217 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "net",
1218 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport),
1221 logger.Error("Can't setup network for dbrd device: %s" %
1225 timeout = time.time() + 10
1227 while time.time() < timeout:
1228 info = cls._GetDevInfo(minor)
1229 if not "local_addr" in info or not "remote_addr" in info:
1232 if (info["local_addr"] != (lhost, lport) or
1233 info["remote_addr"] != (rhost, rport)):
1239 logger.Error("Timeout while configuring network")
1245 def _ShutdownNet(cls, minor):
1246 """Disconnect from the remote peer.
1248 This fails if we don't have a local device.
1251 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1252 logger.Error("Can't shutdown network: %s" % result.output)
1253 return not result.failed
1256 def _SetFromMinor(self, minor):
1257 """Set our parameters based on the given minor.
1259 This sets our minor variable and our dev_path.
1263 self.minor = self.dev_path = None
1266 self.dev_path = self._DevPath(minor)
1270 """Assemble the drbd.
1273 - if we have a local backing device, we bind to it by:
1274 - checking the list of used drbd devices
1275 - check if the local minor use of any of them is our own device
1278 - if we have a local/remote net info:
1279 - redo the local backing device step for the remote device
1280 - check if any drbd device is using the local port,
1282 - check if any remote drbd device is using the remote
1283 port, if yes abort (for now)
1285 - bind the remote net port
1289 if self.minor is not None:
1290 logger.Info("Already assembled")
1293 result = super(DRBDev, self).Assemble()
1297 minor = self._FindUnusedMinor()
1298 need_localdev_teardown = False
1299 if self._children[0]:
1300 result = self._AssembleLocal(minor, self._children[0].dev_path,
1301 self._children[1].dev_path)
1304 need_localdev_teardown = True
1305 if self._lhost and self._lport and self._rhost and self._rport:
1306 result = self._AssembleNet(minor,
1307 (self._lhost, self._lport,
1308 self._rhost, self._rport),
1311 if need_localdev_teardown:
1312 # we will ignore failures from this
1313 logger.Error("net setup failed, tearing down local device")
1314 self._ShutdownAll(minor)
1316 self._SetFromMinor(minor)
1321 """Shutdown the DRBD device.
1324 if self.minor is None and not self.Attach():
1325 logger.Info("DRBD device not attached to a device during Shutdown")
1327 if not self._ShutdownAll(self.minor):
1330 self.dev_path = None
1335 """Find a DRBD device which matches our config and attach to it.
1337 In case of partially attached (local device matches but no network
1338 setup), we perform the network attach. If successful, we re-test
1339 the attach if can return success.
1342 for minor in self._GetUsedDevs():
1343 info = self._GetDevInfo(minor)
1344 match_l = self._MatchesLocal(info)
1345 match_r = self._MatchesNet(info)
1346 if match_l and match_r:
1348 if match_l and not match_r and "local_addr" not in info:
1349 res_r = self._AssembleNet(minor,
1350 (self._lhost, self._lport,
1351 self._rhost, self._rport),
1353 if res_r and self._MatchesNet(self._GetDevInfo(minor)):
1358 self._SetFromMinor(minor)
1359 return minor is not None
1362 def Open(self, force=False):
1363 """Make the local state primary.
1365 If the 'force' parameter is given, the '--do-what-I-say' parameter
1366 is given. Since this is a pottentialy dangerous operation, the
1367 force flag should be only given after creation, when it actually
1371 if self.minor is None and not self.Attach():
1372 logger.Error("DRBD cannot attach to a device during open")
1374 cmd = ["drbdsetup", self.dev_path, "primary"]
1376 cmd.append("--do-what-I-say")
1377 result = utils.RunCmd(cmd)
1379 logger.Error("Can't make drbd device primary: %s" % result.output)
1385 """Make the local state secondary.
1387 This will, of course, fail if the device is in use.
1390 if self.minor is None and not self.Attach():
1391 logger.Info("Instance not attached to a device")
1392 raise errors.BlockDeviceError("Can't find device")
1393 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1395 logger.Error("Can't switch drbd device to secondary: %s" % result.output)
1396 raise errors.BlockDeviceError("Can't switch drbd device to secondary")
1399 def SetSyncSpeed(self, kbytes):
1400 """Set the speed of the DRBD syncer.
1403 children_result = super(DRBDev, self).SetSyncSpeed(kbytes)
1404 if self.minor is None:
1405 logger.Info("Instance not attached to a device")
1407 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1410 logger.Error("Can't change syncer rate: %s " % result.fail_reason)
1411 return not result.failed and children_result
1414 def GetSyncStatus(self):
1415 """Returns the sync status of the device.
1418 (sync_percent, estimated_time)
1420 If sync_percent is None, it means all is ok
1421 If estimated_time is None, it means we can't esimate
1422 the time needed, otherwise it's the time left in seconds
1425 if self.minor is None and not self.Attach():
1426 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1427 proc_info = self._MassageProcData(self._GetProcData())
1428 if self.minor not in proc_info:
1429 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1431 line = proc_info[self.minor]
1432 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
1433 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
1435 sync_percent = float(match.group(1))
1436 hours = int(match.group(2))
1437 minutes = int(match.group(3))
1438 seconds = int(match.group(4))
1439 est_time = hours * 3600 + minutes * 60 + seconds
1443 match = re.match("^ *[0-9]+: cs:([^ ]+).*$", line)
1445 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
1447 client_state = match.group(1)
1448 is_degraded = client_state != "Connected"
1449 return sync_percent, est_time, is_degraded
1453 def _MassageProcData(data):
1454 """Transform the output of _GetProdData into a nicer form.
1457 a dictionary of minor: joined lines from /proc/drbd for that minor
1460 lmatch = re.compile("^ *([0-9]+):.*$")
1462 old_minor = old_line = None
1464 lresult = lmatch.match(line)
1465 if lresult is not None:
1466 if old_minor is not None:
1467 results[old_minor] = old_line
1468 old_minor = int(lresult.group(1))
1471 if old_minor is not None:
1472 old_line += " " + line.strip()
1474 if old_minor is not None:
1475 results[old_minor] = old_line
1479 def GetStatus(self):
1480 """Compute the status of the DRBD device
1482 Note that DRBD devices don't have the STATUS_EXISTING state.
1485 if self.minor is None and not self.Attach():
1486 return self.STATUS_UNKNOWN
1488 data = self._GetProcData()
1489 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
1492 mresult = match.match(line)
1496 logger.Error("Can't find myself!")
1497 return self.STATUS_UNKNOWN
1499 state = mresult.group(2)
1500 if state == "Primary":
1501 result = self.STATUS_ONLINE
1503 result = self.STATUS_STANDBY
1509 def _ZeroDevice(device):
1512 This writes until we get ENOSPC.
1515 f = open(device, "w")
1516 buf = "\0" * 1048576
1520 except IOError, err:
1521 if err.errno != errno.ENOSPC:
1526 def Create(cls, unique_id, children, size):
1527 """Create a new DRBD device.
1529 Since DRBD devices are not created per se, just assembled, this
1530 function just zeroes the meta device.
1533 if len(children) != 2:
1534 raise errors.ProgrammerError("Invalid setup for the drbd device")
1537 if not meta.Attach():
1538 raise errors.BlockDeviceError("Can't attach to meta device")
1539 if not cls._IsValidMeta(meta.dev_path):
1540 raise errors.BlockDeviceError("Invalid meta device")
1541 logger.Info("Started zeroing device %s" % meta.dev_path)
1542 cls._ZeroDevice(meta.dev_path)
1543 logger.Info("Done zeroing device %s" % meta.dev_path)
1544 return cls(unique_id, children)
1548 """Stub remove for DRBD devices.
1551 return self.Shutdown()
1555 constants.LD_LV: LogicalVolume,
1556 constants.LD_MD_R1: MDRaid1,
1557 constants.LD_DRBD7: DRBDev,
1561 def FindDevice(dev_type, unique_id, children):
1562 """Search for an existing, assembled device.
1564 This will succeed only if the device exists and is assembled, but it
1565 does not do any actions in order to activate the device.
1568 if dev_type not in DEV_MAP:
1569 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1570 device = DEV_MAP[dev_type](unique_id, children)
1571 if not device.Attach():
1576 def AttachOrAssemble(dev_type, unique_id, children):
1577 """Try to attach or assemble an existing device.
1579 This will attach to an existing assembled device or will assemble
1580 the device, as needed, to bring it fully up.
1583 if dev_type not in DEV_MAP:
1584 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1585 device = DEV_MAP[dev_type](unique_id, children)
1586 if not device.Attach():
1588 if not device.Attach():
1589 raise errors.BlockDeviceError("Can't find a valid block device for"
1591 (dev_type, unique_id, children))
1595 def Create(dev_type, unique_id, children, size):
1599 if dev_type not in DEV_MAP:
1600 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1601 device = DEV_MAP[dev_type].Create(unique_id, children, size)