4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
28 from ganeti import utils
29 from ganeti import logger
30 from ganeti import errors
33 class BlockDev(object):
34 """Block device abstract class.
36 A block device can be in the following states:
37 - not existing on the system, and by `Create()` it goes into:
38 - existing but not setup/not active, and by `Assemble()` goes into:
39 - active read-write and by `Open()` it goes into
40 - online (=used, or ready for use)
42 A device can also be online but read-only, however we are not using
43 the readonly state (MD and LV have it, if needed in the future)
44 and we are usually looking at this like at a stack, so it's easier
45 to conceptualise the transition from not-existing to online and back
48 The many different states of the device are due to the fact that we
49 need to cover many device types:
50 - logical volumes are created, lvchange -a y $lv, and used
51 - md arrays are created or assembled and used
52 - drbd devices are attached to a local disk/remote peer and made primary
54 The status of the device can be examined by `GetStatus()`, which
55 returns a numerical value, depending on the position in the
56 transition stack of the device.
58 A block device is identified by three items:
59 - the /dev path of the device (dynamic)
60 - a unique ID of the device (static)
61 - it's major/minor pair (dynamic)
63 Not all devices implement both the first two as distinct items. LVM
64 logical volumes have their unique ID (the pair volume group, logical
65 volume name) in a 1-to-1 relation to the dev path. For MD devices,
66 the /dev path is dynamic and the unique ID is the UUID generated at
67 array creation plus the slave list. For DRBD devices, the /dev path
68 is again dynamic and the unique id is the pair (host1, dev1),
71 You can get to a device in two ways:
72 - creating the (real) device, which returns you
73 an attached instance (lvcreate, mdadm --create)
74 - attaching of a python instance to an existing (real) device
76 The second point, the attachement to a device, is different
77 depending on whether the device is assembled or not. At init() time,
78 we search for a device with the same unique_id as us. If found,
79 good. It also means that the device is already assembled. If not,
80 after assembly we'll have our correct major/minor.
89 STATUS_UNKNOWN: "unknown",
90 STATUS_EXISTING: "existing",
91 STATUS_STANDBY: "ready for use",
92 STATUS_ONLINE: "online",
96 def __init__(self, unique_id, children):
97 self._children = children
99 self.unique_id = unique_id
105 """Assemble the device from its components.
107 If this is a plain block device (e.g. LVM) than assemble does
108 nothing, as the LVM has no children and we don't put logical
111 One guarantee is that after the device has been assembled, it
112 knows its major/minor numbers. This allows other devices (usually
113 parents) to probe correctly for their children.
117 for child in self._children:
118 if not isinstance(child, BlockDev):
119 raise TypeError("Invalid child passed of type '%s'" % type(child))
122 status = status and child.Assemble()
125 status = status and child.Open()
128 for child in self._children:
134 """Find a device which matches our config and attach to it.
137 raise NotImplementedError
141 """Notifies that the device will no longer be used for I/O.
144 raise NotImplementedError
148 def Create(cls, unique_id, children, size):
149 """Create the device.
151 If the device cannot be created, it will return None
152 instead. Error messages go to the logging system.
154 Note that for some devices, the unique_id is used, and for other,
155 the children. The idea is that these two, taken together, are
156 enough for both creation and assembly (later).
159 raise NotImplementedError
163 """Remove this device.
165 This makes sense only for some of the device types: LV and to a
166 lesser degree, md devices. Also note that if the device can't
167 attach, the removal can't be completed.
170 raise NotImplementedError
174 """Return the status of the device.
177 raise NotImplementedError
180 def Open(self, force=False):
181 """Make the device ready for use.
183 This makes the device ready for I/O. For now, just the DRBD
186 The force parameter signifies that if the device has any kind of
187 --force thing, it should be used, we know what we are doing.
190 raise NotImplementedError
194 """Shut down the device, freeing its children.
196 This undoes the `Assemble()` work, except for the child
197 assembling; as such, the children on the device are still
198 assembled after this call.
201 raise NotImplementedError
204 def SetSyncSpeed(self, speed):
205 """Adjust the sync speed of the mirror.
207 In case this is not a mirroring device, this is no-op.
212 for child in self._children:
213 result = result and child.SetSyncSpeed(speed)
217 def GetSyncStatus(self):
218 """Returns the sync status of the device.
220 If this device is a mirroring device, this function returns the
221 status of the mirror.
224 (sync_percent, estimated_time, is_degraded)
226 If sync_percent is None, it means all is ok
227 If estimated_time is None, it means we can't estimate
228 the time needed, otherwise it's the time left in seconds
229 If is_degraded is True, it means the device is missing
230 redundancy. This is usually a sign that something went wrong in
231 the device setup, if sync_percent is None.
234 return None, None, False
237 def CombinedSyncStatus(self):
238 """Calculate the mirror status recursively for our children.
240 The return value is the same as for `GetSyncStatus()` except the
241 minimum percent and maximum time are calculated across our
245 min_percent, max_time, is_degraded = self.GetSyncStatus()
247 for child in self._children:
248 c_percent, c_time, c_degraded = child.GetSyncStatus()
249 if min_percent is None:
250 min_percent = c_percent
251 elif c_percent is not None:
252 min_percent = min(min_percent, c_percent)
255 elif c_time is not None:
256 max_time = max(max_time, c_time)
257 is_degraded = is_degraded or c_degraded
258 return min_percent, max_time, is_degraded
261 def SetInfo(self, text):
262 """Update metadata with info text.
264 Only supported for some device types.
267 for child in self._children:
272 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
273 (self.__class__, self.unique_id, self._children,
274 self.major, self.minor, self.dev_path))
277 class LogicalVolume(BlockDev):
278 """Logical Volume block device.
281 def __init__(self, unique_id, children):
282 """Attaches to a LV device.
284 The unique_id is a tuple (vg_name, lv_name)
287 super(LogicalVolume, self).__init__(unique_id, children)
288 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
289 raise ValueError("Invalid configuration data %s" % str(unique_id))
290 self._vg_name, self._lv_name = unique_id
291 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
296 def Create(cls, unique_id, children, size):
297 """Create a new logical volume.
300 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
301 raise ValueError("Invalid configuration data %s" % str(unique_id))
302 vg_name, lv_name = unique_id
303 pvs_info = cls.GetPVInfo(vg_name)
305 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
310 pvlist = [ pv[1] for pv in pvs_info ]
311 free_size = sum([ pv[0] for pv in pvs_info ])
313 # The size constraint should have been checked from the master before
314 # calling the create function.
316 raise errors.BlockDeviceError("Not enough free space: required %s,"
317 " available %s" % (size, free_size))
318 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
321 raise errors.BlockDeviceError(result.fail_reason)
322 return LogicalVolume(unique_id, children)
325 def GetPVInfo(vg_name):
326 """Get the free space info for PVs in a volume group.
329 vg_name: the volume group name
332 list of (free_space, name) with free_space in mebibytes
335 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
336 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
338 result = utils.RunCmd(command)
340 logger.Error("Can't get the PV information: %s" % result.fail_reason)
343 for line in result.stdout.splitlines():
344 fields = line.strip().split(':')
346 logger.Error("Can't parse pvs output: line '%s'" % line)
348 # skip over pvs from another vg or ones which are not allocatable
349 if fields[1] != vg_name or fields[3][0] != 'a':
351 data.append((float(fields[2]), fields[0]))
356 """Remove this logical volume.
359 if not self.minor and not self.Attach():
360 # the LV does not exist
362 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
363 (self._vg_name, self._lv_name)])
365 logger.Error("Can't lvremove: %s" % result.fail_reason)
367 return not result.failed
371 """Attach to an existing LV.
373 This method will try to see if an existing and active LV exists
374 which matches the our name. If so, its major/minor will be
378 result = utils.RunCmd(["lvdisplay", self.dev_path])
380 logger.Error("Can't find LV %s: %s" %
381 (self.dev_path, result.fail_reason))
383 match = re.compile("^ *Block device *([0-9]+):([0-9]+).*$")
384 for line in result.stdout.splitlines():
385 match_result = match.match(line)
387 self.major = int(match_result.group(1))
388 self.minor = int(match_result.group(2))
394 """Assemble the device.
396 This is a no-op for the LV device type. Eventually, we could
397 lvchange -ay here if we see that the LV is not active.
404 """Shutdown the device.
406 This is a no-op for the LV device type, as we don't deactivate the
414 """Return the status of the device.
416 Logical volumes will can be in all four states, although we don't
417 deactivate (lvchange -an) them when shutdown, so STATUS_EXISTING
418 should not be seen for our devices.
421 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
423 logger.Error("Can't display lv: %s" % result.fail_reason)
424 return self.STATUS_UNKNOWN
425 out = result.stdout.strip()
426 # format: type/permissions/alloc/fixed_minor/state/open
428 return self.STATUS_UNKNOWN
429 #writable = (out[1] == "w")
430 active = (out[4] == "a")
431 online = (out[5] == "o")
433 retval = self.STATUS_ONLINE
435 retval = self.STATUS_STANDBY
437 retval = self.STATUS_EXISTING
442 def Open(self, force=False):
443 """Make the device ready for I/O.
445 This is a no-op for the LV device type.
452 """Notifies that the device will no longer be used for I/O.
454 This is a no-op for the LV device type.
460 def Snapshot(self, size):
461 """Create a snapshot copy of an lvm block device.
464 snap_name = self._lv_name + ".snap"
466 # remove existing snapshot if found
467 snap = LogicalVolume((self._vg_name, snap_name), None)
470 pvs_info = self.GetPVInfo(self._vg_name)
472 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
476 free_size, pv_name = pvs_info[0]
478 raise errors.BlockDeviceError("Not enough free space: required %s,"
479 " available %s" % (size, free_size))
481 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
482 "-n%s" % snap_name, self.dev_path])
484 raise errors.BlockDeviceError("command: %s error: %s" %
485 (result.cmd, result.fail_reason))
490 def SetInfo(self, text):
491 """Update metadata with info text.
494 BlockDev.SetInfo(self, text)
496 # Replace invalid characters
497 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
498 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
500 # Only up to 128 characters are allowed
503 result = utils.RunCmd(["lvchange", "--addtag", text,
506 raise errors.BlockDeviceError("Command: %s error: %s" %
507 (result.cmd, result.fail_reason))
510 class MDRaid1(BlockDev):
511 """raid1 device implemented via md.
514 def __init__(self, unique_id, children):
515 super(MDRaid1, self).__init__(unique_id, children)
521 """Find an array which matches our config and attach to it.
523 This tries to find a MD array which has the same UUID as our own.
526 minor = self._FindMDByUUID(self.unique_id)
527 if minor is not None:
528 self._SetFromMinor(minor)
533 return (minor is not None)
538 """Compute the list of in-use MD devices.
540 It doesn't matter if the used device have other raid level, just
541 that they are in use.
544 mdstat = open("/proc/mdstat", "r")
545 data = mdstat.readlines()
549 valid_line = re.compile("^md([0-9]+) : .*$")
551 match = valid_line.match(line)
553 md_no = int(match.group(1))
554 used_md[md_no] = line
560 def _GetDevInfo(minor):
561 """Get info about a MD device.
563 Currently only uuid is returned.
566 result = utils.RunCmd(["mdadm", "-D", "/dev/md%d" % minor])
568 logger.Error("Can't display md: %s" % result.fail_reason)
571 for line in result.stdout.splitlines():
573 kv = line.split(" : ", 1)
576 retval["uuid"] = kv[1].split()[0]
577 elif kv[0] == "State":
578 retval["state"] = kv[1].split(", ")
583 def _FindUnusedMinor():
584 """Compute an unused MD minor.
586 This code assumes that there are 256 minors only.
589 used_md = MDRaid1._GetUsedDevs()
596 logger.Error("Critical: Out of md minor numbers.")
602 def _FindMDByUUID(cls, uuid):
603 """Find the minor of an MD array with a given UUID.
606 md_list = cls._GetUsedDevs()
607 for minor in md_list:
608 info = cls._GetDevInfo(minor)
609 if info and info["uuid"] == uuid:
615 def _ZeroSuperblock(dev_path):
616 """Zero the possible locations for an MD superblock.
618 The zero-ing can't be done via ``mdadm --zero-superblock`` as that
619 fails in versions 2.x with the same error code as non-writable
622 The superblocks are located at (negative values are relative to
623 the end of the block device):
624 - -128k to end for version 0.90 superblock
625 - -8k to -12k for version 1.0 superblock (included in the above)
626 - 0k to 4k for version 1.1 superblock
627 - 4k to 8k for version 1.2 superblock
629 To cover all situations, the zero-ing will be:
633 As such, the minimum device size must be 128k, otherwise we'll get
636 Note that this function depends on the fact that one can open,
637 read and write block devices normally.
640 overwrite_size = 128 * 1024
641 empty_buf = '\0' * overwrite_size
642 fd = open(dev_path, "r+")
648 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
649 fd.seek(-overwrite_size, 2)
653 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
658 def Create(cls, unique_id, children, size):
659 """Create a new MD raid1 array.
662 if not isinstance(children, (tuple, list)):
663 raise ValueError("Invalid setup data for MDRaid1 dev: %s" %
666 if not isinstance(i, BlockDev):
667 raise ValueError("Invalid member in MDRaid1 dev: %s" % type(i))
670 cls._ZeroSuperblock(i.dev_path)
671 except EnvironmentError, err:
672 logger.Error("Can't zero superblock for %s: %s" %
673 (i.dev_path, str(err)))
675 minor = cls._FindUnusedMinor()
676 result = utils.RunCmd(["mdadm", "--create", "/dev/md%d" % minor,
677 "--auto=yes", "--force", "-l1",
678 "-n%d" % len(children)] +
679 [dev.dev_path for dev in children])
682 logger.Error("Can't create md: %s: %s" % (result.fail_reason,
685 info = cls._GetDevInfo(minor)
686 if not info or not "uuid" in info:
687 logger.Error("Wrong information returned from mdadm -D: %s" % str(info))
689 return MDRaid1(info["uuid"], children)
693 """Stub remove function for MD RAID 1 arrays.
695 We don't remove the superblock right now. Mark a to do.
698 #TODO: maybe zero superblock on child devices?
699 return self.Shutdown()
702 def AddChild(self, device):
703 """Add a new member to the md raid1.
706 if self.minor is None and not self.Attach():
707 raise errors.BlockDeviceError("Can't attach to device")
708 if device.dev_path is None:
709 raise errors.BlockDeviceError("New child is not initialised")
710 result = utils.RunCmd(["mdadm", "-a", self.dev_path, device.dev_path])
712 raise errors.BlockDeviceError("Failed to add new device to array: %s" %
714 new_len = len(self._children) + 1
715 result = utils.RunCmd(["mdadm", "--grow", self.dev_path, "-n", new_len])
717 raise errors.BlockDeviceError("Can't grow md array: %s" %
719 self._children.append(device)
722 def RemoveChild(self, dev_path):
723 """Remove member from the md raid1.
726 if self.minor is None and not self.Attach():
727 raise errors.BlockDeviceError("Can't attach to device")
728 if len(self._children) == 1:
729 raise errors.BlockDeviceError("Can't reduce member when only one"
731 for device in self._children:
732 if device.dev_path == dev_path:
735 raise errors.BlockDeviceError("Can't find child with this path")
736 new_len = len(self._children) - 1
737 result = utils.RunCmd(["mdadm", "-f", self.dev_path, dev_path])
739 raise errors.BlockDeviceError("Failed to mark device as failed: %s" %
742 # it seems here we need a short delay for MD to update its
745 result = utils.RunCmd(["mdadm", "-r", self.dev_path, dev_path])
747 raise errors.BlockDeviceError("Failed to remove device from array:"
748 " %s" % result.output)
749 result = utils.RunCmd(["mdadm", "--grow", "--force", self.dev_path,
752 raise errors.BlockDeviceError("Can't shrink md array: %s" %
754 self._children.remove(device)
758 """Return the status of the device.
762 if self.minor is None:
763 retval = self.STATUS_UNKNOWN
765 retval = self.STATUS_ONLINE
769 def _SetFromMinor(self, minor):
770 """Set our parameters based on the given minor.
772 This sets our minor variable and our dev_path.
776 self.dev_path = "/dev/md%d" % minor
780 """Assemble the MD device.
782 At this point we should have:
783 - list of children devices
787 result = super(MDRaid1, self).Assemble()
790 md_list = self._GetUsedDevs()
791 for minor in md_list:
792 info = self._GetDevInfo(minor)
793 if info and info["uuid"] == self.unique_id:
794 self._SetFromMinor(minor)
795 logger.Info("MD array %s already started" % str(self))
797 free_minor = self._FindUnusedMinor()
798 result = utils.RunCmd(["mdadm", "-A", "--auto=yes", "--uuid",
799 self.unique_id, "/dev/md%d" % free_minor] +
800 [bdev.dev_path for bdev in self._children])
802 logger.Error("Can't assemble MD array: %s: %s" %
803 (result.fail_reason, result.output))
806 self.minor = free_minor
807 return not result.failed
811 """Tear down the MD array.
813 This does a 'mdadm --stop' so after this command, the array is no
817 if self.minor is None and not self.Attach():
818 logger.Info("MD object not attached to a device")
821 result = utils.RunCmd(["mdadm", "--stop", "/dev/md%d" % self.minor])
823 logger.Error("Can't stop MD array: %s" % result.fail_reason)
830 def SetSyncSpeed(self, kbytes):
831 """Set the maximum sync speed for the MD array.
834 result = super(MDRaid1, self).SetSyncSpeed(kbytes)
835 if self.minor is None:
836 logger.Error("MD array not attached to a device")
838 f = open("/sys/block/md%d/md/sync_speed_max" % self.minor, "w")
840 f.write("%d" % kbytes)
843 f = open("/sys/block/md%d/md/sync_speed_min" % self.minor, "w")
845 f.write("%d" % (kbytes/2))
851 def GetSyncStatus(self):
852 """Returns the sync status of the device.
855 (sync_percent, estimated_time)
857 If sync_percent is None, it means all is ok
858 If estimated_time is None, it means we can't esimate
859 the time needed, otherwise it's the time left in seconds
862 if self.minor is None and not self.Attach():
863 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
864 dev_info = self._GetDevInfo(self.minor)
865 is_clean = ("state" in dev_info and
866 len(dev_info["state"]) == 1 and
867 dev_info["state"][0] in ("clean", "active"))
868 sys_path = "/sys/block/md%s/md/" % self.minor
869 f = file(sys_path + "sync_action")
870 sync_status = f.readline().strip()
872 if sync_status == "idle":
873 return None, None, not is_clean
874 f = file(sys_path + "sync_completed")
875 sync_completed = f.readline().strip().split(" / ")
877 if len(sync_completed) != 2:
878 return 0, None, not is_clean
879 sync_done, sync_total = [float(i) for i in sync_completed]
880 sync_percent = 100.0*sync_done/sync_total
881 f = file(sys_path + "sync_speed")
882 sync_speed_k = int(f.readline().strip())
883 if sync_speed_k == 0:
886 time_est = (sync_total - sync_done) / 2 / sync_speed_k
887 return sync_percent, time_est, not is_clean
890 def Open(self, force=False):
891 """Make the device ready for I/O.
893 This is a no-op for the MDRaid1 device type, although we could use
894 the 2.6.18's new array_state thing.
901 """Notifies that the device will no longer be used for I/O.
903 This is a no-op for the MDRaid1 device type, but see comment for
911 class DRBDev(BlockDev):
912 """DRBD block device.
914 This implements the local host part of the DRBD device, i.e. it
915 doesn't do anything to the supposed peer. If you need a fully
916 connected DRBD pair, you need to use this class on both hosts.
918 The unique_id for the drbd device is the (local_ip, local_port,
919 remote_ip, remote_port) tuple, and it must have two children: the
920 data device and the meta_device. The meta device is checked for
921 valid size and is zeroed on create.
925 _ST_UNCONFIGURED = "Unconfigured"
926 _ST_WFCONNECTION = "WFConnection"
927 _ST_CONNECTED = "Connected"
929 def __init__(self, unique_id, children):
930 super(DRBDev, self).__init__(unique_id, children)
931 self.major = self._DRBD_MAJOR
932 if len(children) != 2:
933 raise ValueError("Invalid configuration data %s" % str(children))
934 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
935 raise ValueError("Invalid configuration data %s" % str(unique_id))
936 self._lhost, self._lport, self._rhost, self._rport = unique_id
941 """Return the path to a drbd device for a given minor.
944 return "/dev/drbd%d" % minor
948 """Return data from /proc/drbd.
951 stat = open("/proc/drbd", "r")
952 data = stat.read().splitlines()
958 def _GetUsedDevs(cls):
959 """Compute the list of used DRBD devices.
962 data = cls._GetProcData()
965 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
967 match = valid_line.match(line)
970 minor = int(match.group(1))
971 state = match.group(2)
972 if state == cls._ST_UNCONFIGURED:
974 used_devs[minor] = state, line
980 def _FindUnusedMinor(cls):
981 """Find an unused DRBD device.
984 data = cls._GetProcData()
986 valid_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
988 match = valid_line.match(line)
990 return int(match.group(1))
991 logger.Error("Error: no free drbd minors!")
996 def _GetDevInfo(cls, minor):
997 """Get details about a given DRBD minor.
999 This return, if available, the local backing device in (major,
1000 minor) formant and the local and remote (ip, port) information.
1004 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1006 logger.Error("Can't display the drbd config: %s" % result.fail_reason)
1009 if out == "Not configured\n":
1011 for line in out.splitlines():
1012 if "local_dev" not in data:
1013 match = re.match("^Lower device: ([0-9]+):([0-9]+) .*$", line)
1015 data["local_dev"] = (int(match.group(1)), int(match.group(2)))
1017 if "meta_dev" not in data:
1018 match = re.match("^Meta device: (([0-9]+):([0-9]+)|internal).*$", line)
1020 if match.group(2) is not None and match.group(3) is not None:
1021 # matched on the major/minor
1022 data["meta_dev"] = (int(match.group(2)), int(match.group(3)))
1024 # matched on the "internal" string
1025 data["meta_dev"] = match.group(1)
1026 # in this case, no meta_index is in the output
1027 data["meta_index"] = -1
1029 if "meta_index" not in data:
1030 match = re.match("^Meta index: ([0-9]+).*$", line)
1032 data["meta_index"] = int(match.group(1))
1034 if "local_addr" not in data:
1035 match = re.match("^Local address: ([0-9.]+):([0-9]+)$", line)
1037 data["local_addr"] = (match.group(1), int(match.group(2)))
1039 if "remote_addr" not in data:
1040 match = re.match("^Remote address: ([0-9.]+):([0-9]+)$", line)
1042 data["remote_addr"] = (match.group(1), int(match.group(2)))
1047 def _MatchesLocal(self, info):
1048 """Test if our local config matches with an existing device.
1050 The parameter should be as returned from `_GetDevInfo()`. This
1051 method tests if our local backing device is the same as the one in
1052 the info parameter, in effect testing if we look like the given
1056 if not ("local_dev" in info and "meta_dev" in info and
1057 "meta_index" in info):
1060 backend = self._children[0]
1061 if backend is not None:
1062 retval = (info["local_dev"] == (backend.major, backend.minor))
1064 retval = (info["local_dev"] == (0, 0))
1065 meta = self._children[1]
1066 if meta is not None:
1067 retval = retval and (info["meta_dev"] == (meta.major, meta.minor))
1068 retval = retval and (info["meta_index"] == 0)
1070 retval = retval and (info["meta_dev"] == "internal" and
1071 info["meta_index"] == -1)
1075 def _MatchesNet(self, info):
1076 """Test if our network config matches with an existing device.
1078 The parameter should be as returned from `_GetDevInfo()`. This
1079 method tests if our network configuration is the same as the one
1080 in the info parameter, in effect testing if we look like the given
1084 if (((self._lhost is None and not ("local_addr" in info)) and
1085 (self._rhost is None and not ("remote_addr" in info)))):
1088 if self._lhost is None:
1091 if not ("local_addr" in info and
1092 "remote_addr" in info):
1095 retval = (info["local_addr"] == (self._lhost, self._lport))
1096 retval = (retval and
1097 info["remote_addr"] == (self._rhost, self._rport))
1102 def _IsValidMeta(meta_device):
1103 """Check if the given meta device looks like a valid one.
1105 This currently only check the size, which must be around
1109 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1111 logger.Error("Failed to get device size: %s" % result.fail_reason)
1114 sectors = int(result.stdout)
1116 logger.Error("Invalid output from blockdev: '%s'" % result.stdout)
1118 bytes = sectors * 512
1119 if bytes < 128*1024*1024: # less than 128MiB
1120 logger.Error("Meta device too small (%.2fMib)" % (bytes/1024/1024))
1122 if bytes > (128+32)*1024*1024: # account for an extra (big) PE on LVM
1123 logger.Error("Meta device too big (%.2fMiB)" % (bytes/1024/1024))
1129 def _AssembleLocal(cls, minor, backend, meta):
1130 """Configure the local part of a DRBD device.
1132 This is the first thing that must be done on an unconfigured DRBD
1133 device. And it must be done only once.
1136 if not cls._IsValidMeta(meta):
1138 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1139 backend, meta, "0", "-e", "detach"])
1141 logger.Error("Can't attach local disk: %s" % result.output)
1142 return not result.failed
1146 def _ShutdownLocal(cls, minor):
1147 """Detach from the local device.
1149 I/Os will continue to be served from the remote device. If we
1150 don't have a remote device, this operation will fail.
1153 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1155 logger.Error("Can't detach local device: %s" % result.output)
1156 return not result.failed
1160 def _ShutdownAll(minor):
1161 """Deactivate the device.
1163 This will, of course, fail if the device is in use.
1166 result = utils.RunCmd(["drbdsetup", DRBDev._DevPath(minor), "down"])
1168 logger.Error("Can't shutdown drbd device: %s" % result.output)
1169 return not result.failed
1173 def _AssembleNet(cls, minor, net_info, protocol):
1174 """Configure the network part of the device.
1176 This operation can be, in theory, done multiple times, but there
1177 have been cases (in lab testing) in which the network part of the
1178 device had become stuck and couldn't be shut down because activity
1179 from the new peer (also stuck) triggered a timer re-init and
1180 needed remote peer interface shutdown in order to clear. So please
1181 don't change online the net config.
1184 lhost, lport, rhost, rport = net_info
1185 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "net",
1186 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport),
1189 logger.Error("Can't setup network for dbrd device: %s" %
1193 timeout = time.time() + 10
1195 while time.time() < timeout:
1196 info = cls._GetDevInfo(minor)
1197 if not "local_addr" in info or not "remote_addr" in info:
1200 if (info["local_addr"] != (lhost, lport) or
1201 info["remote_addr"] != (rhost, rport)):
1207 logger.Error("Timeout while configuring network")
1213 def _ShutdownNet(cls, minor):
1214 """Disconnect from the remote peer.
1216 This fails if we don't have a local device.
1219 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1220 logger.Error("Can't shutdown network: %s" % result.output)
1221 return not result.failed
1224 def _SetFromMinor(self, minor):
1225 """Set our parameters based on the given minor.
1227 This sets our minor variable and our dev_path.
1231 self.minor = self.dev_path = None
1234 self.dev_path = self._DevPath(minor)
1238 """Assemble the drbd.
1241 - if we have a local backing device, we bind to it by:
1242 - checking the list of used drbd devices
1243 - check if the local minor use of any of them is our own device
1246 - if we have a local/remote net info:
1247 - redo the local backing device step for the remote device
1248 - check if any drbd device is using the local port,
1250 - check if any remote drbd device is using the remote
1251 port, if yes abort (for now)
1253 - bind the remote net port
1257 if self.minor is not None:
1258 logger.Info("Already assembled")
1261 result = super(DRBDev, self).Assemble()
1265 minor = self._FindUnusedMinor()
1267 raise errors.BlockDeviceError("Not enough free minors for DRBD!")
1268 need_localdev_teardown = False
1269 if self._children[0]:
1270 result = self._AssembleLocal(minor, self._children[0].dev_path,
1271 self._children[1].dev_path)
1274 need_localdev_teardown = True
1275 if self._lhost and self._lport and self._rhost and self._rport:
1276 result = self._AssembleNet(minor,
1277 (self._lhost, self._lport,
1278 self._rhost, self._rport),
1281 if need_localdev_teardown:
1282 # we will ignore failures from this
1283 logger.Error("net setup failed, tearing down local device")
1284 self._ShutdownAll(minor)
1286 self._SetFromMinor(minor)
1291 """Shutdown the DRBD device.
1294 if self.minor is None and not self.Attach():
1295 logger.Info("DRBD device not attached to a device during Shutdown")
1297 if not self._ShutdownAll(self.minor):
1300 self.dev_path = None
1305 """Find a DRBD device which matches our config and attach to it.
1307 In case of partially attached (local device matches but no network
1308 setup), we perform the network attach. If successful, we re-test
1309 the attach if can return success.
1312 for minor in self._GetUsedDevs():
1313 info = self._GetDevInfo(minor)
1314 match_l = self._MatchesLocal(info)
1315 match_r = self._MatchesNet(info)
1316 if match_l and match_r:
1318 if match_l and not match_r and "local_addr" not in info:
1319 res_r = self._AssembleNet(minor,
1320 (self._lhost, self._lport,
1321 self._rhost, self._rport),
1323 if res_r and self._MatchesNet(self._GetDevInfo(minor)):
1328 self._SetFromMinor(minor)
1329 return minor is not None
1332 def Open(self, force=False):
1333 """Make the local state primary.
1335 If the 'force' parameter is given, the '--do-what-I-say' parameter
1336 is given. Since this is a pottentialy dangerous operation, the
1337 force flag should be only given after creation, when it actually
1341 if self.minor is None and not self.Attach():
1342 logger.Error("DRBD cannot attach to a device during open")
1344 cmd = ["drbdsetup", self.dev_path, "primary"]
1346 cmd.append("--do-what-I-say")
1347 result = utils.RunCmd(cmd)
1349 logger.Error("Can't make drbd device primary: %s" % result.output)
1355 """Make the local state secondary.
1357 This will, of course, fail if the device is in use.
1360 if self.minor is None and not self.Attach():
1361 logger.Info("Instance not attached to a device")
1362 raise errors.BlockDeviceError("Can't find device")
1363 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1365 logger.Error("Can't switch drbd device to secondary: %s" % result.output)
1366 raise errors.BlockDeviceError("Can't switch drbd device to secondary")
1369 def SetSyncSpeed(self, kbytes):
1370 """Set the speed of the DRBD syncer.
1373 children_result = super(DRBDev, self).SetSyncSpeed(kbytes)
1374 if self.minor is None:
1375 logger.Info("Instance not attached to a device")
1377 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1380 logger.Error("Can't change syncer rate: %s " % result.fail_reason)
1381 return not result.failed and children_result
1384 def GetSyncStatus(self):
1385 """Returns the sync status of the device.
1388 (sync_percent, estimated_time)
1390 If sync_percent is None, it means all is ok
1391 If estimated_time is None, it means we can't esimate
1392 the time needed, otherwise it's the time left in seconds
1395 if self.minor is None and not self.Attach():
1396 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1397 proc_info = self._MassageProcData(self._GetProcData())
1398 if self.minor not in proc_info:
1399 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1401 line = proc_info[self.minor]
1402 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
1403 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
1405 sync_percent = float(match.group(1))
1406 hours = int(match.group(2))
1407 minutes = int(match.group(3))
1408 seconds = int(match.group(4))
1409 est_time = hours * 3600 + minutes * 60 + seconds
1413 match = re.match("^ *[0-9]+: cs:([^ ]+).*$", line)
1415 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
1417 client_state = match.group(1)
1418 is_degraded = client_state != "Connected"
1419 return sync_percent, est_time, is_degraded
1423 def _MassageProcData(data):
1424 """Transform the output of _GetProdData into a nicer form.
1427 a dictionary of minor: joined lines from /proc/drbd for that minor
1430 lmatch = re.compile("^ *([0-9]+):.*$")
1432 old_minor = old_line = None
1434 lresult = lmatch.match(line)
1435 if lresult is not None:
1436 if old_minor is not None:
1437 results[old_minor] = old_line
1438 old_minor = int(lresult.group(1))
1441 if old_minor is not None:
1442 old_line += " " + line.strip()
1444 if old_minor is not None:
1445 results[old_minor] = old_line
1449 def GetStatus(self):
1450 """Compute the status of the DRBD device
1452 Note that DRBD devices don't have the STATUS_EXISTING state.
1455 if self.minor is None and not self.Attach():
1456 return self.STATUS_UNKNOWN
1458 data = self._GetProcData()
1459 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
1462 mresult = match.match(line)
1466 logger.Error("Can't find myself!")
1467 return self.STATUS_UNKNOWN
1469 state = mresult.group(2)
1470 if state == "Primary":
1471 result = self.STATUS_ONLINE
1473 result = self.STATUS_STANDBY
1479 def _ZeroDevice(device):
1482 This writes until we get ENOSPC.
1485 f = open(device, "w")
1486 buf = "\0" * 1048576
1490 except IOError, err:
1491 if err.errno != errno.ENOSPC:
1496 def Create(cls, unique_id, children, size):
1497 """Create a new DRBD device.
1499 Since DRBD devices are not created per se, just assembled, this
1500 function just zeroes the meta device.
1503 if len(children) != 2:
1504 raise errors.ProgrammerError("Invalid setup for the drbd device")
1507 if not meta.Attach():
1508 raise errors.BlockDeviceError("Can't attach to meta device")
1509 if not cls._IsValidMeta(meta.dev_path):
1510 raise errors.BlockDeviceError("Invalid meta device")
1511 logger.Info("Started zeroing device %s" % meta.dev_path)
1512 cls._ZeroDevice(meta.dev_path)
1513 logger.Info("Done zeroing device %s" % meta.dev_path)
1514 return cls(unique_id, children)
1518 """Stub remove for DRBD devices.
1521 return self.Shutdown()
1525 "lvm": LogicalVolume,
1526 "md_raid1": MDRaid1,
1531 def FindDevice(dev_type, unique_id, children):
1532 """Search for an existing, assembled device.
1534 This will succeed only if the device exists and is assembled, but it
1535 does not do any actions in order to activate the device.
1538 if dev_type not in DEV_MAP:
1539 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1540 device = DEV_MAP[dev_type](unique_id, children)
1541 if not device.Attach():
1546 def AttachOrAssemble(dev_type, unique_id, children):
1547 """Try to attach or assemble an existing device.
1549 This will attach to an existing assembled device or will assemble
1550 the device, as needed, to bring it fully up.
1553 if dev_type not in DEV_MAP:
1554 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1555 device = DEV_MAP[dev_type](unique_id, children)
1556 if not device.Attach():
1558 if not device.Attach():
1559 raise errors.BlockDeviceError("Can't find a valid block device for"
1561 (dev_type, unique_id, children))
1565 def Create(dev_type, unique_id, children, size):
1569 if dev_type not in DEV_MAP:
1570 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1571 device = DEV_MAP[dev_type].Create(unique_id, children, size)