4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
29 from ganeti import utils
30 from ganeti import logger
31 from ganeti import errors
32 from ganeti import constants
35 class BlockDev(object):
36 """Block device abstract class.
38 A block device can be in the following states:
39 - not existing on the system, and by `Create()` it goes into:
40 - existing but not setup/not active, and by `Assemble()` goes into:
41 - active read-write and by `Open()` it goes into
42 - online (=used, or ready for use)
44 A device can also be online but read-only, however we are not using
45 the readonly state (MD and LV have it, if needed in the future)
46 and we are usually looking at this like at a stack, so it's easier
47 to conceptualise the transition from not-existing to online and back
50 The many different states of the device are due to the fact that we
51 need to cover many device types:
52 - logical volumes are created, lvchange -a y $lv, and used
53 - md arrays are created or assembled and used
54 - drbd devices are attached to a local disk/remote peer and made primary
56 The status of the device can be examined by `GetStatus()`, which
57 returns a numerical value, depending on the position in the
58 transition stack of the device.
60 A block device is identified by three items:
61 - the /dev path of the device (dynamic)
62 - a unique ID of the device (static)
63 - it's major/minor pair (dynamic)
65 Not all devices implement both the first two as distinct items. LVM
66 logical volumes have their unique ID (the pair volume group, logical
67 volume name) in a 1-to-1 relation to the dev path. For MD devices,
68 the /dev path is dynamic and the unique ID is the UUID generated at
69 array creation plus the slave list. For DRBD devices, the /dev path
70 is again dynamic and the unique id is the pair (host1, dev1),
73 You can get to a device in two ways:
74 - creating the (real) device, which returns you
75 an attached instance (lvcreate, mdadm --create)
76 - attaching of a python instance to an existing (real) device
78 The second point, the attachement to a device, is different
79 depending on whether the device is assembled or not. At init() time,
80 we search for a device with the same unique_id as us. If found,
81 good. It also means that the device is already assembled. If not,
82 after assembly we'll have our correct major/minor.
91 STATUS_UNKNOWN: "unknown",
92 STATUS_EXISTING: "existing",
93 STATUS_STANDBY: "ready for use",
94 STATUS_ONLINE: "online",
97 def __init__(self, unique_id, children):
98 self._children = children
100 self.unique_id = unique_id
105 """Assemble the device from its components.
107 If this is a plain block device (e.g. LVM) than assemble does
108 nothing, as the LVM has no children and we don't put logical
111 One guarantee is that after the device has been assembled, it
112 knows its major/minor numbers. This allows other devices (usually
113 parents) to probe correctly for their children.
117 for child in self._children:
118 if not isinstance(child, BlockDev):
119 raise TypeError("Invalid child passed of type '%s'" % type(child))
122 status = status and child.Assemble()
128 except errors.BlockDeviceError:
129 for child in self._children:
134 for child in self._children:
139 """Find a device which matches our config and attach to it.
142 raise NotImplementedError
145 """Notifies that the device will no longer be used for I/O.
148 raise NotImplementedError
151 def Create(cls, unique_id, children, size):
152 """Create the device.
154 If the device cannot be created, it will return None
155 instead. Error messages go to the logging system.
157 Note that for some devices, the unique_id is used, and for other,
158 the children. The idea is that these two, taken together, are
159 enough for both creation and assembly (later).
162 raise NotImplementedError
165 """Remove this device.
167 This makes sense only for some of the device types: LV and to a
168 lesser degree, md devices. Also note that if the device can't
169 attach, the removal can't be completed.
172 raise NotImplementedError
174 def Rename(self, new_id):
175 """Rename this device.
177 This may or may not make sense for a given device type.
180 raise NotImplementedError
183 """Return the status of the device.
186 raise NotImplementedError
188 def Open(self, force=False):
189 """Make the device ready for use.
191 This makes the device ready for I/O. For now, just the DRBD
194 The force parameter signifies that if the device has any kind of
195 --force thing, it should be used, we know what we are doing.
198 raise NotImplementedError
201 """Shut down the device, freeing its children.
203 This undoes the `Assemble()` work, except for the child
204 assembling; as such, the children on the device are still
205 assembled after this call.
208 raise NotImplementedError
210 def SetSyncSpeed(self, speed):
211 """Adjust the sync speed of the mirror.
213 In case this is not a mirroring device, this is no-op.
218 for child in self._children:
219 result = result and child.SetSyncSpeed(speed)
222 def GetSyncStatus(self):
223 """Returns the sync status of the device.
225 If this device is a mirroring device, this function returns the
226 status of the mirror.
229 (sync_percent, estimated_time, is_degraded, ldisk)
231 If sync_percent is None, it means the device is not syncing.
233 If estimated_time is None, it means we can't estimate
234 the time needed, otherwise it's the time left in seconds.
236 If is_degraded is True, it means the device is missing
237 redundancy. This is usually a sign that something went wrong in
238 the device setup, if sync_percent is None.
240 The ldisk parameter represents the degradation of the local
241 data. This is only valid for some devices, the rest will always
242 return False (not degraded).
245 return None, None, False, False
248 def CombinedSyncStatus(self):
249 """Calculate the mirror status recursively for our children.
251 The return value is the same as for `GetSyncStatus()` except the
252 minimum percent and maximum time are calculated across our
256 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
258 for child in self._children:
259 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
260 if min_percent is None:
261 min_percent = c_percent
262 elif c_percent is not None:
263 min_percent = min(min_percent, c_percent)
266 elif c_time is not None:
267 max_time = max(max_time, c_time)
268 is_degraded = is_degraded or c_degraded
269 ldisk = ldisk or c_ldisk
270 return min_percent, max_time, is_degraded, ldisk
273 def SetInfo(self, text):
274 """Update metadata with info text.
276 Only supported for some device types.
279 for child in self._children:
284 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
285 (self.__class__, self.unique_id, self._children,
286 self.major, self.minor, self.dev_path))
289 class LogicalVolume(BlockDev):
290 """Logical Volume block device.
293 def __init__(self, unique_id, children):
294 """Attaches to a LV device.
296 The unique_id is a tuple (vg_name, lv_name)
299 super(LogicalVolume, self).__init__(unique_id, children)
300 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
301 raise ValueError("Invalid configuration data %s" % str(unique_id))
302 self._vg_name, self._lv_name = unique_id
303 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
307 def Create(cls, unique_id, children, size):
308 """Create a new logical volume.
311 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
312 raise ValueError("Invalid configuration data %s" % str(unique_id))
313 vg_name, lv_name = unique_id
314 pvs_info = cls.GetPVInfo(vg_name)
316 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
321 pvlist = [ pv[1] for pv in pvs_info ]
322 free_size = sum([ pv[0] for pv in pvs_info ])
324 # The size constraint should have been checked from the master before
325 # calling the create function.
327 raise errors.BlockDeviceError("Not enough free space: required %s,"
328 " available %s" % (size, free_size))
329 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
332 raise errors.BlockDeviceError(result.fail_reason)
333 return LogicalVolume(unique_id, children)
336 def GetPVInfo(vg_name):
337 """Get the free space info for PVs in a volume group.
340 vg_name: the volume group name
343 list of (free_space, name) with free_space in mebibytes
346 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
347 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
349 result = utils.RunCmd(command)
351 logger.Error("Can't get the PV information: %s" % result.fail_reason)
354 for line in result.stdout.splitlines():
355 fields = line.strip().split(':')
357 logger.Error("Can't parse pvs output: line '%s'" % line)
359 # skip over pvs from another vg or ones which are not allocatable
360 if fields[1] != vg_name or fields[3][0] != 'a':
362 data.append((float(fields[2]), fields[0]))
367 """Remove this logical volume.
370 if not self.minor and not self.Attach():
371 # the LV does not exist
373 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
374 (self._vg_name, self._lv_name)])
376 logger.Error("Can't lvremove: %s" % result.fail_reason)
378 return not result.failed
380 def Rename(self, new_id):
381 """Rename this logical volume.
384 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
385 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
386 new_vg, new_name = new_id
387 if new_vg != self._vg_name:
388 raise errors.ProgrammerError("Can't move a logical volume across"
389 " volume groups (from %s to to %s)" %
390 (self._vg_name, new_vg))
391 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
393 raise errors.BlockDeviceError("Failed to rename the logical volume: %s" %
395 self._lv_name = new_name
396 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
400 """Attach to an existing LV.
402 This method will try to see if an existing and active LV exists
403 which matches the our name. If so, its major/minor will be
407 result = utils.RunCmd(["lvdisplay", self.dev_path])
409 logger.Error("Can't find LV %s: %s, %s" %
410 (self.dev_path, result.fail_reason, result.output))
412 match = re.compile("^ *Block device *([0-9]+):([0-9]+).*$")
413 for line in result.stdout.splitlines():
414 match_result = match.match(line)
416 self.major = int(match_result.group(1))
417 self.minor = int(match_result.group(2))
422 """Assemble the device.
424 We alway run `lvchange -ay` on the LV to ensure it's active before
425 use, as there were cases when xenvg was not active after boot
426 (also possibly after disk issues).
429 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
431 logger.Error("Can't activate lv %s: %s" % (self.dev_path, result.output))
432 return not result.failed
435 """Shutdown the device.
437 This is a no-op for the LV device type, as we don't deactivate the
444 """Return the status of the device.
446 Logical volumes will can be in all four states, although we don't
447 deactivate (lvchange -an) them when shutdown, so STATUS_EXISTING
448 should not be seen for our devices.
451 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
453 logger.Error("Can't display lv: %s" % result.fail_reason)
454 return self.STATUS_UNKNOWN
455 out = result.stdout.strip()
456 # format: type/permissions/alloc/fixed_minor/state/open
458 return self.STATUS_UNKNOWN
459 #writable = (out[1] == "w")
460 active = (out[4] == "a")
461 online = (out[5] == "o")
463 retval = self.STATUS_ONLINE
465 retval = self.STATUS_STANDBY
467 retval = self.STATUS_EXISTING
471 def GetSyncStatus(self):
472 """Returns the sync status of the device.
474 If this device is a mirroring device, this function returns the
475 status of the mirror.
478 (sync_percent, estimated_time, is_degraded, ldisk)
480 For logical volumes, sync_percent and estimated_time are always
481 None (no recovery in progress, as we don't handle the mirrored LV
482 case). The is_degraded parameter is the inverse of the ldisk
485 For the ldisk parameter, we check if the logical volume has the
486 'virtual' type, which means it's not backed by existing storage
487 anymore (read from it return I/O error). This happens after a
488 physical disk failure and subsequent 'vgreduce --removemissing' on
492 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
494 logger.Error("Can't display lv: %s" % result.fail_reason)
495 return None, None, True, True
496 out = result.stdout.strip()
497 # format: type/permissions/alloc/fixed_minor/state/open
499 logger.Debug("Error in lvs output: attrs=%s, len != 6" % out)
500 return None, None, True, True
501 ldisk = out[0] == 'v' # virtual volume, i.e. doesn't have
503 return None, None, ldisk, ldisk
505 def Open(self, force=False):
506 """Make the device ready for I/O.
508 This is a no-op for the LV device type.
514 """Notifies that the device will no longer be used for I/O.
516 This is a no-op for the LV device type.
521 def Snapshot(self, size):
522 """Create a snapshot copy of an lvm block device.
525 snap_name = self._lv_name + ".snap"
527 # remove existing snapshot if found
528 snap = LogicalVolume((self._vg_name, snap_name), None)
531 pvs_info = self.GetPVInfo(self._vg_name)
533 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
537 free_size, pv_name = pvs_info[0]
539 raise errors.BlockDeviceError("Not enough free space: required %s,"
540 " available %s" % (size, free_size))
542 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
543 "-n%s" % snap_name, self.dev_path])
545 raise errors.BlockDeviceError("command: %s error: %s" %
546 (result.cmd, result.fail_reason))
550 def SetInfo(self, text):
551 """Update metadata with info text.
554 BlockDev.SetInfo(self, text)
556 # Replace invalid characters
557 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
558 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
560 # Only up to 128 characters are allowed
563 result = utils.RunCmd(["lvchange", "--addtag", text,
566 raise errors.BlockDeviceError("Command: %s error: %s" %
567 (result.cmd, result.fail_reason))
570 class MDRaid1(BlockDev):
571 """raid1 device implemented via md.
574 def __init__(self, unique_id, children):
575 super(MDRaid1, self).__init__(unique_id, children)
580 """Find an array which matches our config and attach to it.
582 This tries to find a MD array which has the same UUID as our own.
585 minor = self._FindMDByUUID(self.unique_id)
586 if minor is not None:
587 self._SetFromMinor(minor)
592 return (minor is not None)
596 """Compute the list of in-use MD devices.
598 It doesn't matter if the used device have other raid level, just
599 that they are in use.
602 mdstat = open("/proc/mdstat", "r")
603 data = mdstat.readlines()
607 valid_line = re.compile("^md([0-9]+) : .*$")
609 match = valid_line.match(line)
611 md_no = int(match.group(1))
612 used_md[md_no] = line
617 def _GetDevInfo(minor):
618 """Get info about a MD device.
620 Currently only uuid is returned.
623 result = utils.RunCmd(["mdadm", "-D", "/dev/md%d" % minor])
625 logger.Error("Can't display md: %s" % result.fail_reason)
628 for line in result.stdout.splitlines():
630 kv = line.split(" : ", 1)
633 retval["uuid"] = kv[1].split()[0]
634 elif kv[0] == "State":
635 retval["state"] = kv[1].split(", ")
639 def _FindUnusedMinor():
640 """Compute an unused MD minor.
642 This code assumes that there are 256 minors only.
645 used_md = MDRaid1._GetUsedDevs()
652 logger.Error("Critical: Out of md minor numbers.")
653 raise errors.BlockDeviceError("Can't find a free MD minor")
657 def _FindMDByUUID(cls, uuid):
658 """Find the minor of an MD array with a given UUID.
661 md_list = cls._GetUsedDevs()
662 for minor in md_list:
663 info = cls._GetDevInfo(minor)
664 if info and info["uuid"] == uuid:
669 def _ZeroSuperblock(dev_path):
670 """Zero the possible locations for an MD superblock.
672 The zero-ing can't be done via ``mdadm --zero-superblock`` as that
673 fails in versions 2.x with the same error code as non-writable
676 The superblocks are located at (negative values are relative to
677 the end of the block device):
678 - -128k to end for version 0.90 superblock
679 - -8k to -12k for version 1.0 superblock (included in the above)
680 - 0k to 4k for version 1.1 superblock
681 - 4k to 8k for version 1.2 superblock
683 To cover all situations, the zero-ing will be:
687 As such, the minimum device size must be 128k, otherwise we'll get
690 Note that this function depends on the fact that one can open,
691 read and write block devices normally.
694 overwrite_size = 128 * 1024
695 empty_buf = '\0' * overwrite_size
696 fd = open(dev_path, "r+")
702 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
703 fd.seek(-overwrite_size, 2)
707 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
712 def Create(cls, unique_id, children, size):
713 """Create a new MD raid1 array.
716 if not isinstance(children, (tuple, list)):
717 raise ValueError("Invalid setup data for MDRaid1 dev: %s" %
720 if not isinstance(i, BlockDev):
721 raise ValueError("Invalid member in MDRaid1 dev: %s" % type(i))
724 cls._ZeroSuperblock(i.dev_path)
725 except EnvironmentError, err:
726 logger.Error("Can't zero superblock for %s: %s" %
727 (i.dev_path, str(err)))
729 minor = cls._FindUnusedMinor()
730 result = utils.RunCmd(["mdadm", "--create", "/dev/md%d" % minor,
731 "--auto=yes", "--force", "-l1",
732 "-n%d" % len(children)] +
733 [dev.dev_path for dev in children])
736 logger.Error("Can't create md: %s: %s" % (result.fail_reason,
739 info = cls._GetDevInfo(minor)
740 if not info or not "uuid" in info:
741 logger.Error("Wrong information returned from mdadm -D: %s" % str(info))
743 return MDRaid1(info["uuid"], children)
746 """Stub remove function for MD RAID 1 arrays.
748 We don't remove the superblock right now. Mark a to do.
751 #TODO: maybe zero superblock on child devices?
752 return self.Shutdown()
754 def Rename(self, new_id):
757 This is not supported for md raid1 devices.
760 raise errors.ProgrammerError("Can't rename a md raid1 device")
762 def AddChildren(self, devices):
763 """Add new member(s) to the md raid1.
766 if self.minor is None and not self.Attach():
767 raise errors.BlockDeviceError("Can't attach to device")
769 args = ["mdadm", "-a", self.dev_path]
771 if dev.dev_path is None:
772 raise errors.BlockDeviceError("Child '%s' is not initialised" % dev)
774 args.append(dev.dev_path)
775 result = utils.RunCmd(args)
777 raise errors.BlockDeviceError("Failed to add new device to array: %s" %
779 new_len = len(self._children) + len(devices)
780 result = utils.RunCmd(["mdadm", "--grow", self.dev_path, "-n", new_len])
782 raise errors.BlockDeviceError("Can't grow md array: %s" %
784 self._children.extend(devices)
786 def RemoveChildren(self, devices):
787 """Remove member(s) from the md raid1.
790 if self.minor is None and not self.Attach():
791 raise errors.BlockDeviceError("Can't attach to device")
792 new_len = len(self._children) - len(devices)
794 raise errors.BlockDeviceError("Can't reduce to less than one child")
795 args = ["mdadm", "-f", self.dev_path]
799 for c in self._children:
800 if c.dev_path == dev:
804 raise errors.BlockDeviceError("Can't find device '%s' for removal" %
806 result = utils.RunCmd(args)
808 raise errors.BlockDeviceError("Failed to mark device(s) as failed: %s" %
811 # it seems here we need a short delay for MD to update its
815 result = utils.RunCmd(args)
817 raise errors.BlockDeviceError("Failed to remove device(s) from array:"
818 " %s" % result.output)
819 result = utils.RunCmd(["mdadm", "--grow", "--force", self.dev_path,
822 raise errors.BlockDeviceError("Can't shrink md array: %s" %
824 for dev in orig_devs:
825 self._children.remove(dev)
828 """Return the status of the device.
832 if self.minor is None:
833 retval = self.STATUS_UNKNOWN
835 retval = self.STATUS_ONLINE
838 def _SetFromMinor(self, minor):
839 """Set our parameters based on the given minor.
841 This sets our minor variable and our dev_path.
845 self.dev_path = "/dev/md%d" % minor
848 """Assemble the MD device.
850 At this point we should have:
851 - list of children devices
855 result = super(MDRaid1, self).Assemble()
858 md_list = self._GetUsedDevs()
859 for minor in md_list:
860 info = self._GetDevInfo(minor)
861 if info and info["uuid"] == self.unique_id:
862 self._SetFromMinor(minor)
863 logger.Info("MD array %s already started" % str(self))
865 free_minor = self._FindUnusedMinor()
866 result = utils.RunCmd(["mdadm", "-A", "--auto=yes", "--uuid",
867 self.unique_id, "/dev/md%d" % free_minor] +
868 [bdev.dev_path for bdev in self._children])
870 logger.Error("Can't assemble MD array: %s: %s" %
871 (result.fail_reason, result.output))
874 self.minor = free_minor
875 return not result.failed
878 """Tear down the MD array.
880 This does a 'mdadm --stop' so after this command, the array is no
884 if self.minor is None and not self.Attach():
885 logger.Info("MD object not attached to a device")
888 result = utils.RunCmd(["mdadm", "--stop", "/dev/md%d" % self.minor])
890 logger.Error("Can't stop MD array: %s" % result.fail_reason)
896 def SetSyncSpeed(self, kbytes):
897 """Set the maximum sync speed for the MD array.
900 result = super(MDRaid1, self).SetSyncSpeed(kbytes)
901 if self.minor is None:
902 logger.Error("MD array not attached to a device")
904 f = open("/sys/block/md%d/md/sync_speed_max" % self.minor, "w")
906 f.write("%d" % kbytes)
909 f = open("/sys/block/md%d/md/sync_speed_min" % self.minor, "w")
911 f.write("%d" % (kbytes/2))
916 def GetSyncStatus(self):
917 """Returns the sync status of the device.
920 (sync_percent, estimated_time, is_degraded, ldisk)
922 If sync_percent is None, it means all is ok
923 If estimated_time is None, it means we can't esimate
924 the time needed, otherwise it's the time left in seconds.
926 The ldisk parameter is always true for MD devices.
929 if self.minor is None and not self.Attach():
930 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
931 dev_info = self._GetDevInfo(self.minor)
932 is_clean = ("state" in dev_info and
933 len(dev_info["state"]) == 1 and
934 dev_info["state"][0] in ("clean", "active"))
935 sys_path = "/sys/block/md%s/md/" % self.minor
936 f = file(sys_path + "sync_action")
937 sync_status = f.readline().strip()
939 if sync_status == "idle":
940 return None, None, not is_clean, False
941 f = file(sys_path + "sync_completed")
942 sync_completed = f.readline().strip().split(" / ")
944 if len(sync_completed) != 2:
945 return 0, None, not is_clean, False
946 sync_done, sync_total = [float(i) for i in sync_completed]
947 sync_percent = 100.0*sync_done/sync_total
948 f = file(sys_path + "sync_speed")
949 sync_speed_k = int(f.readline().strip())
950 if sync_speed_k == 0:
953 time_est = (sync_total - sync_done) / 2 / sync_speed_k
954 return sync_percent, time_est, not is_clean, False
956 def Open(self, force=False):
957 """Make the device ready for I/O.
959 This is a no-op for the MDRaid1 device type, although we could use
960 the 2.6.18's new array_state thing.
966 """Notifies that the device will no longer be used for I/O.
968 This is a no-op for the MDRaid1 device type, but see comment for
975 class BaseDRBD(BlockDev):
978 This class contains a few bits of common functionality between the
979 0.7 and 8.x versions of DRBD.
982 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
983 r" \(api:(\d+)/proto:(\d+)\)")
985 _ST_UNCONFIGURED = "Unconfigured"
986 _ST_WFCONNECTION = "WFConnection"
987 _ST_CONNECTED = "Connected"
991 """Return data from /proc/drbd.
994 stat = open("/proc/drbd", "r")
996 data = stat.read().splitlines()
1000 raise errors.BlockDeviceError("Can't read any data from /proc/drbd")
1004 def _MassageProcData(data):
1005 """Transform the output of _GetProdData into a nicer form.
1008 a dictionary of minor: joined lines from /proc/drbd for that minor
1011 lmatch = re.compile("^ *([0-9]+):.*$")
1013 old_minor = old_line = None
1015 lresult = lmatch.match(line)
1016 if lresult is not None:
1017 if old_minor is not None:
1018 results[old_minor] = old_line
1019 old_minor = int(lresult.group(1))
1022 if old_minor is not None:
1023 old_line += " " + line.strip()
1025 if old_minor is not None:
1026 results[old_minor] = old_line
1030 def _GetVersion(cls):
1031 """Return the DRBD version.
1033 This will return a list [k_major, k_minor, k_point, api, proto].
1036 proc_data = cls._GetProcData()
1037 first_line = proc_data[0].strip()
1038 version = cls._VERSION_RE.match(first_line)
1040 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
1042 return [int(val) for val in version.groups()]
1045 def _DevPath(minor):
1046 """Return the path to a drbd device for a given minor.
1049 return "/dev/drbd%d" % minor
1052 def _GetUsedDevs(cls):
1053 """Compute the list of used DRBD devices.
1056 data = cls._GetProcData()
1059 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
1061 match = valid_line.match(line)
1064 minor = int(match.group(1))
1065 state = match.group(2)
1066 if state == cls._ST_UNCONFIGURED:
1068 used_devs[minor] = state, line
1072 def _SetFromMinor(self, minor):
1073 """Set our parameters based on the given minor.
1075 This sets our minor variable and our dev_path.
1079 self.minor = self.dev_path = None
1082 self.dev_path = self._DevPath(minor)
1085 def _CheckMetaSize(meta_device):
1086 """Check if the given meta device looks like a valid one.
1088 This currently only check the size, which must be around
1092 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1094 logger.Error("Failed to get device size: %s" % result.fail_reason)
1097 sectors = int(result.stdout)
1099 logger.Error("Invalid output from blockdev: '%s'" % result.stdout)
1101 bytes = sectors * 512
1102 if bytes < 128 * 1024 * 1024: # less than 128MiB
1103 logger.Error("Meta device too small (%.2fMib)" % (bytes / 1024 / 1024))
1105 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
1106 logger.Error("Meta device too big (%.2fMiB)" % (bytes / 1024 / 1024))
1110 def Rename(self, new_id):
1113 This is not supported for drbd devices.
1116 raise errors.ProgrammerError("Can't rename a drbd device")
1119 class DRBDev(BaseDRBD):
1120 """DRBD block device.
1122 This implements the local host part of the DRBD device, i.e. it
1123 doesn't do anything to the supposed peer. If you need a fully
1124 connected DRBD pair, you need to use this class on both hosts.
1126 The unique_id for the drbd device is the (local_ip, local_port,
1127 remote_ip, remote_port) tuple, and it must have two children: the
1128 data device and the meta_device. The meta device is checked for
1129 valid size and is zeroed on create.
1132 def __init__(self, unique_id, children):
1133 super(DRBDev, self).__init__(unique_id, children)
1134 self.major = self._DRBD_MAJOR
1135 [kmaj, kmin, kfix, api, proto] = self._GetVersion()
1136 if kmaj != 0 and kmin != 7:
1137 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
1138 " requested ganeti usage: kernel is"
1139 " %s.%s, ganeti wants 0.7" % (kmaj, kmin))
1141 if len(children) != 2:
1142 raise ValueError("Invalid configuration data %s" % str(children))
1143 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
1144 raise ValueError("Invalid configuration data %s" % str(unique_id))
1145 self._lhost, self._lport, self._rhost, self._rport = unique_id
1149 def _FindUnusedMinor(cls):
1150 """Find an unused DRBD device.
1153 data = cls._GetProcData()
1155 valid_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
1157 match = valid_line.match(line)
1159 return int(match.group(1))
1160 logger.Error("Error: no free drbd minors!")
1161 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1164 def _GetDevInfo(cls, minor):
1165 """Get details about a given DRBD minor.
1167 This return, if available, the local backing device in (major,
1168 minor) formant and the local and remote (ip, port) information.
1172 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1174 logger.Error("Can't display the drbd config: %s" % result.fail_reason)
1177 if out == "Not configured\n":
1179 for line in out.splitlines():
1180 if "local_dev" not in data:
1181 match = re.match("^Lower device: ([0-9]+):([0-9]+) .*$", line)
1183 data["local_dev"] = (int(match.group(1)), int(match.group(2)))
1185 if "meta_dev" not in data:
1186 match = re.match("^Meta device: (([0-9]+):([0-9]+)|internal).*$", line)
1188 if match.group(2) is not None and match.group(3) is not None:
1189 # matched on the major/minor
1190 data["meta_dev"] = (int(match.group(2)), int(match.group(3)))
1192 # matched on the "internal" string
1193 data["meta_dev"] = match.group(1)
1194 # in this case, no meta_index is in the output
1195 data["meta_index"] = -1
1197 if "meta_index" not in data:
1198 match = re.match("^Meta index: ([0-9]+).*$", line)
1200 data["meta_index"] = int(match.group(1))
1202 if "local_addr" not in data:
1203 match = re.match("^Local address: ([0-9.]+):([0-9]+)$", line)
1205 data["local_addr"] = (match.group(1), int(match.group(2)))
1207 if "remote_addr" not in data:
1208 match = re.match("^Remote address: ([0-9.]+):([0-9]+)$", line)
1210 data["remote_addr"] = (match.group(1), int(match.group(2)))
1214 def _MatchesLocal(self, info):
1215 """Test if our local config matches with an existing device.
1217 The parameter should be as returned from `_GetDevInfo()`. This
1218 method tests if our local backing device is the same as the one in
1219 the info parameter, in effect testing if we look like the given
1223 if not ("local_dev" in info and "meta_dev" in info and
1224 "meta_index" in info):
1227 backend = self._children[0]
1228 if backend is not None:
1229 retval = (info["local_dev"] == (backend.major, backend.minor))
1231 retval = (info["local_dev"] == (0, 0))
1232 meta = self._children[1]
1233 if meta is not None:
1234 retval = retval and (info["meta_dev"] == (meta.major, meta.minor))
1235 retval = retval and (info["meta_index"] == 0)
1237 retval = retval and (info["meta_dev"] == "internal" and
1238 info["meta_index"] == -1)
1241 def _MatchesNet(self, info):
1242 """Test if our network config matches with an existing device.
1244 The parameter should be as returned from `_GetDevInfo()`. This
1245 method tests if our network configuration is the same as the one
1246 in the info parameter, in effect testing if we look like the given
1250 if (((self._lhost is None and not ("local_addr" in info)) and
1251 (self._rhost is None and not ("remote_addr" in info)))):
1254 if self._lhost is None:
1257 if not ("local_addr" in info and
1258 "remote_addr" in info):
1261 retval = (info["local_addr"] == (self._lhost, self._lport))
1262 retval = (retval and
1263 info["remote_addr"] == (self._rhost, self._rport))
1267 def _AssembleLocal(cls, minor, backend, meta):
1268 """Configure the local part of a DRBD device.
1270 This is the first thing that must be done on an unconfigured DRBD
1271 device. And it must be done only once.
1274 if not cls._CheckMetaSize(meta):
1276 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1277 backend, meta, "0", "-e", "detach"])
1279 logger.Error("Can't attach local disk: %s" % result.output)
1280 return not result.failed
1283 def _ShutdownLocal(cls, minor):
1284 """Detach from the local device.
1286 I/Os will continue to be served from the remote device. If we
1287 don't have a remote device, this operation will fail.
1290 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1292 logger.Error("Can't detach local device: %s" % result.output)
1293 return not result.failed
1296 def _ShutdownAll(minor):
1297 """Deactivate the device.
1299 This will, of course, fail if the device is in use.
1302 result = utils.RunCmd(["drbdsetup", DRBDev._DevPath(minor), "down"])
1304 logger.Error("Can't shutdown drbd device: %s" % result.output)
1305 return not result.failed
1308 def _AssembleNet(cls, minor, net_info, protocol):
1309 """Configure the network part of the device.
1311 This operation can be, in theory, done multiple times, but there
1312 have been cases (in lab testing) in which the network part of the
1313 device had become stuck and couldn't be shut down because activity
1314 from the new peer (also stuck) triggered a timer re-init and
1315 needed remote peer interface shutdown in order to clear. So please
1316 don't change online the net config.
1319 lhost, lport, rhost, rport = net_info
1320 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "net",
1321 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport),
1324 logger.Error("Can't setup network for dbrd device: %s" %
1328 timeout = time.time() + 10
1330 while time.time() < timeout:
1331 info = cls._GetDevInfo(minor)
1332 if not "local_addr" in info or not "remote_addr" in info:
1335 if (info["local_addr"] != (lhost, lport) or
1336 info["remote_addr"] != (rhost, rport)):
1342 logger.Error("Timeout while configuring network")
1347 def _ShutdownNet(cls, minor):
1348 """Disconnect from the remote peer.
1350 This fails if we don't have a local device.
1353 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1355 logger.Error("Can't shutdown network: %s" % result.output)
1356 return not result.failed
1359 """Assemble the drbd.
1362 - if we have a local backing device, we bind to it by:
1363 - checking the list of used drbd devices
1364 - check if the local minor use of any of them is our own device
1367 - if we have a local/remote net info:
1368 - redo the local backing device step for the remote device
1369 - check if any drbd device is using the local port,
1371 - check if any remote drbd device is using the remote
1372 port, if yes abort (for now)
1374 - bind the remote net port
1378 if self.minor is not None:
1379 logger.Info("Already assembled")
1382 result = super(DRBDev, self).Assemble()
1386 minor = self._FindUnusedMinor()
1387 need_localdev_teardown = False
1388 if self._children[0]:
1389 result = self._AssembleLocal(minor, self._children[0].dev_path,
1390 self._children[1].dev_path)
1393 need_localdev_teardown = True
1394 if self._lhost and self._lport and self._rhost and self._rport:
1395 result = self._AssembleNet(minor,
1396 (self._lhost, self._lport,
1397 self._rhost, self._rport),
1400 if need_localdev_teardown:
1401 # we will ignore failures from this
1402 logger.Error("net setup failed, tearing down local device")
1403 self._ShutdownAll(minor)
1405 self._SetFromMinor(minor)
1409 """Shutdown the DRBD device.
1412 if self.minor is None and not self.Attach():
1413 logger.Info("DRBD device not attached to a device during Shutdown")
1415 if not self._ShutdownAll(self.minor):
1418 self.dev_path = None
1422 """Find a DRBD device which matches our config and attach to it.
1424 In case of partially attached (local device matches but no network
1425 setup), we perform the network attach. If successful, we re-test
1426 the attach if can return success.
1429 for minor in self._GetUsedDevs():
1430 info = self._GetDevInfo(minor)
1431 match_l = self._MatchesLocal(info)
1432 match_r = self._MatchesNet(info)
1433 if match_l and match_r:
1435 if match_l and not match_r and "local_addr" not in info:
1436 res_r = self._AssembleNet(minor,
1437 (self._lhost, self._lport,
1438 self._rhost, self._rport),
1440 if res_r and self._MatchesNet(self._GetDevInfo(minor)):
1445 self._SetFromMinor(minor)
1446 return minor is not None
1448 def Open(self, force=False):
1449 """Make the local state primary.
1451 If the 'force' parameter is given, the '--do-what-I-say' parameter
1452 is given. Since this is a pottentialy dangerous operation, the
1453 force flag should be only given after creation, when it actually
1457 if self.minor is None and not self.Attach():
1458 logger.Error("DRBD cannot attach to a device during open")
1460 cmd = ["drbdsetup", self.dev_path, "primary"]
1462 cmd.append("--do-what-I-say")
1463 result = utils.RunCmd(cmd)
1465 msg = ("Can't make drbd device primary: %s" % result.output)
1467 raise errors.BlockDeviceError(msg)
1470 """Make the local state secondary.
1472 This will, of course, fail if the device is in use.
1475 if self.minor is None and not self.Attach():
1476 logger.Info("Instance not attached to a device")
1477 raise errors.BlockDeviceError("Can't find device")
1478 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1480 msg = ("Can't switch drbd device to"
1481 " secondary: %s" % result.output)
1483 raise errors.BlockDeviceError(msg)
1485 def SetSyncSpeed(self, kbytes):
1486 """Set the speed of the DRBD syncer.
1489 children_result = super(DRBDev, self).SetSyncSpeed(kbytes)
1490 if self.minor is None:
1491 logger.Info("Instance not attached to a device")
1493 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1496 logger.Error("Can't change syncer rate: %s " % result.fail_reason)
1497 return not result.failed and children_result
1499 def GetSyncStatus(self):
1500 """Returns the sync status of the device.
1503 (sync_percent, estimated_time, is_degraded, ldisk)
1505 If sync_percent is None, it means all is ok
1506 If estimated_time is None, it means we can't esimate
1507 the time needed, otherwise it's the time left in seconds.
1509 The ldisk parameter will be returned as True, since the DRBD7
1510 devices have not been converted.
1513 if self.minor is None and not self.Attach():
1514 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1515 proc_info = self._MassageProcData(self._GetProcData())
1516 if self.minor not in proc_info:
1517 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1519 line = proc_info[self.minor]
1520 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
1521 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
1523 sync_percent = float(match.group(1))
1524 hours = int(match.group(2))
1525 minutes = int(match.group(3))
1526 seconds = int(match.group(4))
1527 est_time = hours * 3600 + minutes * 60 + seconds
1531 match = re.match("^ *[0-9]+: cs:([^ ]+).*$", line)
1533 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
1535 client_state = match.group(1)
1536 is_degraded = client_state != "Connected"
1537 return sync_percent, est_time, is_degraded, False
1539 def GetStatus(self):
1540 """Compute the status of the DRBD device
1542 Note that DRBD devices don't have the STATUS_EXISTING state.
1545 if self.minor is None and not self.Attach():
1546 return self.STATUS_UNKNOWN
1548 data = self._GetProcData()
1549 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
1552 mresult = match.match(line)
1556 logger.Error("Can't find myself!")
1557 return self.STATUS_UNKNOWN
1559 state = mresult.group(2)
1560 if state == "Primary":
1561 result = self.STATUS_ONLINE
1563 result = self.STATUS_STANDBY
1568 def _ZeroDevice(device):
1571 This writes until we get ENOSPC.
1574 f = open(device, "w")
1575 buf = "\0" * 1048576
1579 except IOError, err:
1580 if err.errno != errno.ENOSPC:
1584 def Create(cls, unique_id, children, size):
1585 """Create a new DRBD device.
1587 Since DRBD devices are not created per se, just assembled, this
1588 function just zeroes the meta device.
1591 if len(children) != 2:
1592 raise errors.ProgrammerError("Invalid setup for the drbd device")
1595 if not meta.Attach():
1596 raise errors.BlockDeviceError("Can't attach to meta device")
1597 if not cls._CheckMetaSize(meta.dev_path):
1598 raise errors.BlockDeviceError("Invalid meta device")
1599 logger.Info("Started zeroing device %s" % meta.dev_path)
1600 cls._ZeroDevice(meta.dev_path)
1601 logger.Info("Done zeroing device %s" % meta.dev_path)
1602 return cls(unique_id, children)
1605 """Stub remove for DRBD devices.
1608 return self.Shutdown()
1611 class DRBD8(BaseDRBD):
1612 """DRBD v8.x block device.
1614 This implements the local host part of the DRBD device, i.e. it
1615 doesn't do anything to the supposed peer. If you need a fully
1616 connected DRBD pair, you need to use this class on both hosts.
1618 The unique_id for the drbd device is the (local_ip, local_port,
1619 remote_ip, remote_port) tuple, and it must have two children: the
1620 data device and the meta_device. The meta device is checked for
1621 valid size and is zeroed on create.
1627 def __init__(self, unique_id, children):
1628 if children and children.count(None) > 0:
1630 super(DRBD8, self).__init__(unique_id, children)
1631 self.major = self._DRBD_MAJOR
1632 [kmaj, kmin, kfix, api, proto] = self._GetVersion()
1634 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
1635 " requested ganeti usage: kernel is"
1636 " %s.%s, ganeti wants 8.x" % (kmaj, kmin))
1638 if len(children) not in (0, 2):
1639 raise ValueError("Invalid configuration data %s" % str(children))
1640 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
1641 raise ValueError("Invalid configuration data %s" % str(unique_id))
1642 self._lhost, self._lport, self._rhost, self._rport = unique_id
1646 def _InitMeta(cls, minor, dev_path):
1647 """Initialize a meta device.
1649 This will not work if the given minor is in use.
1652 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
1653 "v08", dev_path, "0", "create-md"])
1655 raise errors.BlockDeviceError("Can't initialize meta device: %s" %
1659 def _FindUnusedMinor(cls):
1660 """Find an unused DRBD device.
1662 This is specific to 8.x as the minors are allocated dynamically,
1663 so non-existing numbers up to a max minor count are actually free.
1666 data = cls._GetProcData()
1668 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
1669 used_line = re.compile("^ *([0-9]+): cs:")
1672 match = unused_line.match(line)
1674 return int(match.group(1))
1675 match = used_line.match(line)
1677 minor = int(match.group(1))
1678 highest = max(highest, minor)
1679 if highest is None: # there are no minors in use at all
1681 if highest >= cls._MAX_MINORS:
1682 logger.Error("Error: no free drbd minors!")
1683 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1687 def _IsValidMeta(cls, meta_device):
1688 """Check if the given meta device looks like a valid one.
1691 minor = cls._FindUnusedMinor()
1692 minor_path = cls._DevPath(minor)
1693 result = utils.RunCmd(["drbdmeta", minor_path,
1694 "v08", meta_device, "0",
1697 logger.Error("Invalid meta device %s: %s" % (meta_device, result.output))
1702 def _GetShowParser(cls):
1703 """Return a parser for `drbd show` output.
1705 This will either create or return an already-create parser for the
1706 output of the command `drbd show`.
1709 if cls._PARSE_SHOW is not None:
1710 return cls._PARSE_SHOW
1713 lbrace = pyp.Literal("{").suppress()
1714 rbrace = pyp.Literal("}").suppress()
1715 semi = pyp.Literal(";").suppress()
1716 # this also converts the value to an int
1717 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
1719 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
1720 defa = pyp.Literal("_is_default").suppress()
1721 dbl_quote = pyp.Literal('"').suppress()
1723 keyword = pyp.Word(pyp.alphanums + '-')
1726 value = pyp.Word(pyp.alphanums + '_-/.:')
1727 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
1728 addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() +
1730 # meta device, extended syntax
1731 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
1732 number + pyp.Word(']').suppress())
1735 stmt = (~rbrace + keyword + ~lbrace +
1736 (addr_port ^ value ^ quoted ^ meta_value) +
1737 pyp.Optional(defa) + semi +
1738 pyp.Optional(pyp.restOfLine).suppress())
1741 section_name = pyp.Word(pyp.alphas + '_')
1742 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
1744 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
1747 cls._PARSE_SHOW = bnf
1752 def _GetShowData(cls, minor):
1753 """Return the `drbdsetup show` data for a minor.
1756 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1758 logger.Error("Can't display the drbd config: %s" % result.fail_reason)
1760 return result.stdout
1763 def _GetDevInfo(cls, out):
1764 """Parse details about a given DRBD minor.
1766 This return, if available, the local backing device (as a path)
1767 and the local and remote (ip, port) information from a string
1768 containing the output of the `drbdsetup show` command as returned
1776 bnf = cls._GetShowParser()
1780 results = bnf.parseString(out)
1781 except pyp.ParseException, err:
1782 raise errors.BlockDeviceError("Can't parse drbdsetup show output: %s" %
1785 # and massage the results into our desired format
1786 for section in results:
1788 if sname == "_this_host":
1789 for lst in section[1:]:
1790 if lst[0] == "disk":
1791 data["local_dev"] = lst[1]
1792 elif lst[0] == "meta-disk":
1793 data["meta_dev"] = lst[1]
1794 data["meta_index"] = lst[2]
1795 elif lst[0] == "address":
1796 data["local_addr"] = tuple(lst[1:])
1797 elif sname == "_remote_host":
1798 for lst in section[1:]:
1799 if lst[0] == "address":
1800 data["remote_addr"] = tuple(lst[1:])
1803 def _MatchesLocal(self, info):
1804 """Test if our local config matches with an existing device.
1806 The parameter should be as returned from `_GetDevInfo()`. This
1807 method tests if our local backing device is the same as the one in
1808 the info parameter, in effect testing if we look like the given
1813 backend, meta = self._children
1815 backend = meta = None
1817 if backend is not None:
1818 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
1820 retval = ("local_dev" not in info)
1822 if meta is not None:
1823 retval = retval and ("meta_dev" in info and
1824 info["meta_dev"] == meta.dev_path)
1825 retval = retval and ("meta_index" in info and
1826 info["meta_index"] == 0)
1828 retval = retval and ("meta_dev" not in info and
1829 "meta_index" not in info)
1832 def _MatchesNet(self, info):
1833 """Test if our network config matches with an existing device.
1835 The parameter should be as returned from `_GetDevInfo()`. This
1836 method tests if our network configuration is the same as the one
1837 in the info parameter, in effect testing if we look like the given
1841 if (((self._lhost is None and not ("local_addr" in info)) and
1842 (self._rhost is None and not ("remote_addr" in info)))):
1845 if self._lhost is None:
1848 if not ("local_addr" in info and
1849 "remote_addr" in info):
1852 retval = (info["local_addr"] == (self._lhost, self._lport))
1853 retval = (retval and
1854 info["remote_addr"] == (self._rhost, self._rport))
1858 def _AssembleLocal(cls, minor, backend, meta):
1859 """Configure the local part of a DRBD device.
1861 This is the first thing that must be done on an unconfigured DRBD
1862 device. And it must be done only once.
1865 if not cls._IsValidMeta(meta):
1867 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1868 backend, meta, "0", "-e", "detach",
1871 logger.Error("Can't attach local disk: %s" % result.output)
1872 return not result.failed
1875 def _AssembleNet(cls, minor, net_info, protocol,
1876 dual_pri=False, hmac=None, secret=None):
1877 """Configure the network part of the device.
1880 lhost, lport, rhost, rport = net_info
1881 if None in net_info:
1882 # we don't want network connection and actually want to make
1884 return cls._ShutdownNet(minor)
1886 args = ["drbdsetup", cls._DevPath(minor), "net",
1887 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
1888 "-A", "discard-zero-changes",
1894 args.extend(["-a", hmac, "-x", secret])
1895 result = utils.RunCmd(args)
1897 logger.Error("Can't setup network for dbrd device: %s" %
1901 timeout = time.time() + 10
1903 while time.time() < timeout:
1904 info = cls._GetDevInfo(cls._GetShowData(minor))
1905 if not "local_addr" in info or not "remote_addr" in info:
1908 if (info["local_addr"] != (lhost, lport) or
1909 info["remote_addr"] != (rhost, rport)):
1915 logger.Error("Timeout while configuring network")
1919 def AddChildren(self, devices):
1920 """Add a disk to the DRBD device.
1923 if self.minor is None:
1924 raise errors.BlockDeviceError("Can't attach to dbrd8 during AddChildren")
1925 if len(devices) != 2:
1926 raise errors.BlockDeviceError("Need two devices for AddChildren")
1927 info = self._GetDevInfo(self._GetShowData(self.minor))
1928 if "local_dev" in info:
1929 raise errors.BlockDeviceError("DRBD8 already attached to a local disk")
1930 backend, meta = devices
1931 if backend.dev_path is None or meta.dev_path is None:
1932 raise errors.BlockDeviceError("Children not ready during AddChildren")
1935 if not self._CheckMetaSize(meta.dev_path):
1936 raise errors.BlockDeviceError("Invalid meta device size")
1937 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1938 if not self._IsValidMeta(meta.dev_path):
1939 raise errors.BlockDeviceError("Cannot initalize meta device")
1941 if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path):
1942 raise errors.BlockDeviceError("Can't attach to local storage")
1943 self._children = devices
1945 def RemoveChildren(self, devices):
1946 """Detach the drbd device from local storage.
1949 if self.minor is None:
1950 raise errors.BlockDeviceError("Can't attach to drbd8 during"
1952 # early return if we don't actually have backing storage
1953 info = self._GetDevInfo(self._GetShowData(self.minor))
1954 if "local_dev" not in info:
1956 if len(self._children) != 2:
1957 raise errors.BlockDeviceError("We don't have two children: %s" %
1959 if self._children.count(None) == 2: # we don't actually have children :)
1960 logger.Error("Requested detach while detached")
1962 if len(devices) != 2:
1963 raise errors.BlockDeviceError("We need two children in RemoveChildren")
1964 for child, dev in zip(self._children, devices):
1965 if dev != child.dev_path:
1966 raise errors.BlockDeviceError("Mismatch in local storage"
1967 " (%s != %s) in RemoveChildren" %
1968 (dev, child.dev_path))
1970 if not self._ShutdownLocal(self.minor):
1971 raise errors.BlockDeviceError("Can't detach from local storage")
1974 def SetSyncSpeed(self, kbytes):
1975 """Set the speed of the DRBD syncer.
1978 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1979 if self.minor is None:
1980 logger.Info("Instance not attached to a device")
1982 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1985 logger.Error("Can't change syncer rate: %s " % result.fail_reason)
1986 return not result.failed and children_result
1988 def GetSyncStatus(self):
1989 """Returns the sync status of the device.
1992 (sync_percent, estimated_time, is_degraded)
1994 If sync_percent is None, it means all is ok
1995 If estimated_time is None, it means we can't esimate
1996 the time needed, otherwise it's the time left in seconds.
1999 We set the is_degraded parameter to True on two conditions:
2000 network not connected or local disk missing.
2002 We compute the ldisk parameter based on wheter we have a local
2006 if self.minor is None and not self.Attach():
2007 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
2008 proc_info = self._MassageProcData(self._GetProcData())
2009 if self.minor not in proc_info:
2010 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
2012 line = proc_info[self.minor]
2013 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
2014 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
2016 sync_percent = float(match.group(1))
2017 hours = int(match.group(2))
2018 minutes = int(match.group(3))
2019 seconds = int(match.group(4))
2020 est_time = hours * 3600 + minutes * 60 + seconds
2024 match = re.match("^ *\d+: cs:(\w+).*ds:(\w+)/(\w+).*$", line)
2026 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
2028 client_state = match.group(1)
2029 local_disk_state = match.group(2)
2030 ldisk = local_disk_state != "UpToDate"
2031 is_degraded = client_state != "Connected"
2032 return sync_percent, est_time, is_degraded or ldisk, ldisk
2034 def GetStatus(self):
2035 """Compute the status of the DRBD device
2037 Note that DRBD devices don't have the STATUS_EXISTING state.
2040 if self.minor is None and not self.Attach():
2041 return self.STATUS_UNKNOWN
2043 data = self._GetProcData()
2044 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
2047 mresult = match.match(line)
2051 logger.Error("Can't find myself!")
2052 return self.STATUS_UNKNOWN
2054 state = mresult.group(2)
2055 if state == "Primary":
2056 result = self.STATUS_ONLINE
2058 result = self.STATUS_STANDBY
2062 def Open(self, force=False):
2063 """Make the local state primary.
2065 If the 'force' parameter is given, the '--do-what-I-say' parameter
2066 is given. Since this is a pottentialy dangerous operation, the
2067 force flag should be only given after creation, when it actually
2071 if self.minor is None and not self.Attach():
2072 logger.Error("DRBD cannot attach to a device during open")
2074 cmd = ["drbdsetup", self.dev_path, "primary"]
2077 result = utils.RunCmd(cmd)
2079 msg = ("Can't make drbd device primary: %s" % result.output)
2081 raise errors.BlockDeviceError(msg)
2084 """Make the local state secondary.
2086 This will, of course, fail if the device is in use.
2089 if self.minor is None and not self.Attach():
2090 logger.Info("Instance not attached to a device")
2091 raise errors.BlockDeviceError("Can't find device")
2092 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
2094 msg = ("Can't switch drbd device to"
2095 " secondary: %s" % result.output)
2097 raise errors.BlockDeviceError(msg)
2100 """Find a DRBD device which matches our config and attach to it.
2102 In case of partially attached (local device matches but no network
2103 setup), we perform the network attach. If successful, we re-test
2104 the attach if can return success.
2107 for minor in self._GetUsedDevs():
2108 info = self._GetDevInfo(self._GetShowData(minor))
2109 match_l = self._MatchesLocal(info)
2110 match_r = self._MatchesNet(info)
2111 if match_l and match_r:
2113 if match_l and not match_r and "local_addr" not in info:
2114 res_r = self._AssembleNet(minor,
2115 (self._lhost, self._lport,
2116 self._rhost, self._rport),
2119 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
2121 # the weakest case: we find something that is only net attached
2122 # even though we were passed some children at init time
2123 if match_r and "local_dev" not in info:
2125 if match_l and not match_r and "local_addr" in info:
2126 # strange case - the device network part points to somewhere
2127 # else, even though its local storage is ours; as we own the
2128 # drbd space, we try to disconnect from the remote peer and
2129 # reconnect to our correct one
2130 if not self._ShutdownNet(minor):
2131 raise errors.BlockDeviceError("Device has correct local storage,"
2132 " wrong remote peer and is unable to"
2133 " disconnect in order to attach to"
2134 " the correct peer")
2135 # note: _AssembleNet also handles the case when we don't want
2136 # local storage (i.e. one or more of the _[lr](host|port) is
2138 if (self._AssembleNet(minor, (self._lhost, self._lport,
2139 self._rhost, self._rport), "C") and
2140 self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))):
2146 self._SetFromMinor(minor)
2147 return minor is not None
2150 """Assemble the drbd.
2153 - if we have a local backing device, we bind to it by:
2154 - checking the list of used drbd devices
2155 - check if the local minor use of any of them is our own device
2158 - if we have a local/remote net info:
2159 - redo the local backing device step for the remote device
2160 - check if any drbd device is using the local port,
2162 - check if any remote drbd device is using the remote
2163 port, if yes abort (for now)
2165 - bind the remote net port
2169 if self.minor is not None:
2170 logger.Info("Already assembled")
2173 result = super(DRBD8, self).Assemble()
2177 minor = self._FindUnusedMinor()
2178 need_localdev_teardown = False
2179 if self._children and self._children[0] and self._children[1]:
2180 result = self._AssembleLocal(minor, self._children[0].dev_path,
2181 self._children[1].dev_path)
2184 need_localdev_teardown = True
2185 if self._lhost and self._lport and self._rhost and self._rport:
2186 result = self._AssembleNet(minor,
2187 (self._lhost, self._lport,
2188 self._rhost, self._rport),
2191 if need_localdev_teardown:
2192 # we will ignore failures from this
2193 logger.Error("net setup failed, tearing down local device")
2194 self._ShutdownAll(minor)
2196 self._SetFromMinor(minor)
2200 def _ShutdownLocal(cls, minor):
2201 """Detach from the local device.
2203 I/Os will continue to be served from the remote device. If we
2204 don't have a remote device, this operation will fail.
2207 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
2209 logger.Error("Can't detach local device: %s" % result.output)
2210 return not result.failed
2213 def _ShutdownNet(cls, minor):
2214 """Disconnect from the remote peer.
2216 This fails if we don't have a local device.
2219 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
2221 logger.Error("Can't shutdown network: %s" % result.output)
2222 return not result.failed
2225 def _ShutdownAll(cls, minor):
2226 """Deactivate the device.
2228 This will, of course, fail if the device is in use.
2231 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
2233 logger.Error("Can't shutdown drbd device: %s" % result.output)
2234 return not result.failed
2237 """Shutdown the DRBD device.
2240 if self.minor is None and not self.Attach():
2241 logger.Info("DRBD device not attached to a device during Shutdown")
2243 if not self._ShutdownAll(self.minor):
2246 self.dev_path = None
2250 """Stub remove for DRBD devices.
2253 return self.Shutdown()
2256 def Create(cls, unique_id, children, size):
2257 """Create a new DRBD8 device.
2259 Since DRBD devices are not created per se, just assembled, this
2260 function only initializes the metadata.
2263 if len(children) != 2:
2264 raise errors.ProgrammerError("Invalid setup for the drbd device")
2267 if not meta.Attach():
2268 raise errors.BlockDeviceError("Can't attach to meta device")
2269 if not cls._CheckMetaSize(meta.dev_path):
2270 raise errors.BlockDeviceError("Invalid meta device size")
2271 cls._InitMeta(cls._FindUnusedMinor(), meta.dev_path)
2272 if not cls._IsValidMeta(meta.dev_path):
2273 raise errors.BlockDeviceError("Cannot initalize meta device")
2274 return cls(unique_id, children)
2278 constants.LD_LV: LogicalVolume,
2279 constants.LD_MD_R1: MDRaid1,
2280 constants.LD_DRBD7: DRBDev,
2281 constants.LD_DRBD8: DRBD8,
2285 def FindDevice(dev_type, unique_id, children):
2286 """Search for an existing, assembled device.
2288 This will succeed only if the device exists and is assembled, but it
2289 does not do any actions in order to activate the device.
2292 if dev_type not in DEV_MAP:
2293 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2294 device = DEV_MAP[dev_type](unique_id, children)
2295 if not device.Attach():
2300 def AttachOrAssemble(dev_type, unique_id, children):
2301 """Try to attach or assemble an existing device.
2303 This will attach to an existing assembled device or will assemble
2304 the device, as needed, to bring it fully up.
2307 if dev_type not in DEV_MAP:
2308 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2309 device = DEV_MAP[dev_type](unique_id, children)
2310 if not device.Attach():
2312 if not device.Attach():
2313 raise errors.BlockDeviceError("Can't find a valid block device for"
2315 (dev_type, unique_id, children))
2319 def Create(dev_type, unique_id, children, size):
2323 if dev_type not in DEV_MAP:
2324 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2325 device = DEV_MAP[dev_type].Create(unique_id, children, size)