4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
29 from ganeti import utils
30 from ganeti import logger
31 from ganeti import errors
32 from ganeti import constants
35 class BlockDev(object):
36 """Block device abstract class.
38 A block device can be in the following states:
39 - not existing on the system, and by `Create()` it goes into:
40 - existing but not setup/not active, and by `Assemble()` goes into:
41 - active read-write and by `Open()` it goes into
42 - online (=used, or ready for use)
44 A device can also be online but read-only, however we are not using
45 the readonly state (MD and LV have it, if needed in the future)
46 and we are usually looking at this like at a stack, so it's easier
47 to conceptualise the transition from not-existing to online and back
50 The many different states of the device are due to the fact that we
51 need to cover many device types:
52 - logical volumes are created, lvchange -a y $lv, and used
53 - md arrays are created or assembled and used
54 - drbd devices are attached to a local disk/remote peer and made primary
56 The status of the device can be examined by `GetStatus()`, which
57 returns a numerical value, depending on the position in the
58 transition stack of the device.
60 A block device is identified by three items:
61 - the /dev path of the device (dynamic)
62 - a unique ID of the device (static)
63 - it's major/minor pair (dynamic)
65 Not all devices implement both the first two as distinct items. LVM
66 logical volumes have their unique ID (the pair volume group, logical
67 volume name) in a 1-to-1 relation to the dev path. For MD devices,
68 the /dev path is dynamic and the unique ID is the UUID generated at
69 array creation plus the slave list. For DRBD devices, the /dev path
70 is again dynamic and the unique id is the pair (host1, dev1),
73 You can get to a device in two ways:
74 - creating the (real) device, which returns you
75 an attached instance (lvcreate, mdadm --create)
76 - attaching of a python instance to an existing (real) device
78 The second point, the attachement to a device, is different
79 depending on whether the device is assembled or not. At init() time,
80 we search for a device with the same unique_id as us. If found,
81 good. It also means that the device is already assembled. If not,
82 after assembly we'll have our correct major/minor.
91 STATUS_UNKNOWN: "unknown",
92 STATUS_EXISTING: "existing",
93 STATUS_STANDBY: "ready for use",
94 STATUS_ONLINE: "online",
97 def __init__(self, unique_id, children):
98 self._children = children
100 self.unique_id = unique_id
105 """Assemble the device from its components.
107 If this is a plain block device (e.g. LVM) than assemble does
108 nothing, as the LVM has no children and we don't put logical
111 One guarantee is that after the device has been assembled, it
112 knows its major/minor numbers. This allows other devices (usually
113 parents) to probe correctly for their children.
117 for child in self._children:
118 if not isinstance(child, BlockDev):
119 raise TypeError("Invalid child passed of type '%s'" % type(child))
122 status = status and child.Assemble()
128 except errors.BlockDeviceError:
129 for child in self._children:
134 for child in self._children:
139 """Find a device which matches our config and attach to it.
142 raise NotImplementedError
145 """Notifies that the device will no longer be used for I/O.
148 raise NotImplementedError
151 def Create(cls, unique_id, children, size):
152 """Create the device.
154 If the device cannot be created, it will return None
155 instead. Error messages go to the logging system.
157 Note that for some devices, the unique_id is used, and for other,
158 the children. The idea is that these two, taken together, are
159 enough for both creation and assembly (later).
162 raise NotImplementedError
165 """Remove this device.
167 This makes sense only for some of the device types: LV and to a
168 lesser degree, md devices. Also note that if the device can't
169 attach, the removal can't be completed.
172 raise NotImplementedError
174 def Rename(self, new_id):
175 """Rename this device.
177 This may or may not make sense for a given device type.
180 raise NotImplementedError
183 """Return the status of the device.
186 raise NotImplementedError
188 def Open(self, force=False):
189 """Make the device ready for use.
191 This makes the device ready for I/O. For now, just the DRBD
194 The force parameter signifies that if the device has any kind of
195 --force thing, it should be used, we know what we are doing.
198 raise NotImplementedError
201 """Shut down the device, freeing its children.
203 This undoes the `Assemble()` work, except for the child
204 assembling; as such, the children on the device are still
205 assembled after this call.
208 raise NotImplementedError
210 def SetSyncSpeed(self, speed):
211 """Adjust the sync speed of the mirror.
213 In case this is not a mirroring device, this is no-op.
218 for child in self._children:
219 result = result and child.SetSyncSpeed(speed)
222 def GetSyncStatus(self):
223 """Returns the sync status of the device.
225 If this device is a mirroring device, this function returns the
226 status of the mirror.
229 (sync_percent, estimated_time, is_degraded, ldisk)
231 If sync_percent is None, it means the device is not syncing.
233 If estimated_time is None, it means we can't estimate
234 the time needed, otherwise it's the time left in seconds.
236 If is_degraded is True, it means the device is missing
237 redundancy. This is usually a sign that something went wrong in
238 the device setup, if sync_percent is None.
240 The ldisk parameter represents the degradation of the local
241 data. This is only valid for some devices, the rest will always
242 return False (not degraded).
245 return None, None, False, False
247 def CombinedSyncStatus(self):
248 """Calculate the mirror status recursively for our children.
250 The return value is the same as for `GetSyncStatus()` except the
251 minimum percent and maximum time are calculated across our
255 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
257 for child in self._children:
258 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
259 if min_percent is None:
260 min_percent = c_percent
261 elif c_percent is not None:
262 min_percent = min(min_percent, c_percent)
265 elif c_time is not None:
266 max_time = max(max_time, c_time)
267 is_degraded = is_degraded or c_degraded
268 ldisk = ldisk or c_ldisk
269 return min_percent, max_time, is_degraded, ldisk
271 def SetInfo(self, text):
272 """Update metadata with info text.
274 Only supported for some device types.
277 for child in self._children:
280 def Grow(self, amount):
281 """Grow the block device.
284 amount: the amount (in mebibytes) to grow with
289 raise NotImplementedError
292 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
293 (self.__class__, self.unique_id, self._children,
294 self.major, self.minor, self.dev_path))
297 class LogicalVolume(BlockDev):
298 """Logical Volume block device.
301 def __init__(self, unique_id, children):
302 """Attaches to a LV device.
304 The unique_id is a tuple (vg_name, lv_name)
307 super(LogicalVolume, self).__init__(unique_id, children)
308 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
309 raise ValueError("Invalid configuration data %s" % str(unique_id))
310 self._vg_name, self._lv_name = unique_id
311 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
312 self._degraded = True
313 self.major = self.minor = None
317 def Create(cls, unique_id, children, size):
318 """Create a new logical volume.
321 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
322 raise ValueError("Invalid configuration data %s" % str(unique_id))
323 vg_name, lv_name = unique_id
324 pvs_info = cls.GetPVInfo(vg_name)
326 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
331 pvlist = [ pv[1] for pv in pvs_info ]
332 free_size = sum([ pv[0] for pv in pvs_info ])
334 # The size constraint should have been checked from the master before
335 # calling the create function.
337 raise errors.BlockDeviceError("Not enough free space: required %s,"
338 " available %s" % (size, free_size))
339 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
342 raise errors.BlockDeviceError("%s - %s" % (result.fail_reason,
344 return LogicalVolume(unique_id, children)
347 def GetPVInfo(vg_name):
348 """Get the free space info for PVs in a volume group.
351 vg_name: the volume group name
354 list of (free_space, name) with free_space in mebibytes
357 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
358 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
360 result = utils.RunCmd(command)
362 logger.Error("Can't get the PV information: %s - %s" %
363 (result.fail_reason, result.output))
366 for line in result.stdout.splitlines():
367 fields = line.strip().split(':')
369 logger.Error("Can't parse pvs output: line '%s'" % line)
371 # skip over pvs from another vg or ones which are not allocatable
372 if fields[1] != vg_name or fields[3][0] != 'a':
374 data.append((float(fields[2]), fields[0]))
379 """Remove this logical volume.
382 if not self.minor and not self.Attach():
383 # the LV does not exist
385 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
386 (self._vg_name, self._lv_name)])
388 logger.Error("Can't lvremove: %s - %s" %
389 (result.fail_reason, result.output))
391 return not result.failed
393 def Rename(self, new_id):
394 """Rename this logical volume.
397 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
398 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
399 new_vg, new_name = new_id
400 if new_vg != self._vg_name:
401 raise errors.ProgrammerError("Can't move a logical volume across"
402 " volume groups (from %s to to %s)" %
403 (self._vg_name, new_vg))
404 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
406 raise errors.BlockDeviceError("Failed to rename the logical volume: %s" %
408 self._lv_name = new_name
409 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
412 """Attach to an existing LV.
414 This method will try to see if an existing and active LV exists
415 which matches our name. If so, its major/minor will be
419 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
420 "-olv_attr,lv_kernel_major,lv_kernel_minor",
423 logger.Error("Can't find LV %s: %s, %s" %
424 (self.dev_path, result.fail_reason, result.output))
426 out = result.stdout.strip().rstrip(',')
429 logger.Error("Can't parse LVS output, len(%s) != 3" % str(out))
432 status, major, minor = out[:3]
434 logger.Error("lvs lv_attr is not 6 characters (%s)" % status)
440 except ValueError, err:
441 logger.Error("lvs major/minor cannot be parsed: %s" % str(err))
445 self._degraded = status[0] == 'v' # virtual volume, i.e. doesn't backing
450 """Assemble the device.
452 We alway run `lvchange -ay` on the LV to ensure it's active before
453 use, as there were cases when xenvg was not active after boot
454 (also possibly after disk issues).
457 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
459 logger.Error("Can't activate lv %s: %s" % (self.dev_path, result.output))
460 return not result.failed
463 """Shutdown the device.
465 This is a no-op for the LV device type, as we don't deactivate the
472 """Return the status of the device.
474 Logical volumes will can be in all four states, although we don't
475 deactivate (lvchange -an) them when shutdown, so STATUS_EXISTING
476 should not be seen for our devices.
479 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
481 logger.Error("Can't display lv: %s - %s" %
482 (result.fail_reason, result.output))
483 return self.STATUS_UNKNOWN
484 out = result.stdout.strip()
485 # format: type/permissions/alloc/fixed_minor/state/open
487 return self.STATUS_UNKNOWN
488 #writable = (out[1] == "w")
489 active = (out[4] == "a")
490 online = (out[5] == "o")
492 retval = self.STATUS_ONLINE
494 retval = self.STATUS_STANDBY
496 retval = self.STATUS_EXISTING
500 def GetSyncStatus(self):
501 """Returns the sync status of the device.
503 If this device is a mirroring device, this function returns the
504 status of the mirror.
507 (sync_percent, estimated_time, is_degraded, ldisk)
509 For logical volumes, sync_percent and estimated_time are always
510 None (no recovery in progress, as we don't handle the mirrored LV
511 case). The is_degraded parameter is the inverse of the ldisk
514 For the ldisk parameter, we check if the logical volume has the
515 'virtual' type, which means it's not backed by existing storage
516 anymore (read from it return I/O error). This happens after a
517 physical disk failure and subsequent 'vgreduce --removemissing' on
520 The status was already read in Attach, so we just return it.
523 return None, None, self._degraded, self._degraded
525 def Open(self, force=False):
526 """Make the device ready for I/O.
528 This is a no-op for the LV device type.
534 """Notifies that the device will no longer be used for I/O.
536 This is a no-op for the LV device type.
541 def Snapshot(self, size):
542 """Create a snapshot copy of an lvm block device.
545 snap_name = self._lv_name + ".snap"
547 # remove existing snapshot if found
548 snap = LogicalVolume((self._vg_name, snap_name), None)
551 pvs_info = self.GetPVInfo(self._vg_name)
553 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
557 free_size, pv_name = pvs_info[0]
559 raise errors.BlockDeviceError("Not enough free space: required %s,"
560 " available %s" % (size, free_size))
562 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
563 "-n%s" % snap_name, self.dev_path])
565 raise errors.BlockDeviceError("command: %s error: %s - %s" %
566 (result.cmd, result.fail_reason,
571 def SetInfo(self, text):
572 """Update metadata with info text.
575 BlockDev.SetInfo(self, text)
577 # Replace invalid characters
578 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
579 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
581 # Only up to 128 characters are allowed
584 result = utils.RunCmd(["lvchange", "--addtag", text,
587 raise errors.BlockDeviceError("Command: %s error: %s - %s" %
588 (result.cmd, result.fail_reason,
590 def Grow(self, amount):
591 """Grow the logical volume.
594 # we try multiple algorithms since the 'best' ones might not have
595 # space available in the right place, but later ones might (since
596 # they have less constraints); also note that only recent LVM
598 for alloc_policy in "contiguous", "cling", "normal":
599 result = utils.RunCmd(["lvextend", "--alloc", alloc_policy,
600 "-L", "+%dm" % amount, self.dev_path])
601 if not result.failed:
603 raise errors.BlockDeviceError("Can't grow LV %s: %s" %
604 (self.dev_path, result.output))
607 class MDRaid1(BlockDev):
608 """raid1 device implemented via md.
611 def __init__(self, unique_id, children):
612 super(MDRaid1, self).__init__(unique_id, children)
617 """Find an array which matches our config and attach to it.
619 This tries to find a MD array which has the same UUID as our own.
622 minor = self._FindMDByUUID(self.unique_id)
623 if minor is not None:
624 self._SetFromMinor(minor)
629 return (minor is not None)
633 """Compute the list of in-use MD devices.
635 It doesn't matter if the used device have other raid level, just
636 that they are in use.
639 mdstat = open("/proc/mdstat", "r")
640 data = mdstat.readlines()
644 valid_line = re.compile("^md([0-9]+) : .*$")
646 match = valid_line.match(line)
648 md_no = int(match.group(1))
649 used_md[md_no] = line
654 def _GetDevInfo(minor):
655 """Get info about a MD device.
657 Currently only uuid is returned.
660 result = utils.RunCmd(["mdadm", "-D", "/dev/md%d" % minor])
662 logger.Error("Can't display md: %s - %s" %
663 (result.fail_reason, result.output))
666 for line in result.stdout.splitlines():
668 kv = line.split(" : ", 1)
671 retval["uuid"] = kv[1].split()[0]
672 elif kv[0] == "State":
673 retval["state"] = kv[1].split(", ")
677 def _FindUnusedMinor():
678 """Compute an unused MD minor.
680 This code assumes that there are 256 minors only.
683 used_md = MDRaid1._GetUsedDevs()
690 logger.Error("Critical: Out of md minor numbers.")
691 raise errors.BlockDeviceError("Can't find a free MD minor")
695 def _FindMDByUUID(cls, uuid):
696 """Find the minor of an MD array with a given UUID.
699 md_list = cls._GetUsedDevs()
700 for minor in md_list:
701 info = cls._GetDevInfo(minor)
702 if info and info["uuid"] == uuid:
707 def _ZeroSuperblock(dev_path):
708 """Zero the possible locations for an MD superblock.
710 The zero-ing can't be done via ``mdadm --zero-superblock`` as that
711 fails in versions 2.x with the same error code as non-writable
714 The superblocks are located at (negative values are relative to
715 the end of the block device):
716 - -128k to end for version 0.90 superblock
717 - -8k to -12k for version 1.0 superblock (included in the above)
718 - 0k to 4k for version 1.1 superblock
719 - 4k to 8k for version 1.2 superblock
721 To cover all situations, the zero-ing will be:
725 As such, the minimum device size must be 128k, otherwise we'll get
728 Note that this function depends on the fact that one can open,
729 read and write block devices normally.
732 overwrite_size = 128 * 1024
733 empty_buf = '\0' * overwrite_size
734 fd = open(dev_path, "r+")
740 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
741 fd.seek(-overwrite_size, 2)
745 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
750 def Create(cls, unique_id, children, size):
751 """Create a new MD raid1 array.
754 if not isinstance(children, (tuple, list)):
755 raise ValueError("Invalid setup data for MDRaid1 dev: %s" %
758 if not isinstance(i, BlockDev):
759 raise ValueError("Invalid member in MDRaid1 dev: %s" % type(i))
762 cls._ZeroSuperblock(i.dev_path)
763 except EnvironmentError, err:
764 logger.Error("Can't zero superblock for %s: %s" %
765 (i.dev_path, str(err)))
767 minor = cls._FindUnusedMinor()
768 result = utils.RunCmd(["mdadm", "--create", "/dev/md%d" % minor,
769 "--auto=yes", "--force", "-l1",
770 "-n%d" % len(children)] +
771 [dev.dev_path for dev in children])
774 logger.Error("Can't create md: %s: %s" % (result.fail_reason,
777 info = cls._GetDevInfo(minor)
778 if not info or not "uuid" in info:
779 logger.Error("Wrong information returned from mdadm -D: %s" % str(info))
781 return MDRaid1(info["uuid"], children)
784 """Stub remove function for MD RAID 1 arrays.
786 We don't remove the superblock right now. Mark a to do.
789 #TODO: maybe zero superblock on child devices?
790 return self.Shutdown()
792 def Rename(self, new_id):
795 This is not supported for md raid1 devices.
798 raise errors.ProgrammerError("Can't rename a md raid1 device")
800 def AddChildren(self, devices):
801 """Add new member(s) to the md raid1.
804 if self.minor is None and not self.Attach():
805 raise errors.BlockDeviceError("Can't attach to device")
807 args = ["mdadm", "-a", self.dev_path]
809 if dev.dev_path is None:
810 raise errors.BlockDeviceError("Child '%s' is not initialised" % dev)
812 args.append(dev.dev_path)
813 result = utils.RunCmd(args)
815 raise errors.BlockDeviceError("Failed to add new device to array: %s" %
817 new_len = len(self._children) + len(devices)
818 result = utils.RunCmd(["mdadm", "--grow", self.dev_path, "-n", new_len])
820 raise errors.BlockDeviceError("Can't grow md array: %s" %
822 self._children.extend(devices)
824 def RemoveChildren(self, devices):
825 """Remove member(s) from the md raid1.
828 if self.minor is None and not self.Attach():
829 raise errors.BlockDeviceError("Can't attach to device")
830 new_len = len(self._children) - len(devices)
832 raise errors.BlockDeviceError("Can't reduce to less than one child")
833 args = ["mdadm", "-f", self.dev_path]
837 for c in self._children:
838 if c.dev_path == dev:
842 raise errors.BlockDeviceError("Can't find device '%s' for removal" %
844 result = utils.RunCmd(args)
846 raise errors.BlockDeviceError("Failed to mark device(s) as failed: %s" %
849 # it seems here we need a short delay for MD to update its
853 result = utils.RunCmd(args)
855 raise errors.BlockDeviceError("Failed to remove device(s) from array:"
856 " %s" % result.output)
857 result = utils.RunCmd(["mdadm", "--grow", "--force", self.dev_path,
860 raise errors.BlockDeviceError("Can't shrink md array: %s" %
862 for dev in orig_devs:
863 self._children.remove(dev)
866 """Return the status of the device.
870 if self.minor is None:
871 retval = self.STATUS_UNKNOWN
873 retval = self.STATUS_ONLINE
876 def _SetFromMinor(self, minor):
877 """Set our parameters based on the given minor.
879 This sets our minor variable and our dev_path.
883 self.dev_path = "/dev/md%d" % minor
886 """Assemble the MD device.
888 At this point we should have:
889 - list of children devices
893 result = super(MDRaid1, self).Assemble()
896 md_list = self._GetUsedDevs()
897 for minor in md_list:
898 info = self._GetDevInfo(minor)
899 if info and info["uuid"] == self.unique_id:
900 self._SetFromMinor(minor)
901 logger.Info("MD array %s already started" % str(self))
903 free_minor = self._FindUnusedMinor()
904 result = utils.RunCmd(["mdadm", "-A", "--auto=yes", "--uuid",
905 self.unique_id, "/dev/md%d" % free_minor] +
906 [bdev.dev_path for bdev in self._children])
908 logger.Error("Can't assemble MD array: %s: %s" %
909 (result.fail_reason, result.output))
912 self.minor = free_minor
913 return not result.failed
916 """Tear down the MD array.
918 This does a 'mdadm --stop' so after this command, the array is no
922 if self.minor is None and not self.Attach():
923 logger.Info("MD object not attached to a device")
926 result = utils.RunCmd(["mdadm", "--stop", "/dev/md%d" % self.minor])
928 logger.Error("Can't stop MD array: %s - %s" %
929 (result.fail_reason, result.output))
935 def SetSyncSpeed(self, kbytes):
936 """Set the maximum sync speed for the MD array.
939 result = super(MDRaid1, self).SetSyncSpeed(kbytes)
940 if self.minor is None:
941 logger.Error("MD array not attached to a device")
943 f = open("/sys/block/md%d/md/sync_speed_max" % self.minor, "w")
945 f.write("%d" % kbytes)
948 f = open("/sys/block/md%d/md/sync_speed_min" % self.minor, "w")
950 f.write("%d" % (kbytes/2))
955 def GetSyncStatus(self):
956 """Returns the sync status of the device.
959 (sync_percent, estimated_time, is_degraded, ldisk)
961 If sync_percent is None, it means all is ok
962 If estimated_time is None, it means we can't esimate
963 the time needed, otherwise it's the time left in seconds.
965 The ldisk parameter is always true for MD devices.
968 if self.minor is None and not self.Attach():
969 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
970 dev_info = self._GetDevInfo(self.minor)
971 is_clean = ("state" in dev_info and
972 len(dev_info["state"]) == 1 and
973 dev_info["state"][0] in ("clean", "active"))
974 sys_path = "/sys/block/md%s/md/" % self.minor
975 f = file(sys_path + "sync_action")
976 sync_status = f.readline().strip()
978 if sync_status == "idle":
979 return None, None, not is_clean, False
980 f = file(sys_path + "sync_completed")
981 sync_completed = f.readline().strip().split(" / ")
983 if len(sync_completed) != 2:
984 return 0, None, not is_clean, False
985 sync_done, sync_total = [float(i) for i in sync_completed]
986 sync_percent = 100.0*sync_done/sync_total
987 f = file(sys_path + "sync_speed")
988 sync_speed_k = int(f.readline().strip())
989 if sync_speed_k == 0:
992 time_est = (sync_total - sync_done) / 2 / sync_speed_k
993 return sync_percent, time_est, not is_clean, False
995 def Open(self, force=False):
996 """Make the device ready for I/O.
998 This is a no-op for the MDRaid1 device type, although we could use
999 the 2.6.18's new array_state thing.
1005 """Notifies that the device will no longer be used for I/O.
1007 This is a no-op for the MDRaid1 device type, but see comment for
1014 class DRBD8Status(object):
1015 """A DRBD status representation class.
1017 Note that this doesn't support unconfigured devices (cs:Unconfigured).
1020 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+st:([^/]+)/(\S+)"
1021 "\s+ds:([^/]+)/(\S+)\s+.*$")
1022 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
1023 "\sfinish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
1025 def __init__(self, procline):
1026 m = self.LINE_RE.match(procline)
1028 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
1029 self.cstatus = m.group(1)
1030 self.lrole = m.group(2)
1031 self.rrole = m.group(3)
1032 self.ldisk = m.group(4)
1033 self.rdisk = m.group(5)
1035 self.is_standalone = self.cstatus == "StandAlone"
1036 self.is_wfconn = self.cstatus == "WFConnection"
1037 self.is_connected = self.cstatus == "Connected"
1038 self.is_primary = self.lrole == "Primary"
1039 self.is_secondary = self.lrole == "Secondary"
1040 self.peer_primary = self.rrole == "Primary"
1041 self.peer_secondary = self.rrole == "Secondary"
1042 self.both_primary = self.is_primary and self.peer_primary
1043 self.both_secondary = self.is_secondary and self.peer_secondary
1045 self.is_diskless = self.ldisk == "Diskless"
1046 self.is_disk_uptodate = self.ldisk == "UpToDate"
1048 self.is_in_resync = self.cstatus in ('SyncSource', 'SyncTarget')
1050 m = self.SYNC_RE.match(procline)
1052 self.sync_percent = float(m.group(1))
1053 hours = int(m.group(2))
1054 minutes = int(m.group(3))
1055 seconds = int(m.group(4))
1056 self.est_time = hours * 3600 + minutes * 60 + seconds
1058 self.sync_percent = None
1059 self.est_time = None
1061 self.is_sync_target = self.peer_sync_source = self.cstatus == "SyncTarget"
1062 self.peer_sync_target = self.is_sync_source = self.cstatus == "SyncSource"
1063 self.is_resync = self.is_sync_target or self.is_sync_source
1066 class BaseDRBD(BlockDev):
1069 This class contains a few bits of common functionality between the
1070 0.7 and 8.x versions of DRBD.
1073 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
1074 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
1077 _ST_UNCONFIGURED = "Unconfigured"
1078 _ST_WFCONNECTION = "WFConnection"
1079 _ST_CONNECTED = "Connected"
1081 _STATUS_FILE = "/proc/drbd"
1084 def _GetProcData(filename=_STATUS_FILE):
1085 """Return data from /proc/drbd.
1088 stat = open(filename, "r")
1090 data = stat.read().splitlines()
1094 raise errors.BlockDeviceError("Can't read any data from %s" % filename)
1098 def _MassageProcData(data):
1099 """Transform the output of _GetProdData into a nicer form.
1102 a dictionary of minor: joined lines from /proc/drbd for that minor
1105 lmatch = re.compile("^ *([0-9]+):.*$")
1107 old_minor = old_line = None
1109 lresult = lmatch.match(line)
1110 if lresult is not None:
1111 if old_minor is not None:
1112 results[old_minor] = old_line
1113 old_minor = int(lresult.group(1))
1116 if old_minor is not None:
1117 old_line += " " + line.strip()
1119 if old_minor is not None:
1120 results[old_minor] = old_line
1124 def _GetVersion(cls):
1125 """Return the DRBD version.
1127 This will return a dict with keys:
1133 proto2 (only on drbd > 8.2.X)
1136 proc_data = cls._GetProcData()
1137 first_line = proc_data[0].strip()
1138 version = cls._VERSION_RE.match(first_line)
1140 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
1143 values = version.groups()
1144 retval = {'k_major': int(values[0]),
1145 'k_minor': int(values[1]),
1146 'k_point': int(values[2]),
1147 'api': int(values[3]),
1148 'proto': int(values[4]),
1150 if values[5] is not None:
1151 retval['proto2'] = values[5]
1156 def _DevPath(minor):
1157 """Return the path to a drbd device for a given minor.
1160 return "/dev/drbd%d" % minor
1163 def _GetUsedDevs(cls):
1164 """Compute the list of used DRBD devices.
1167 data = cls._GetProcData()
1170 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
1172 match = valid_line.match(line)
1175 minor = int(match.group(1))
1176 state = match.group(2)
1177 if state == cls._ST_UNCONFIGURED:
1179 used_devs[minor] = state, line
1183 def _SetFromMinor(self, minor):
1184 """Set our parameters based on the given minor.
1186 This sets our minor variable and our dev_path.
1190 self.minor = self.dev_path = None
1193 self.dev_path = self._DevPath(minor)
1196 def _CheckMetaSize(meta_device):
1197 """Check if the given meta device looks like a valid one.
1199 This currently only check the size, which must be around
1203 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1205 logger.Error("Failed to get device size: %s - %s" %
1206 (result.fail_reason, result.output))
1209 sectors = int(result.stdout)
1211 logger.Error("Invalid output from blockdev: '%s'" % result.stdout)
1213 bytes = sectors * 512
1214 if bytes < 128 * 1024 * 1024: # less than 128MiB
1215 logger.Error("Meta device too small (%.2fMib)" % (bytes / 1024 / 1024))
1217 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
1218 logger.Error("Meta device too big (%.2fMiB)" % (bytes / 1024 / 1024))
1222 def Rename(self, new_id):
1225 This is not supported for drbd devices.
1228 raise errors.ProgrammerError("Can't rename a drbd device")
1231 class DRBDev(BaseDRBD):
1232 """DRBD block device.
1234 This implements the local host part of the DRBD device, i.e. it
1235 doesn't do anything to the supposed peer. If you need a fully
1236 connected DRBD pair, you need to use this class on both hosts.
1238 The unique_id for the drbd device is the (local_ip, local_port,
1239 remote_ip, remote_port) tuple, and it must have two children: the
1240 data device and the meta_device. The meta device is checked for
1241 valid size and is zeroed on create.
1244 def __init__(self, unique_id, children):
1245 super(DRBDev, self).__init__(unique_id, children)
1246 self.major = self._DRBD_MAJOR
1247 version = self._GetVersion()
1248 if version['k_major'] != 0 and version['k_minor'] != 7:
1249 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
1250 " requested ganeti usage: kernel is"
1251 " %s.%s, ganeti wants 0.7" %
1252 (version['k_major'], version['k_minor']))
1253 if len(children) != 2:
1254 raise ValueError("Invalid configuration data %s" % str(children))
1255 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
1256 raise ValueError("Invalid configuration data %s" % str(unique_id))
1257 self._lhost, self._lport, self._rhost, self._rport = unique_id
1261 def _FindUnusedMinor(cls):
1262 """Find an unused DRBD device.
1265 data = cls._GetProcData()
1267 valid_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
1269 match = valid_line.match(line)
1271 return int(match.group(1))
1272 logger.Error("Error: no free drbd minors!")
1273 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1276 def _GetDevInfo(cls, minor):
1277 """Get details about a given DRBD minor.
1279 This return, if available, the local backing device in (major,
1280 minor) formant and the local and remote (ip, port) information.
1284 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1286 logger.Error("Can't display the drbd config: %s - %s" %
1287 (result.fail_reason, result.output))
1290 if out == "Not configured\n":
1292 for line in out.splitlines():
1293 if "local_dev" not in data:
1294 match = re.match("^Lower device: ([0-9]+):([0-9]+) .*$", line)
1296 data["local_dev"] = (int(match.group(1)), int(match.group(2)))
1298 if "meta_dev" not in data:
1299 match = re.match("^Meta device: (([0-9]+):([0-9]+)|internal).*$", line)
1301 if match.group(2) is not None and match.group(3) is not None:
1302 # matched on the major/minor
1303 data["meta_dev"] = (int(match.group(2)), int(match.group(3)))
1305 # matched on the "internal" string
1306 data["meta_dev"] = match.group(1)
1307 # in this case, no meta_index is in the output
1308 data["meta_index"] = -1
1310 if "meta_index" not in data:
1311 match = re.match("^Meta index: ([0-9]+).*$", line)
1313 data["meta_index"] = int(match.group(1))
1315 if "local_addr" not in data:
1316 match = re.match("^Local address: ([0-9.]+):([0-9]+)$", line)
1318 data["local_addr"] = (match.group(1), int(match.group(2)))
1320 if "remote_addr" not in data:
1321 match = re.match("^Remote address: ([0-9.]+):([0-9]+)$", line)
1323 data["remote_addr"] = (match.group(1), int(match.group(2)))
1327 def _MatchesLocal(self, info):
1328 """Test if our local config matches with an existing device.
1330 The parameter should be as returned from `_GetDevInfo()`. This
1331 method tests if our local backing device is the same as the one in
1332 the info parameter, in effect testing if we look like the given
1336 if not ("local_dev" in info and "meta_dev" in info and
1337 "meta_index" in info):
1340 backend = self._children[0]
1341 if backend is not None:
1342 retval = (info["local_dev"] == (backend.major, backend.minor))
1344 retval = (info["local_dev"] == (0, 0))
1345 meta = self._children[1]
1346 if meta is not None:
1347 retval = retval and (info["meta_dev"] == (meta.major, meta.minor))
1348 retval = retval and (info["meta_index"] == 0)
1350 retval = retval and (info["meta_dev"] == "internal" and
1351 info["meta_index"] == -1)
1354 def _MatchesNet(self, info):
1355 """Test if our network config matches with an existing device.
1357 The parameter should be as returned from `_GetDevInfo()`. This
1358 method tests if our network configuration is the same as the one
1359 in the info parameter, in effect testing if we look like the given
1363 if (((self._lhost is None and not ("local_addr" in info)) and
1364 (self._rhost is None and not ("remote_addr" in info)))):
1367 if self._lhost is None:
1370 if not ("local_addr" in info and
1371 "remote_addr" in info):
1374 retval = (info["local_addr"] == (self._lhost, self._lport))
1375 retval = (retval and
1376 info["remote_addr"] == (self._rhost, self._rport))
1380 def _AssembleLocal(cls, minor, backend, meta):
1381 """Configure the local part of a DRBD device.
1383 This is the first thing that must be done on an unconfigured DRBD
1384 device. And it must be done only once.
1387 if not cls._CheckMetaSize(meta):
1389 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1390 backend, meta, "0", "-e", "detach"])
1392 logger.Error("Can't attach local disk: %s" % result.output)
1393 return not result.failed
1396 def _ShutdownLocal(cls, minor):
1397 """Detach from the local device.
1399 I/Os will continue to be served from the remote device. If we
1400 don't have a remote device, this operation will fail.
1403 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1405 logger.Error("Can't detach local device: %s" % result.output)
1406 return not result.failed
1409 def _ShutdownAll(minor):
1410 """Deactivate the device.
1412 This will, of course, fail if the device is in use.
1415 result = utils.RunCmd(["drbdsetup", DRBDev._DevPath(minor), "down"])
1417 logger.Error("Can't shutdown drbd device: %s" % result.output)
1418 return not result.failed
1421 def _AssembleNet(cls, minor, net_info, protocol):
1422 """Configure the network part of the device.
1424 This operation can be, in theory, done multiple times, but there
1425 have been cases (in lab testing) in which the network part of the
1426 device had become stuck and couldn't be shut down because activity
1427 from the new peer (also stuck) triggered a timer re-init and
1428 needed remote peer interface shutdown in order to clear. So please
1429 don't change online the net config.
1432 lhost, lport, rhost, rport = net_info
1433 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "net",
1434 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport),
1437 logger.Error("Can't setup network for dbrd device: %s - %s" %
1438 (result.fail_reason, result.output))
1441 timeout = time.time() + 10
1443 while time.time() < timeout:
1444 info = cls._GetDevInfo(minor)
1445 if not "local_addr" in info or not "remote_addr" in info:
1448 if (info["local_addr"] != (lhost, lport) or
1449 info["remote_addr"] != (rhost, rport)):
1455 logger.Error("Timeout while configuring network")
1460 def _ShutdownNet(cls, minor):
1461 """Disconnect from the remote peer.
1463 This fails if we don't have a local device.
1466 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1468 logger.Error("Can't shutdown network: %s" % result.output)
1469 return not result.failed
1472 """Assemble the drbd.
1475 - if we have a local backing device, we bind to it by:
1476 - checking the list of used drbd devices
1477 - check if the local minor use of any of them is our own device
1480 - if we have a local/remote net info:
1481 - redo the local backing device step for the remote device
1482 - check if any drbd device is using the local port,
1484 - check if any remote drbd device is using the remote
1485 port, if yes abort (for now)
1487 - bind the remote net port
1491 if self.minor is not None:
1492 logger.Info("Already assembled")
1495 result = super(DRBDev, self).Assemble()
1499 minor = self._FindUnusedMinor()
1500 need_localdev_teardown = False
1501 if self._children[0]:
1502 result = self._AssembleLocal(minor, self._children[0].dev_path,
1503 self._children[1].dev_path)
1506 need_localdev_teardown = True
1507 if self._lhost and self._lport and self._rhost and self._rport:
1508 result = self._AssembleNet(minor,
1509 (self._lhost, self._lport,
1510 self._rhost, self._rport),
1513 if need_localdev_teardown:
1514 # we will ignore failures from this
1515 logger.Error("net setup failed, tearing down local device")
1516 self._ShutdownAll(minor)
1518 self._SetFromMinor(minor)
1522 """Shutdown the DRBD device.
1525 if self.minor is None and not self.Attach():
1526 logger.Info("DRBD device not attached to a device during Shutdown")
1528 if not self._ShutdownAll(self.minor):
1531 self.dev_path = None
1535 """Find a DRBD device which matches our config and attach to it.
1537 In case of partially attached (local device matches but no network
1538 setup), we perform the network attach. If successful, we re-test
1539 the attach if can return success.
1542 for minor in self._GetUsedDevs():
1543 info = self._GetDevInfo(minor)
1544 match_l = self._MatchesLocal(info)
1545 match_r = self._MatchesNet(info)
1546 if match_l and match_r:
1548 if match_l and not match_r and "local_addr" not in info:
1549 res_r = self._AssembleNet(minor,
1550 (self._lhost, self._lport,
1551 self._rhost, self._rport),
1553 if res_r and self._MatchesNet(self._GetDevInfo(minor)):
1558 self._SetFromMinor(minor)
1559 return minor is not None
1561 def Open(self, force=False):
1562 """Make the local state primary.
1564 If the 'force' parameter is given, the '--do-what-I-say' option is
1565 is passed to drbdsetup. Since this is a potentially dangerous operation,
1566 the force flag should be only given after creation, when it actually
1570 if self.minor is None and not self.Attach():
1571 logger.Error("DRBD cannot attach to a device during open")
1573 cmd = ["drbdsetup", self.dev_path, "primary"]
1575 cmd.append("--do-what-I-say")
1576 result = utils.RunCmd(cmd)
1578 msg = ("Can't make drbd device primary: %s" % result.output)
1580 raise errors.BlockDeviceError(msg)
1583 """Make the local state secondary.
1585 This will, of course, fail if the device is in use.
1588 if self.minor is None and not self.Attach():
1589 logger.Info("Instance not attached to a device")
1590 raise errors.BlockDeviceError("Can't find device")
1591 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1593 msg = ("Can't switch drbd device to"
1594 " secondary: %s" % result.output)
1596 raise errors.BlockDeviceError(msg)
1598 def SetSyncSpeed(self, kbytes):
1599 """Set the speed of the DRBD syncer.
1602 children_result = super(DRBDev, self).SetSyncSpeed(kbytes)
1603 if self.minor is None:
1604 logger.Info("Instance not attached to a device")
1606 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1609 logger.Error("Can't change syncer rate: %s - %s" %
1610 (result.fail_reason, result.output))
1611 return not result.failed and children_result
1613 def GetSyncStatus(self):
1614 """Returns the sync status of the device.
1617 (sync_percent, estimated_time, is_degraded, ldisk)
1619 If sync_percent is None, it means all is ok
1620 If estimated_time is None, it means we can't esimate
1621 the time needed, otherwise it's the time left in seconds.
1623 The ldisk parameter will be returned as True, since the DRBD7
1624 devices have not been converted.
1627 if self.minor is None and not self.Attach():
1628 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1629 proc_info = self._MassageProcData(self._GetProcData())
1630 if self.minor not in proc_info:
1631 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1633 line = proc_info[self.minor]
1634 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
1635 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
1637 sync_percent = float(match.group(1))
1638 hours = int(match.group(2))
1639 minutes = int(match.group(3))
1640 seconds = int(match.group(4))
1641 est_time = hours * 3600 + minutes * 60 + seconds
1645 match = re.match("^ *[0-9]+: cs:([^ ]+).*$", line)
1647 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
1649 client_state = match.group(1)
1650 is_degraded = client_state != "Connected"
1651 return sync_percent, est_time, is_degraded, False
1653 def GetStatus(self):
1654 """Compute the status of the DRBD device
1656 Note that DRBD devices don't have the STATUS_EXISTING state.
1659 if self.minor is None and not self.Attach():
1660 return self.STATUS_UNKNOWN
1662 data = self._GetProcData()
1663 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
1666 mresult = match.match(line)
1670 logger.Error("Can't find myself!")
1671 return self.STATUS_UNKNOWN
1673 state = mresult.group(2)
1674 if state == "Primary":
1675 result = self.STATUS_ONLINE
1677 result = self.STATUS_STANDBY
1682 def _ZeroDevice(device):
1685 This writes until we get ENOSPC.
1688 f = open(device, "w")
1689 buf = "\0" * 1048576
1693 except IOError, err:
1694 if err.errno != errno.ENOSPC:
1698 def Create(cls, unique_id, children, size):
1699 """Create a new DRBD device.
1701 Since DRBD devices are not created per se, just assembled, this
1702 function just zeroes the meta device.
1705 if len(children) != 2:
1706 raise errors.ProgrammerError("Invalid setup for the drbd device")
1709 if not meta.Attach():
1710 raise errors.BlockDeviceError("Can't attach to meta device")
1711 if not cls._CheckMetaSize(meta.dev_path):
1712 raise errors.BlockDeviceError("Invalid meta device")
1713 logger.Info("Started zeroing device %s" % meta.dev_path)
1714 cls._ZeroDevice(meta.dev_path)
1715 logger.Info("Done zeroing device %s" % meta.dev_path)
1716 return cls(unique_id, children)
1719 """Stub remove for DRBD devices.
1722 return self.Shutdown()
1725 class DRBD8(BaseDRBD):
1726 """DRBD v8.x block device.
1728 This implements the local host part of the DRBD device, i.e. it
1729 doesn't do anything to the supposed peer. If you need a fully
1730 connected DRBD pair, you need to use this class on both hosts.
1732 The unique_id for the drbd device is the (local_ip, local_port,
1733 remote_ip, remote_port) tuple, and it must have two children: the
1734 data device and the meta_device. The meta device is checked for
1735 valid size and is zeroed on create.
1742 _NET_RECONFIG_TIMEOUT = 60
1744 def __init__(self, unique_id, children):
1745 if children and children.count(None) > 0:
1747 super(DRBD8, self).__init__(unique_id, children)
1748 self.major = self._DRBD_MAJOR
1749 version = self._GetVersion()
1750 if version['k_major'] != 8 :
1751 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
1752 " requested ganeti usage: kernel is"
1753 " %s.%s, ganeti wants 8.x" %
1754 (version['k_major'], version['k_minor']))
1756 if len(children) not in (0, 2):
1757 raise ValueError("Invalid configuration data %s" % str(children))
1758 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
1759 raise ValueError("Invalid configuration data %s" % str(unique_id))
1760 self._lhost, self._lport, self._rhost, self._rport = unique_id
1764 def _InitMeta(cls, minor, dev_path):
1765 """Initialize a meta device.
1767 This will not work if the given minor is in use.
1770 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
1771 "v08", dev_path, "0", "create-md"])
1773 raise errors.BlockDeviceError("Can't initialize meta device: %s" %
1777 def _FindUnusedMinor(cls):
1778 """Find an unused DRBD device.
1780 This is specific to 8.x as the minors are allocated dynamically,
1781 so non-existing numbers up to a max minor count are actually free.
1784 data = cls._GetProcData()
1786 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
1787 used_line = re.compile("^ *([0-9]+): cs:")
1790 match = unused_line.match(line)
1792 return int(match.group(1))
1793 match = used_line.match(line)
1795 minor = int(match.group(1))
1796 highest = max(highest, minor)
1797 if highest is None: # there are no minors in use at all
1799 if highest >= cls._MAX_MINORS:
1800 logger.Error("Error: no free drbd minors!")
1801 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1805 def _IsValidMeta(cls, meta_device):
1806 """Check if the given meta device looks like a valid one.
1809 minor = cls._FindUnusedMinor()
1810 minor_path = cls._DevPath(minor)
1811 result = utils.RunCmd(["drbdmeta", minor_path,
1812 "v08", meta_device, "0",
1815 logger.Error("Invalid meta device %s: %s" % (meta_device, result.output))
1820 def _GetShowParser(cls):
1821 """Return a parser for `drbd show` output.
1823 This will either create or return an already-create parser for the
1824 output of the command `drbd show`.
1827 if cls._PARSE_SHOW is not None:
1828 return cls._PARSE_SHOW
1831 lbrace = pyp.Literal("{").suppress()
1832 rbrace = pyp.Literal("}").suppress()
1833 semi = pyp.Literal(";").suppress()
1834 # this also converts the value to an int
1835 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
1837 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
1838 defa = pyp.Literal("_is_default").suppress()
1839 dbl_quote = pyp.Literal('"').suppress()
1841 keyword = pyp.Word(pyp.alphanums + '-')
1844 value = pyp.Word(pyp.alphanums + '_-/.:')
1845 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
1846 addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() +
1848 # meta device, extended syntax
1849 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
1850 number + pyp.Word(']').suppress())
1853 stmt = (~rbrace + keyword + ~lbrace +
1854 pyp.Optional(addr_port ^ value ^ quoted ^ meta_value) +
1855 pyp.Optional(defa) + semi +
1856 pyp.Optional(pyp.restOfLine).suppress())
1859 section_name = pyp.Word(pyp.alphas + '_')
1860 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
1862 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
1865 cls._PARSE_SHOW = bnf
1870 def _GetShowData(cls, minor):
1871 """Return the `drbdsetup show` data for a minor.
1874 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1876 logger.Error("Can't display the drbd config: %s - %s" %
1877 (result.fail_reason, result.output))
1879 return result.stdout
1882 def _GetDevInfo(cls, out):
1883 """Parse details about a given DRBD minor.
1885 This return, if available, the local backing device (as a path)
1886 and the local and remote (ip, port) information from a string
1887 containing the output of the `drbdsetup show` command as returned
1895 bnf = cls._GetShowParser()
1899 results = bnf.parseString(out)
1900 except pyp.ParseException, err:
1901 raise errors.BlockDeviceError("Can't parse drbdsetup show output: %s" %
1904 # and massage the results into our desired format
1905 for section in results:
1907 if sname == "_this_host":
1908 for lst in section[1:]:
1909 if lst[0] == "disk":
1910 data["local_dev"] = lst[1]
1911 elif lst[0] == "meta-disk":
1912 data["meta_dev"] = lst[1]
1913 data["meta_index"] = lst[2]
1914 elif lst[0] == "address":
1915 data["local_addr"] = tuple(lst[1:])
1916 elif sname == "_remote_host":
1917 for lst in section[1:]:
1918 if lst[0] == "address":
1919 data["remote_addr"] = tuple(lst[1:])
1922 def _MatchesLocal(self, info):
1923 """Test if our local config matches with an existing device.
1925 The parameter should be as returned from `_GetDevInfo()`. This
1926 method tests if our local backing device is the same as the one in
1927 the info parameter, in effect testing if we look like the given
1932 backend, meta = self._children
1934 backend = meta = None
1936 if backend is not None:
1937 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
1939 retval = ("local_dev" not in info)
1941 if meta is not None:
1942 retval = retval and ("meta_dev" in info and
1943 info["meta_dev"] == meta.dev_path)
1944 retval = retval and ("meta_index" in info and
1945 info["meta_index"] == 0)
1947 retval = retval and ("meta_dev" not in info and
1948 "meta_index" not in info)
1951 def _MatchesNet(self, info):
1952 """Test if our network config matches with an existing device.
1954 The parameter should be as returned from `_GetDevInfo()`. This
1955 method tests if our network configuration is the same as the one
1956 in the info parameter, in effect testing if we look like the given
1960 if (((self._lhost is None and not ("local_addr" in info)) and
1961 (self._rhost is None and not ("remote_addr" in info)))):
1964 if self._lhost is None:
1967 if not ("local_addr" in info and
1968 "remote_addr" in info):
1971 retval = (info["local_addr"] == (self._lhost, self._lport))
1972 retval = (retval and
1973 info["remote_addr"] == (self._rhost, self._rport))
1977 def _AssembleLocal(cls, minor, backend, meta):
1978 """Configure the local part of a DRBD device.
1980 This is the first thing that must be done on an unconfigured DRBD
1981 device. And it must be done only once.
1984 if not cls._IsValidMeta(meta):
1986 args = ["drbdsetup", cls._DevPath(minor), "disk",
1987 backend, meta, "0", "-e", "detach", "--create-device"]
1988 result = utils.RunCmd(args)
1990 logger.Error("Can't attach local disk: %s" % result.output)
1991 return not result.failed
1994 def _AssembleNet(cls, minor, net_info, protocol,
1995 dual_pri=False, hmac=None, secret=None):
1996 """Configure the network part of the device.
1999 lhost, lport, rhost, rport = net_info
2000 if None in net_info:
2001 # we don't want network connection and actually want to make
2003 return cls._ShutdownNet(minor)
2005 args = ["drbdsetup", cls._DevPath(minor), "net",
2006 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
2007 "-A", "discard-zero-changes",
2014 args.extend(["-a", hmac, "-x", secret])
2015 result = utils.RunCmd(args)
2017 logger.Error("Can't setup network for dbrd device: %s - %s" %
2018 (result.fail_reason, result.output))
2021 timeout = time.time() + 10
2023 while time.time() < timeout:
2024 info = cls._GetDevInfo(cls._GetShowData(minor))
2025 if not "local_addr" in info or not "remote_addr" in info:
2028 if (info["local_addr"] != (lhost, lport) or
2029 info["remote_addr"] != (rhost, rport)):
2035 logger.Error("Timeout while configuring network")
2039 def AddChildren(self, devices):
2040 """Add a disk to the DRBD device.
2043 if self.minor is None:
2044 raise errors.BlockDeviceError("Can't attach to dbrd8 during AddChildren")
2045 if len(devices) != 2:
2046 raise errors.BlockDeviceError("Need two devices for AddChildren")
2047 info = self._GetDevInfo(self._GetShowData(self.minor))
2048 if "local_dev" in info:
2049 raise errors.BlockDeviceError("DRBD8 already attached to a local disk")
2050 backend, meta = devices
2051 if backend.dev_path is None or meta.dev_path is None:
2052 raise errors.BlockDeviceError("Children not ready during AddChildren")
2055 if not self._CheckMetaSize(meta.dev_path):
2056 raise errors.BlockDeviceError("Invalid meta device size")
2057 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
2058 if not self._IsValidMeta(meta.dev_path):
2059 raise errors.BlockDeviceError("Cannot initalize meta device")
2061 if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path):
2062 raise errors.BlockDeviceError("Can't attach to local storage")
2063 self._children = devices
2065 def RemoveChildren(self, devices):
2066 """Detach the drbd device from local storage.
2069 if self.minor is None:
2070 raise errors.BlockDeviceError("Can't attach to drbd8 during"
2072 # early return if we don't actually have backing storage
2073 info = self._GetDevInfo(self._GetShowData(self.minor))
2074 if "local_dev" not in info:
2076 if len(self._children) != 2:
2077 raise errors.BlockDeviceError("We don't have two children: %s" %
2079 if self._children.count(None) == 2: # we don't actually have children :)
2080 logger.Error("Requested detach while detached")
2082 if len(devices) != 2:
2083 raise errors.BlockDeviceError("We need two children in RemoveChildren")
2084 for child, dev in zip(self._children, devices):
2085 if dev != child.dev_path:
2086 raise errors.BlockDeviceError("Mismatch in local storage"
2087 " (%s != %s) in RemoveChildren" %
2088 (dev, child.dev_path))
2090 if not self._ShutdownLocal(self.minor):
2091 raise errors.BlockDeviceError("Can't detach from local storage")
2094 def SetSyncSpeed(self, kbytes):
2095 """Set the speed of the DRBD syncer.
2098 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
2099 if self.minor is None:
2100 logger.Info("Instance not attached to a device")
2102 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
2105 logger.Error("Can't change syncer rate: %s - %s" %
2106 (result.fail_reason, result.output))
2107 return not result.failed and children_result
2109 def GetProcStatus(self):
2110 """Return device data from /proc.
2113 if self.minor is None:
2114 raise errors.BlockDeviceError("GetStats() called while not attached")
2115 proc_info = self._MassageProcData(self._GetProcData())
2116 if self.minor not in proc_info:
2117 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
2119 return DRBD8Status(proc_info[self.minor])
2121 def GetSyncStatus(self):
2122 """Returns the sync status of the device.
2125 (sync_percent, estimated_time, is_degraded)
2127 If sync_percent is None, it means all is ok
2128 If estimated_time is None, it means we can't esimate
2129 the time needed, otherwise it's the time left in seconds.
2132 We set the is_degraded parameter to True on two conditions:
2133 network not connected or local disk missing.
2135 We compute the ldisk parameter based on wheter we have a local
2139 if self.minor is None and not self.Attach():
2140 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
2141 stats = self.GetProcStatus()
2142 ldisk = not stats.is_disk_uptodate
2143 is_degraded = not stats.is_connected
2144 return stats.sync_percent, stats.est_time, is_degraded or ldisk, ldisk
2146 def GetStatus(self):
2147 """Compute the status of the DRBD device
2149 Note that DRBD devices don't have the STATUS_EXISTING state.
2152 if self.minor is None and not self.Attach():
2153 return self.STATUS_UNKNOWN
2155 data = self._GetProcData()
2156 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
2159 mresult = match.match(line)
2163 logger.Error("Can't find myself!")
2164 return self.STATUS_UNKNOWN
2166 state = mresult.group(2)
2167 if state == "Primary":
2168 result = self.STATUS_ONLINE
2170 result = self.STATUS_STANDBY
2174 def Open(self, force=False):
2175 """Make the local state primary.
2177 If the 'force' parameter is given, the '-o' option is passed to
2178 drbdsetup. Since this is a potentially dangerous operation, the
2179 force flag should be only given after creation, when it actually
2183 if self.minor is None and not self.Attach():
2184 logger.Error("DRBD cannot attach to a device during open")
2186 cmd = ["drbdsetup", self.dev_path, "primary"]
2189 result = utils.RunCmd(cmd)
2191 msg = ("Can't make drbd device primary: %s" % result.output)
2193 raise errors.BlockDeviceError(msg)
2196 """Make the local state secondary.
2198 This will, of course, fail if the device is in use.
2201 if self.minor is None and not self.Attach():
2202 logger.Info("Instance not attached to a device")
2203 raise errors.BlockDeviceError("Can't find device")
2204 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
2206 msg = ("Can't switch drbd device to"
2207 " secondary: %s" % result.output)
2209 raise errors.BlockDeviceError(msg)
2211 def DisconnectNet(self):
2212 """Removes network configuration.
2214 This method shutdowns the network side of the device.
2216 The method will wait up to a hardcoded timeout for the device to
2217 go into standalone after the 'disconnect' command before
2218 re-configuring it, as sometimes it takes a while for the
2219 disconnect to actually propagate and thus we might issue a 'net'
2220 command while the device is still connected. If the device will
2221 still be attached to the network and we time out, we raise an
2225 if self.minor is None:
2226 raise errors.BlockDeviceError("DRBD disk not attached in re-attach net")
2228 if None in (self._lhost, self._lport, self._rhost, self._rport):
2229 raise errors.BlockDeviceError("DRBD disk missing network info in"
2232 ever_disconnected = self._ShutdownNet(self.minor)
2233 timeout_limit = time.time() + self._NET_RECONFIG_TIMEOUT
2234 sleep_time = 0.100 # we start the retry time at 100 miliseconds
2235 while time.time() < timeout_limit:
2236 status = self.GetProcStatus()
2237 if status.is_standalone:
2239 # retry the disconnect, it seems possible that due to a
2240 # well-time disconnect on the peer, my disconnect command might
2241 # be ingored and forgotten
2242 ever_disconnected = self._ShutdownNet(self.minor) or ever_disconnected
2243 time.sleep(sleep_time)
2244 sleep_time = min(2, sleep_time * 1.5)
2246 if not status.is_standalone:
2247 if ever_disconnected:
2248 msg = ("Device did not react to the"
2249 " 'disconnect' command in a timely manner")
2251 msg = ("Can't shutdown network, even after multiple retries")
2252 raise errors.BlockDeviceError(msg)
2254 reconfig_time = time.time() - timeout_limit + self._NET_RECONFIG_TIMEOUT
2255 if reconfig_time > 15: # hardcoded alert limit
2256 logger.Debug("DRBD8.ReAttachNet: detach took %.3f seconds" %
2259 def ReAttachNet(self, multimaster):
2260 """Reconnects the network.
2262 This method reconnects the network side of the device with a
2263 specified multi-master flag. The device needs to be 'Standalone'
2264 but have valid network configuration data.
2267 - multimaster: init the network in dual-primary mode
2270 if self.minor is None:
2271 raise errors.BlockDeviceError("DRBD disk not attached in re-attach net")
2273 if None in (self._lhost, self._lport, self._rhost, self._rport):
2274 raise errors.BlockDeviceError("DRBD disk missing network info in"
2277 status = self.GetProcStatus()
2279 if not status.is_standalone:
2280 raise errors.BlockDeviceError("Device is not standalone in"
2283 return self._AssembleNet(self.minor,
2284 (self._lhost, self._lport,
2285 self._rhost, self._rport),
2286 "C", dual_pri=multimaster)
2289 """Find a DRBD device which matches our config and attach to it.
2291 In case of partially attached (local device matches but no network
2292 setup), we perform the network attach. If successful, we re-test
2293 the attach if can return success.
2296 for minor in self._GetUsedDevs():
2297 info = self._GetDevInfo(self._GetShowData(minor))
2298 match_l = self._MatchesLocal(info)
2299 match_r = self._MatchesNet(info)
2300 if match_l and match_r:
2302 if match_l and not match_r and "local_addr" not in info:
2303 res_r = self._AssembleNet(minor,
2304 (self._lhost, self._lport,
2305 self._rhost, self._rport),
2308 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
2310 # the weakest case: we find something that is only net attached
2311 # even though we were passed some children at init time
2312 if match_r and "local_dev" not in info:
2315 # this case must be considered only if we actually have local
2316 # storage, i.e. not in diskless mode, because all diskless
2317 # devices are equal from the point of view of local
2319 if (match_l and "local_dev" in info and
2320 not match_r and "local_addr" in info):
2321 # strange case - the device network part points to somewhere
2322 # else, even though its local storage is ours; as we own the
2323 # drbd space, we try to disconnect from the remote peer and
2324 # reconnect to our correct one
2325 if not self._ShutdownNet(minor):
2326 raise errors.BlockDeviceError("Device has correct local storage,"
2327 " wrong remote peer and is unable to"
2328 " disconnect in order to attach to"
2329 " the correct peer")
2330 # note: _AssembleNet also handles the case when we don't want
2331 # local storage (i.e. one or more of the _[lr](host|port) is
2333 if (self._AssembleNet(minor, (self._lhost, self._lport,
2334 self._rhost, self._rport), "C") and
2335 self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))):
2341 self._SetFromMinor(minor)
2342 return minor is not None
2345 """Assemble the drbd.
2348 - if we have a local backing device, we bind to it by:
2349 - checking the list of used drbd devices
2350 - check if the local minor use of any of them is our own device
2353 - if we have a local/remote net info:
2354 - redo the local backing device step for the remote device
2355 - check if any drbd device is using the local port,
2357 - check if any remote drbd device is using the remote
2358 port, if yes abort (for now)
2360 - bind the remote net port
2364 if self.minor is not None:
2365 logger.Info("Already assembled")
2368 result = super(DRBD8, self).Assemble()
2372 minor = self._FindUnusedMinor()
2373 need_localdev_teardown = False
2374 if self._children and self._children[0] and self._children[1]:
2375 result = self._AssembleLocal(minor, self._children[0].dev_path,
2376 self._children[1].dev_path)
2379 need_localdev_teardown = True
2380 if self._lhost and self._lport and self._rhost and self._rport:
2381 result = self._AssembleNet(minor,
2382 (self._lhost, self._lport,
2383 self._rhost, self._rport),
2386 if need_localdev_teardown:
2387 # we will ignore failures from this
2388 logger.Error("net setup failed, tearing down local device")
2389 self._ShutdownAll(minor)
2391 self._SetFromMinor(minor)
2395 def _ShutdownLocal(cls, minor):
2396 """Detach from the local device.
2398 I/Os will continue to be served from the remote device. If we
2399 don't have a remote device, this operation will fail.
2402 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
2404 logger.Error("Can't detach local device: %s" % result.output)
2405 return not result.failed
2408 def _ShutdownNet(cls, minor):
2409 """Disconnect from the remote peer.
2411 This fails if we don't have a local device.
2414 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
2416 logger.Error("Can't shutdown network: %s" % result.output)
2417 return not result.failed
2420 def _ShutdownAll(cls, minor):
2421 """Deactivate the device.
2423 This will, of course, fail if the device is in use.
2426 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
2428 logger.Error("Can't shutdown drbd device: %s" % result.output)
2429 return not result.failed
2432 """Shutdown the DRBD device.
2435 if self.minor is None and not self.Attach():
2436 logger.Info("DRBD device not attached to a device during Shutdown")
2438 if not self._ShutdownAll(self.minor):
2441 self.dev_path = None
2445 """Stub remove for DRBD devices.
2448 return self.Shutdown()
2451 def Create(cls, unique_id, children, size):
2452 """Create a new DRBD8 device.
2454 Since DRBD devices are not created per se, just assembled, this
2455 function only initializes the metadata.
2458 if len(children) != 2:
2459 raise errors.ProgrammerError("Invalid setup for the drbd device")
2462 if not meta.Attach():
2463 raise errors.BlockDeviceError("Can't attach to meta device")
2464 if not cls._CheckMetaSize(meta.dev_path):
2465 raise errors.BlockDeviceError("Invalid meta device size")
2466 cls._InitMeta(cls._FindUnusedMinor(), meta.dev_path)
2467 if not cls._IsValidMeta(meta.dev_path):
2468 raise errors.BlockDeviceError("Cannot initalize meta device")
2469 return cls(unique_id, children)
2471 def Grow(self, amount):
2472 """Resize the DRBD device and its backing storage.
2475 if self.minor is None:
2476 raise errors.ProgrammerError("drbd8: Grow called while not attached")
2477 if len(self._children) != 2 or None in self._children:
2478 raise errors.BlockDeviceError("Cannot grow diskless DRBD8 device")
2479 self._children[0].Grow(amount)
2480 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize"])
2482 raise errors.BlockDeviceError("resize failed for %s: %s" %
2483 (self.dev_path, result.output))
2488 constants.LD_LV: LogicalVolume,
2489 constants.LD_MD_R1: MDRaid1,
2490 constants.LD_DRBD7: DRBDev,
2491 constants.LD_DRBD8: DRBD8,
2495 def FindDevice(dev_type, unique_id, children):
2496 """Search for an existing, assembled device.
2498 This will succeed only if the device exists and is assembled, but it
2499 does not do any actions in order to activate the device.
2502 if dev_type not in DEV_MAP:
2503 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2504 device = DEV_MAP[dev_type](unique_id, children)
2505 if not device.Attach():
2510 def AttachOrAssemble(dev_type, unique_id, children):
2511 """Try to attach or assemble an existing device.
2513 This will attach to an existing assembled device or will assemble
2514 the device, as needed, to bring it fully up.
2517 if dev_type not in DEV_MAP:
2518 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2519 device = DEV_MAP[dev_type](unique_id, children)
2520 if not device.Attach():
2522 if not device.Attach():
2523 raise errors.BlockDeviceError("Can't find a valid block device for"
2525 (dev_type, unique_id, children))
2529 def Create(dev_type, unique_id, children, size):
2533 if dev_type not in DEV_MAP:
2534 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2535 device = DEV_MAP[dev_type].Create(unique_id, children, size)