4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
29 from ganeti import utils
30 from ganeti import logger
31 from ganeti import errors
32 from ganeti import constants
35 class BlockDev(object):
36 """Block device abstract class.
38 A block device can be in the following states:
39 - not existing on the system, and by `Create()` it goes into:
40 - existing but not setup/not active, and by `Assemble()` goes into:
41 - active read-write and by `Open()` it goes into
42 - online (=used, or ready for use)
44 A device can also be online but read-only, however we are not using
45 the readonly state (MD and LV have it, if needed in the future)
46 and we are usually looking at this like at a stack, so it's easier
47 to conceptualise the transition from not-existing to online and back
50 The many different states of the device are due to the fact that we
51 need to cover many device types:
52 - logical volumes are created, lvchange -a y $lv, and used
53 - md arrays are created or assembled and used
54 - drbd devices are attached to a local disk/remote peer and made primary
56 The status of the device can be examined by `GetStatus()`, which
57 returns a numerical value, depending on the position in the
58 transition stack of the device.
60 A block device is identified by three items:
61 - the /dev path of the device (dynamic)
62 - a unique ID of the device (static)
63 - it's major/minor pair (dynamic)
65 Not all devices implement both the first two as distinct items. LVM
66 logical volumes have their unique ID (the pair volume group, logical
67 volume name) in a 1-to-1 relation to the dev path. For MD devices,
68 the /dev path is dynamic and the unique ID is the UUID generated at
69 array creation plus the slave list. For DRBD devices, the /dev path
70 is again dynamic and the unique id is the pair (host1, dev1),
73 You can get to a device in two ways:
74 - creating the (real) device, which returns you
75 an attached instance (lvcreate, mdadm --create)
76 - attaching of a python instance to an existing (real) device
78 The second point, the attachement to a device, is different
79 depending on whether the device is assembled or not. At init() time,
80 we search for a device with the same unique_id as us. If found,
81 good. It also means that the device is already assembled. If not,
82 after assembly we'll have our correct major/minor.
91 STATUS_UNKNOWN: "unknown",
92 STATUS_EXISTING: "existing",
93 STATUS_STANDBY: "ready for use",
94 STATUS_ONLINE: "online",
97 def __init__(self, unique_id, children):
98 self._children = children
100 self.unique_id = unique_id
105 """Assemble the device from its components.
107 If this is a plain block device (e.g. LVM) than assemble does
108 nothing, as the LVM has no children and we don't put logical
111 One guarantee is that after the device has been assembled, it
112 knows its major/minor numbers. This allows other devices (usually
113 parents) to probe correctly for their children.
117 for child in self._children:
118 if not isinstance(child, BlockDev):
119 raise TypeError("Invalid child passed of type '%s'" % type(child))
122 status = status and child.Assemble()
125 status = status and child.Open()
128 for child in self._children:
133 """Find a device which matches our config and attach to it.
136 raise NotImplementedError
139 """Notifies that the device will no longer be used for I/O.
142 raise NotImplementedError
145 def Create(cls, unique_id, children, size):
146 """Create the device.
148 If the device cannot be created, it will return None
149 instead. Error messages go to the logging system.
151 Note that for some devices, the unique_id is used, and for other,
152 the children. The idea is that these two, taken together, are
153 enough for both creation and assembly (later).
156 raise NotImplementedError
159 """Remove this device.
161 This makes sense only for some of the device types: LV and to a
162 lesser degree, md devices. Also note that if the device can't
163 attach, the removal can't be completed.
166 raise NotImplementedError
168 def Rename(self, new_id):
169 """Rename this device.
171 This may or may not make sense for a given device type.
174 raise NotImplementedError
177 """Return the status of the device.
180 raise NotImplementedError
182 def Open(self, force=False):
183 """Make the device ready for use.
185 This makes the device ready for I/O. For now, just the DRBD
188 The force parameter signifies that if the device has any kind of
189 --force thing, it should be used, we know what we are doing.
192 raise NotImplementedError
195 """Shut down the device, freeing its children.
197 This undoes the `Assemble()` work, except for the child
198 assembling; as such, the children on the device are still
199 assembled after this call.
202 raise NotImplementedError
204 def SetSyncSpeed(self, speed):
205 """Adjust the sync speed of the mirror.
207 In case this is not a mirroring device, this is no-op.
212 for child in self._children:
213 result = result and child.SetSyncSpeed(speed)
216 def GetSyncStatus(self):
217 """Returns the sync status of the device.
219 If this device is a mirroring device, this function returns the
220 status of the mirror.
223 (sync_percent, estimated_time, is_degraded, ldisk)
225 If sync_percent is None, it means the device is not syncing.
227 If estimated_time is None, it means we can't estimate
228 the time needed, otherwise it's the time left in seconds.
230 If is_degraded is True, it means the device is missing
231 redundancy. This is usually a sign that something went wrong in
232 the device setup, if sync_percent is None.
234 The ldisk parameter represents the degradation of the local
235 data. This is only valid for some devices, the rest will always
236 return False (not degraded).
239 return None, None, False, False
242 def CombinedSyncStatus(self):
243 """Calculate the mirror status recursively for our children.
245 The return value is the same as for `GetSyncStatus()` except the
246 minimum percent and maximum time are calculated across our
250 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
252 for child in self._children:
253 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
254 if min_percent is None:
255 min_percent = c_percent
256 elif c_percent is not None:
257 min_percent = min(min_percent, c_percent)
260 elif c_time is not None:
261 max_time = max(max_time, c_time)
262 is_degraded = is_degraded or c_degraded
263 ldisk = ldisk or c_ldisk
264 return min_percent, max_time, is_degraded, ldisk
267 def SetInfo(self, text):
268 """Update metadata with info text.
270 Only supported for some device types.
273 for child in self._children:
278 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
279 (self.__class__, self.unique_id, self._children,
280 self.major, self.minor, self.dev_path))
283 class LogicalVolume(BlockDev):
284 """Logical Volume block device.
287 def __init__(self, unique_id, children):
288 """Attaches to a LV device.
290 The unique_id is a tuple (vg_name, lv_name)
293 super(LogicalVolume, self).__init__(unique_id, children)
294 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
295 raise ValueError("Invalid configuration data %s" % str(unique_id))
296 self._vg_name, self._lv_name = unique_id
297 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
301 def Create(cls, unique_id, children, size):
302 """Create a new logical volume.
305 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
306 raise ValueError("Invalid configuration data %s" % str(unique_id))
307 vg_name, lv_name = unique_id
308 pvs_info = cls.GetPVInfo(vg_name)
310 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
315 pvlist = [ pv[1] for pv in pvs_info ]
316 free_size = sum([ pv[0] for pv in pvs_info ])
318 # The size constraint should have been checked from the master before
319 # calling the create function.
321 raise errors.BlockDeviceError("Not enough free space: required %s,"
322 " available %s" % (size, free_size))
323 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
326 raise errors.BlockDeviceError(result.fail_reason)
327 return LogicalVolume(unique_id, children)
330 def GetPVInfo(vg_name):
331 """Get the free space info for PVs in a volume group.
334 vg_name: the volume group name
337 list of (free_space, name) with free_space in mebibytes
340 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
341 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
343 result = utils.RunCmd(command)
345 logger.Error("Can't get the PV information: %s" % result.fail_reason)
348 for line in result.stdout.splitlines():
349 fields = line.strip().split(':')
351 logger.Error("Can't parse pvs output: line '%s'" % line)
353 # skip over pvs from another vg or ones which are not allocatable
354 if fields[1] != vg_name or fields[3][0] != 'a':
356 data.append((float(fields[2]), fields[0]))
361 """Remove this logical volume.
364 if not self.minor and not self.Attach():
365 # the LV does not exist
367 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
368 (self._vg_name, self._lv_name)])
370 logger.Error("Can't lvremove: %s" % result.fail_reason)
372 return not result.failed
374 def Rename(self, new_id):
375 """Rename this logical volume.
378 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
379 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
380 new_vg, new_name = new_id
381 if new_vg != self._vg_name:
382 raise errors.ProgrammerError("Can't move a logical volume across"
383 " volume groups (from %s to to %s)" %
384 (self._vg_name, new_vg))
385 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
387 raise errors.BlockDeviceError("Failed to rename the logical volume: %s" %
389 self._lv_name = new_name
390 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
394 """Attach to an existing LV.
396 This method will try to see if an existing and active LV exists
397 which matches the our name. If so, its major/minor will be
401 result = utils.RunCmd(["lvdisplay", self.dev_path])
403 logger.Error("Can't find LV %s: %s, %s" %
404 (self.dev_path, result.fail_reason, result.output))
406 match = re.compile("^ *Block device *([0-9]+):([0-9]+).*$")
407 for line in result.stdout.splitlines():
408 match_result = match.match(line)
410 self.major = int(match_result.group(1))
411 self.minor = int(match_result.group(2))
416 """Assemble the device.
418 We alway run `lvchange -ay` on the LV to ensure it's active before
419 use, as there were cases when xenvg was not active after boot
420 (also possibly after disk issues).
423 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
425 logger.Error("Can't activate lv %s: %s" % (self.dev_path, result.output))
426 return not result.failed
429 """Shutdown the device.
431 This is a no-op for the LV device type, as we don't deactivate the
438 """Return the status of the device.
440 Logical volumes will can be in all four states, although we don't
441 deactivate (lvchange -an) them when shutdown, so STATUS_EXISTING
442 should not be seen for our devices.
445 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
447 logger.Error("Can't display lv: %s" % result.fail_reason)
448 return self.STATUS_UNKNOWN
449 out = result.stdout.strip()
450 # format: type/permissions/alloc/fixed_minor/state/open
452 return self.STATUS_UNKNOWN
453 #writable = (out[1] == "w")
454 active = (out[4] == "a")
455 online = (out[5] == "o")
457 retval = self.STATUS_ONLINE
459 retval = self.STATUS_STANDBY
461 retval = self.STATUS_EXISTING
465 def GetSyncStatus(self):
466 """Returns the sync status of the device.
468 If this device is a mirroring device, this function returns the
469 status of the mirror.
472 (sync_percent, estimated_time, is_degraded, ldisk)
474 For logical volumes, sync_percent and estimated_time are always
475 None (no recovery in progress, as we don't handle the mirrored LV
476 case). The is_degraded parameter is the inverse of the ldisk
479 For the ldisk parameter, we check if the logical volume has the
480 'virtual' type, which means it's not backed by existing storage
481 anymore (read from it return I/O error). This happens after a
482 physical disk failure and subsequent 'vgreduce --removemissing' on
486 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
488 logger.Error("Can't display lv: %s" % result.fail_reason)
489 return None, None, True, True
490 out = result.stdout.strip()
491 # format: type/permissions/alloc/fixed_minor/state/open
493 logger.Debug("Error in lvs output: attrs=%s, len != 6" % out)
494 return None, None, True, True
495 ldisk = out[0] == 'v' # virtual volume, i.e. doesn't have
497 return None, None, ldisk, ldisk
499 def Open(self, force=False):
500 """Make the device ready for I/O.
502 This is a no-op for the LV device type.
508 """Notifies that the device will no longer be used for I/O.
510 This is a no-op for the LV device type.
515 def Snapshot(self, size):
516 """Create a snapshot copy of an lvm block device.
519 snap_name = self._lv_name + ".snap"
521 # remove existing snapshot if found
522 snap = LogicalVolume((self._vg_name, snap_name), None)
525 pvs_info = self.GetPVInfo(self._vg_name)
527 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
531 free_size, pv_name = pvs_info[0]
533 raise errors.BlockDeviceError("Not enough free space: required %s,"
534 " available %s" % (size, free_size))
536 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
537 "-n%s" % snap_name, self.dev_path])
539 raise errors.BlockDeviceError("command: %s error: %s" %
540 (result.cmd, result.fail_reason))
544 def SetInfo(self, text):
545 """Update metadata with info text.
548 BlockDev.SetInfo(self, text)
550 # Replace invalid characters
551 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
552 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
554 # Only up to 128 characters are allowed
557 result = utils.RunCmd(["lvchange", "--addtag", text,
560 raise errors.BlockDeviceError("Command: %s error: %s" %
561 (result.cmd, result.fail_reason))
564 class MDRaid1(BlockDev):
565 """raid1 device implemented via md.
568 def __init__(self, unique_id, children):
569 super(MDRaid1, self).__init__(unique_id, children)
574 """Find an array which matches our config and attach to it.
576 This tries to find a MD array which has the same UUID as our own.
579 minor = self._FindMDByUUID(self.unique_id)
580 if minor is not None:
581 self._SetFromMinor(minor)
586 return (minor is not None)
590 """Compute the list of in-use MD devices.
592 It doesn't matter if the used device have other raid level, just
593 that they are in use.
596 mdstat = open("/proc/mdstat", "r")
597 data = mdstat.readlines()
601 valid_line = re.compile("^md([0-9]+) : .*$")
603 match = valid_line.match(line)
605 md_no = int(match.group(1))
606 used_md[md_no] = line
611 def _GetDevInfo(minor):
612 """Get info about a MD device.
614 Currently only uuid is returned.
617 result = utils.RunCmd(["mdadm", "-D", "/dev/md%d" % minor])
619 logger.Error("Can't display md: %s" % result.fail_reason)
622 for line in result.stdout.splitlines():
624 kv = line.split(" : ", 1)
627 retval["uuid"] = kv[1].split()[0]
628 elif kv[0] == "State":
629 retval["state"] = kv[1].split(", ")
633 def _FindUnusedMinor():
634 """Compute an unused MD minor.
636 This code assumes that there are 256 minors only.
639 used_md = MDRaid1._GetUsedDevs()
646 logger.Error("Critical: Out of md minor numbers.")
647 raise errors.BlockDeviceError("Can't find a free MD minor")
651 def _FindMDByUUID(cls, uuid):
652 """Find the minor of an MD array with a given UUID.
655 md_list = cls._GetUsedDevs()
656 for minor in md_list:
657 info = cls._GetDevInfo(minor)
658 if info and info["uuid"] == uuid:
663 def _ZeroSuperblock(dev_path):
664 """Zero the possible locations for an MD superblock.
666 The zero-ing can't be done via ``mdadm --zero-superblock`` as that
667 fails in versions 2.x with the same error code as non-writable
670 The superblocks are located at (negative values are relative to
671 the end of the block device):
672 - -128k to end for version 0.90 superblock
673 - -8k to -12k for version 1.0 superblock (included in the above)
674 - 0k to 4k for version 1.1 superblock
675 - 4k to 8k for version 1.2 superblock
677 To cover all situations, the zero-ing will be:
681 As such, the minimum device size must be 128k, otherwise we'll get
684 Note that this function depends on the fact that one can open,
685 read and write block devices normally.
688 overwrite_size = 128 * 1024
689 empty_buf = '\0' * overwrite_size
690 fd = open(dev_path, "r+")
696 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
697 fd.seek(-overwrite_size, 2)
701 logger.Debug("Zeroed %s from %d to %d" % (dev_path, p1, p2))
706 def Create(cls, unique_id, children, size):
707 """Create a new MD raid1 array.
710 if not isinstance(children, (tuple, list)):
711 raise ValueError("Invalid setup data for MDRaid1 dev: %s" %
714 if not isinstance(i, BlockDev):
715 raise ValueError("Invalid member in MDRaid1 dev: %s" % type(i))
718 cls._ZeroSuperblock(i.dev_path)
719 except EnvironmentError, err:
720 logger.Error("Can't zero superblock for %s: %s" %
721 (i.dev_path, str(err)))
723 minor = cls._FindUnusedMinor()
724 result = utils.RunCmd(["mdadm", "--create", "/dev/md%d" % minor,
725 "--auto=yes", "--force", "-l1",
726 "-n%d" % len(children)] +
727 [dev.dev_path for dev in children])
730 logger.Error("Can't create md: %s: %s" % (result.fail_reason,
733 info = cls._GetDevInfo(minor)
734 if not info or not "uuid" in info:
735 logger.Error("Wrong information returned from mdadm -D: %s" % str(info))
737 return MDRaid1(info["uuid"], children)
740 """Stub remove function for MD RAID 1 arrays.
742 We don't remove the superblock right now. Mark a to do.
745 #TODO: maybe zero superblock on child devices?
746 return self.Shutdown()
748 def Rename(self, new_id):
751 This is not supported for md raid1 devices.
754 raise errors.ProgrammerError("Can't rename a md raid1 device")
756 def AddChildren(self, devices):
757 """Add new member(s) to the md raid1.
760 if self.minor is None and not self.Attach():
761 raise errors.BlockDeviceError("Can't attach to device")
763 args = ["mdadm", "-a", self.dev_path]
765 if dev.dev_path is None:
766 raise errors.BlockDeviceError("Child '%s' is not initialised" % dev)
768 args.append(dev.dev_path)
769 result = utils.RunCmd(args)
771 raise errors.BlockDeviceError("Failed to add new device to array: %s" %
773 new_len = len(self._children) + len(devices)
774 result = utils.RunCmd(["mdadm", "--grow", self.dev_path, "-n", new_len])
776 raise errors.BlockDeviceError("Can't grow md array: %s" %
778 self._children.extend(devices)
780 def RemoveChildren(self, devices):
781 """Remove member(s) from the md raid1.
784 if self.minor is None and not self.Attach():
785 raise errors.BlockDeviceError("Can't attach to device")
786 new_len = len(self._children) - len(devices)
788 raise errors.BlockDeviceError("Can't reduce to less than one child")
789 args = ["mdadm", "-f", self.dev_path]
793 for c in self._children:
794 if c.dev_path == dev:
798 raise errors.BlockDeviceError("Can't find device '%s' for removal" %
800 result = utils.RunCmd(args)
802 raise errors.BlockDeviceError("Failed to mark device(s) as failed: %s" %
805 # it seems here we need a short delay for MD to update its
809 result = utils.RunCmd(args)
811 raise errors.BlockDeviceError("Failed to remove device(s) from array:"
812 " %s" % result.output)
813 result = utils.RunCmd(["mdadm", "--grow", "--force", self.dev_path,
816 raise errors.BlockDeviceError("Can't shrink md array: %s" %
818 for dev in orig_devs:
819 self._children.remove(dev)
822 """Return the status of the device.
826 if self.minor is None:
827 retval = self.STATUS_UNKNOWN
829 retval = self.STATUS_ONLINE
832 def _SetFromMinor(self, minor):
833 """Set our parameters based on the given minor.
835 This sets our minor variable and our dev_path.
839 self.dev_path = "/dev/md%d" % minor
842 """Assemble the MD device.
844 At this point we should have:
845 - list of children devices
849 result = super(MDRaid1, self).Assemble()
852 md_list = self._GetUsedDevs()
853 for minor in md_list:
854 info = self._GetDevInfo(minor)
855 if info and info["uuid"] == self.unique_id:
856 self._SetFromMinor(minor)
857 logger.Info("MD array %s already started" % str(self))
859 free_minor = self._FindUnusedMinor()
860 result = utils.RunCmd(["mdadm", "-A", "--auto=yes", "--uuid",
861 self.unique_id, "/dev/md%d" % free_minor] +
862 [bdev.dev_path for bdev in self._children])
864 logger.Error("Can't assemble MD array: %s: %s" %
865 (result.fail_reason, result.output))
868 self.minor = free_minor
869 return not result.failed
872 """Tear down the MD array.
874 This does a 'mdadm --stop' so after this command, the array is no
878 if self.minor is None and not self.Attach():
879 logger.Info("MD object not attached to a device")
882 result = utils.RunCmd(["mdadm", "--stop", "/dev/md%d" % self.minor])
884 logger.Error("Can't stop MD array: %s" % result.fail_reason)
890 def SetSyncSpeed(self, kbytes):
891 """Set the maximum sync speed for the MD array.
894 result = super(MDRaid1, self).SetSyncSpeed(kbytes)
895 if self.minor is None:
896 logger.Error("MD array not attached to a device")
898 f = open("/sys/block/md%d/md/sync_speed_max" % self.minor, "w")
900 f.write("%d" % kbytes)
903 f = open("/sys/block/md%d/md/sync_speed_min" % self.minor, "w")
905 f.write("%d" % (kbytes/2))
910 def GetSyncStatus(self):
911 """Returns the sync status of the device.
914 (sync_percent, estimated_time, is_degraded, ldisk)
916 If sync_percent is None, it means all is ok
917 If estimated_time is None, it means we can't esimate
918 the time needed, otherwise it's the time left in seconds.
920 The ldisk parameter is always true for MD devices.
923 if self.minor is None and not self.Attach():
924 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
925 dev_info = self._GetDevInfo(self.minor)
926 is_clean = ("state" in dev_info and
927 len(dev_info["state"]) == 1 and
928 dev_info["state"][0] in ("clean", "active"))
929 sys_path = "/sys/block/md%s/md/" % self.minor
930 f = file(sys_path + "sync_action")
931 sync_status = f.readline().strip()
933 if sync_status == "idle":
934 return None, None, not is_clean, False
935 f = file(sys_path + "sync_completed")
936 sync_completed = f.readline().strip().split(" / ")
938 if len(sync_completed) != 2:
939 return 0, None, not is_clean, False
940 sync_done, sync_total = [float(i) for i in sync_completed]
941 sync_percent = 100.0*sync_done/sync_total
942 f = file(sys_path + "sync_speed")
943 sync_speed_k = int(f.readline().strip())
944 if sync_speed_k == 0:
947 time_est = (sync_total - sync_done) / 2 / sync_speed_k
948 return sync_percent, time_est, not is_clean, False
950 def Open(self, force=False):
951 """Make the device ready for I/O.
953 This is a no-op for the MDRaid1 device type, although we could use
954 the 2.6.18's new array_state thing.
960 """Notifies that the device will no longer be used for I/O.
962 This is a no-op for the MDRaid1 device type, but see comment for
969 class BaseDRBD(BlockDev):
972 This class contains a few bits of common functionality between the
973 0.7 and 8.x versions of DRBD.
976 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
977 r" \(api:(\d+)/proto:(\d+)\)")
979 _ST_UNCONFIGURED = "Unconfigured"
980 _ST_WFCONNECTION = "WFConnection"
981 _ST_CONNECTED = "Connected"
985 """Return data from /proc/drbd.
988 stat = open("/proc/drbd", "r")
990 data = stat.read().splitlines()
994 raise errors.BlockDeviceError("Can't read any data from /proc/drbd")
998 def _MassageProcData(data):
999 """Transform the output of _GetProdData into a nicer form.
1002 a dictionary of minor: joined lines from /proc/drbd for that minor
1005 lmatch = re.compile("^ *([0-9]+):.*$")
1007 old_minor = old_line = None
1009 lresult = lmatch.match(line)
1010 if lresult is not None:
1011 if old_minor is not None:
1012 results[old_minor] = old_line
1013 old_minor = int(lresult.group(1))
1016 if old_minor is not None:
1017 old_line += " " + line.strip()
1019 if old_minor is not None:
1020 results[old_minor] = old_line
1024 def _GetVersion(cls):
1025 """Return the DRBD version.
1027 This will return a list [k_major, k_minor, k_point, api, proto].
1030 proc_data = cls._GetProcData()
1031 first_line = proc_data[0].strip()
1032 version = cls._VERSION_RE.match(first_line)
1034 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
1036 return [int(val) for val in version.groups()]
1039 def _DevPath(minor):
1040 """Return the path to a drbd device for a given minor.
1043 return "/dev/drbd%d" % minor
1046 def _GetUsedDevs(cls):
1047 """Compute the list of used DRBD devices.
1050 data = cls._GetProcData()
1053 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
1055 match = valid_line.match(line)
1058 minor = int(match.group(1))
1059 state = match.group(2)
1060 if state == cls._ST_UNCONFIGURED:
1062 used_devs[minor] = state, line
1066 def _SetFromMinor(self, minor):
1067 """Set our parameters based on the given minor.
1069 This sets our minor variable and our dev_path.
1073 self.minor = self.dev_path = None
1076 self.dev_path = self._DevPath(minor)
1079 def _CheckMetaSize(meta_device):
1080 """Check if the given meta device looks like a valid one.
1082 This currently only check the size, which must be around
1086 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1088 logger.Error("Failed to get device size: %s" % result.fail_reason)
1091 sectors = int(result.stdout)
1093 logger.Error("Invalid output from blockdev: '%s'" % result.stdout)
1095 bytes = sectors * 512
1096 if bytes < 128 * 1024 * 1024: # less than 128MiB
1097 logger.Error("Meta device too small (%.2fMib)" % (bytes / 1024 / 1024))
1099 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
1100 logger.Error("Meta device too big (%.2fMiB)" % (bytes / 1024 / 1024))
1104 def Rename(self, new_id):
1107 This is not supported for drbd devices.
1110 raise errors.ProgrammerError("Can't rename a drbd device")
1113 class DRBDev(BaseDRBD):
1114 """DRBD block device.
1116 This implements the local host part of the DRBD device, i.e. it
1117 doesn't do anything to the supposed peer. If you need a fully
1118 connected DRBD pair, you need to use this class on both hosts.
1120 The unique_id for the drbd device is the (local_ip, local_port,
1121 remote_ip, remote_port) tuple, and it must have two children: the
1122 data device and the meta_device. The meta device is checked for
1123 valid size and is zeroed on create.
1126 def __init__(self, unique_id, children):
1127 super(DRBDev, self).__init__(unique_id, children)
1128 self.major = self._DRBD_MAJOR
1129 [kmaj, kmin, kfix, api, proto] = self._GetVersion()
1130 if kmaj != 0 and kmin != 7:
1131 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
1132 " requested ganeti usage: kernel is"
1133 " %s.%s, ganeti wants 0.7" % (kmaj, kmin))
1135 if len(children) != 2:
1136 raise ValueError("Invalid configuration data %s" % str(children))
1137 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
1138 raise ValueError("Invalid configuration data %s" % str(unique_id))
1139 self._lhost, self._lport, self._rhost, self._rport = unique_id
1143 def _FindUnusedMinor(cls):
1144 """Find an unused DRBD device.
1147 data = cls._GetProcData()
1149 valid_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
1151 match = valid_line.match(line)
1153 return int(match.group(1))
1154 logger.Error("Error: no free drbd minors!")
1155 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1158 def _GetDevInfo(cls, minor):
1159 """Get details about a given DRBD minor.
1161 This return, if available, the local backing device in (major,
1162 minor) formant and the local and remote (ip, port) information.
1166 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1168 logger.Error("Can't display the drbd config: %s" % result.fail_reason)
1171 if out == "Not configured\n":
1173 for line in out.splitlines():
1174 if "local_dev" not in data:
1175 match = re.match("^Lower device: ([0-9]+):([0-9]+) .*$", line)
1177 data["local_dev"] = (int(match.group(1)), int(match.group(2)))
1179 if "meta_dev" not in data:
1180 match = re.match("^Meta device: (([0-9]+):([0-9]+)|internal).*$", line)
1182 if match.group(2) is not None and match.group(3) is not None:
1183 # matched on the major/minor
1184 data["meta_dev"] = (int(match.group(2)), int(match.group(3)))
1186 # matched on the "internal" string
1187 data["meta_dev"] = match.group(1)
1188 # in this case, no meta_index is in the output
1189 data["meta_index"] = -1
1191 if "meta_index" not in data:
1192 match = re.match("^Meta index: ([0-9]+).*$", line)
1194 data["meta_index"] = int(match.group(1))
1196 if "local_addr" not in data:
1197 match = re.match("^Local address: ([0-9.]+):([0-9]+)$", line)
1199 data["local_addr"] = (match.group(1), int(match.group(2)))
1201 if "remote_addr" not in data:
1202 match = re.match("^Remote address: ([0-9.]+):([0-9]+)$", line)
1204 data["remote_addr"] = (match.group(1), int(match.group(2)))
1208 def _MatchesLocal(self, info):
1209 """Test if our local config matches with an existing device.
1211 The parameter should be as returned from `_GetDevInfo()`. This
1212 method tests if our local backing device is the same as the one in
1213 the info parameter, in effect testing if we look like the given
1217 if not ("local_dev" in info and "meta_dev" in info and
1218 "meta_index" in info):
1221 backend = self._children[0]
1222 if backend is not None:
1223 retval = (info["local_dev"] == (backend.major, backend.minor))
1225 retval = (info["local_dev"] == (0, 0))
1226 meta = self._children[1]
1227 if meta is not None:
1228 retval = retval and (info["meta_dev"] == (meta.major, meta.minor))
1229 retval = retval and (info["meta_index"] == 0)
1231 retval = retval and (info["meta_dev"] == "internal" and
1232 info["meta_index"] == -1)
1235 def _MatchesNet(self, info):
1236 """Test if our network config matches with an existing device.
1238 The parameter should be as returned from `_GetDevInfo()`. This
1239 method tests if our network configuration is the same as the one
1240 in the info parameter, in effect testing if we look like the given
1244 if (((self._lhost is None and not ("local_addr" in info)) and
1245 (self._rhost is None and not ("remote_addr" in info)))):
1248 if self._lhost is None:
1251 if not ("local_addr" in info and
1252 "remote_addr" in info):
1255 retval = (info["local_addr"] == (self._lhost, self._lport))
1256 retval = (retval and
1257 info["remote_addr"] == (self._rhost, self._rport))
1261 def _AssembleLocal(cls, minor, backend, meta):
1262 """Configure the local part of a DRBD device.
1264 This is the first thing that must be done on an unconfigured DRBD
1265 device. And it must be done only once.
1268 if not cls._CheckMetaSize(meta):
1270 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1271 backend, meta, "0", "-e", "detach"])
1273 logger.Error("Can't attach local disk: %s" % result.output)
1274 return not result.failed
1277 def _ShutdownLocal(cls, minor):
1278 """Detach from the local device.
1280 I/Os will continue to be served from the remote device. If we
1281 don't have a remote device, this operation will fail.
1284 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1286 logger.Error("Can't detach local device: %s" % result.output)
1287 return not result.failed
1290 def _ShutdownAll(minor):
1291 """Deactivate the device.
1293 This will, of course, fail if the device is in use.
1296 result = utils.RunCmd(["drbdsetup", DRBDev._DevPath(minor), "down"])
1298 logger.Error("Can't shutdown drbd device: %s" % result.output)
1299 return not result.failed
1302 def _AssembleNet(cls, minor, net_info, protocol):
1303 """Configure the network part of the device.
1305 This operation can be, in theory, done multiple times, but there
1306 have been cases (in lab testing) in which the network part of the
1307 device had become stuck and couldn't be shut down because activity
1308 from the new peer (also stuck) triggered a timer re-init and
1309 needed remote peer interface shutdown in order to clear. So please
1310 don't change online the net config.
1313 lhost, lport, rhost, rport = net_info
1314 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "net",
1315 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport),
1318 logger.Error("Can't setup network for dbrd device: %s" %
1322 timeout = time.time() + 10
1324 while time.time() < timeout:
1325 info = cls._GetDevInfo(minor)
1326 if not "local_addr" in info or not "remote_addr" in info:
1329 if (info["local_addr"] != (lhost, lport) or
1330 info["remote_addr"] != (rhost, rport)):
1336 logger.Error("Timeout while configuring network")
1341 def _ShutdownNet(cls, minor):
1342 """Disconnect from the remote peer.
1344 This fails if we don't have a local device.
1347 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1349 logger.Error("Can't shutdown network: %s" % result.output)
1350 return not result.failed
1353 """Assemble the drbd.
1356 - if we have a local backing device, we bind to it by:
1357 - checking the list of used drbd devices
1358 - check if the local minor use of any of them is our own device
1361 - if we have a local/remote net info:
1362 - redo the local backing device step for the remote device
1363 - check if any drbd device is using the local port,
1365 - check if any remote drbd device is using the remote
1366 port, if yes abort (for now)
1368 - bind the remote net port
1372 if self.minor is not None:
1373 logger.Info("Already assembled")
1376 result = super(DRBDev, self).Assemble()
1380 minor = self._FindUnusedMinor()
1381 need_localdev_teardown = False
1382 if self._children[0]:
1383 result = self._AssembleLocal(minor, self._children[0].dev_path,
1384 self._children[1].dev_path)
1387 need_localdev_teardown = True
1388 if self._lhost and self._lport and self._rhost and self._rport:
1389 result = self._AssembleNet(minor,
1390 (self._lhost, self._lport,
1391 self._rhost, self._rport),
1394 if need_localdev_teardown:
1395 # we will ignore failures from this
1396 logger.Error("net setup failed, tearing down local device")
1397 self._ShutdownAll(minor)
1399 self._SetFromMinor(minor)
1403 """Shutdown the DRBD device.
1406 if self.minor is None and not self.Attach():
1407 logger.Info("DRBD device not attached to a device during Shutdown")
1409 if not self._ShutdownAll(self.minor):
1412 self.dev_path = None
1416 """Find a DRBD device which matches our config and attach to it.
1418 In case of partially attached (local device matches but no network
1419 setup), we perform the network attach. If successful, we re-test
1420 the attach if can return success.
1423 for minor in self._GetUsedDevs():
1424 info = self._GetDevInfo(minor)
1425 match_l = self._MatchesLocal(info)
1426 match_r = self._MatchesNet(info)
1427 if match_l and match_r:
1429 if match_l and not match_r and "local_addr" not in info:
1430 res_r = self._AssembleNet(minor,
1431 (self._lhost, self._lport,
1432 self._rhost, self._rport),
1434 if res_r and self._MatchesNet(self._GetDevInfo(minor)):
1439 self._SetFromMinor(minor)
1440 return minor is not None
1442 def Open(self, force=False):
1443 """Make the local state primary.
1445 If the 'force' parameter is given, the '--do-what-I-say' parameter
1446 is given. Since this is a pottentialy dangerous operation, the
1447 force flag should be only given after creation, when it actually
1451 if self.minor is None and not self.Attach():
1452 logger.Error("DRBD cannot attach to a device during open")
1454 cmd = ["drbdsetup", self.dev_path, "primary"]
1456 cmd.append("--do-what-I-say")
1457 result = utils.RunCmd(cmd)
1459 logger.Error("Can't make drbd device primary: %s" % result.output)
1464 """Make the local state secondary.
1466 This will, of course, fail if the device is in use.
1469 if self.minor is None and not self.Attach():
1470 logger.Info("Instance not attached to a device")
1471 raise errors.BlockDeviceError("Can't find device")
1472 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1474 logger.Error("Can't switch drbd device to secondary: %s" % result.output)
1475 raise errors.BlockDeviceError("Can't switch drbd device to secondary")
1477 def SetSyncSpeed(self, kbytes):
1478 """Set the speed of the DRBD syncer.
1481 children_result = super(DRBDev, self).SetSyncSpeed(kbytes)
1482 if self.minor is None:
1483 logger.Info("Instance not attached to a device")
1485 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1488 logger.Error("Can't change syncer rate: %s " % result.fail_reason)
1489 return not result.failed and children_result
1491 def GetSyncStatus(self):
1492 """Returns the sync status of the device.
1495 (sync_percent, estimated_time, is_degraded, ldisk)
1497 If sync_percent is None, it means all is ok
1498 If estimated_time is None, it means we can't esimate
1499 the time needed, otherwise it's the time left in seconds.
1501 The ldisk parameter will be returned as True, since the DRBD7
1502 devices have not been converted.
1505 if self.minor is None and not self.Attach():
1506 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1507 proc_info = self._MassageProcData(self._GetProcData())
1508 if self.minor not in proc_info:
1509 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1511 line = proc_info[self.minor]
1512 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
1513 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
1515 sync_percent = float(match.group(1))
1516 hours = int(match.group(2))
1517 minutes = int(match.group(3))
1518 seconds = int(match.group(4))
1519 est_time = hours * 3600 + minutes * 60 + seconds
1523 match = re.match("^ *[0-9]+: cs:([^ ]+).*$", line)
1525 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
1527 client_state = match.group(1)
1528 is_degraded = client_state != "Connected"
1529 return sync_percent, est_time, is_degraded, False
1531 def GetStatus(self):
1532 """Compute the status of the DRBD device
1534 Note that DRBD devices don't have the STATUS_EXISTING state.
1537 if self.minor is None and not self.Attach():
1538 return self.STATUS_UNKNOWN
1540 data = self._GetProcData()
1541 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
1544 mresult = match.match(line)
1548 logger.Error("Can't find myself!")
1549 return self.STATUS_UNKNOWN
1551 state = mresult.group(2)
1552 if state == "Primary":
1553 result = self.STATUS_ONLINE
1555 result = self.STATUS_STANDBY
1560 def _ZeroDevice(device):
1563 This writes until we get ENOSPC.
1566 f = open(device, "w")
1567 buf = "\0" * 1048576
1571 except IOError, err:
1572 if err.errno != errno.ENOSPC:
1576 def Create(cls, unique_id, children, size):
1577 """Create a new DRBD device.
1579 Since DRBD devices are not created per se, just assembled, this
1580 function just zeroes the meta device.
1583 if len(children) != 2:
1584 raise errors.ProgrammerError("Invalid setup for the drbd device")
1587 if not meta.Attach():
1588 raise errors.BlockDeviceError("Can't attach to meta device")
1589 if not cls._CheckMetaSize(meta.dev_path):
1590 raise errors.BlockDeviceError("Invalid meta device")
1591 logger.Info("Started zeroing device %s" % meta.dev_path)
1592 cls._ZeroDevice(meta.dev_path)
1593 logger.Info("Done zeroing device %s" % meta.dev_path)
1594 return cls(unique_id, children)
1597 """Stub remove for DRBD devices.
1600 return self.Shutdown()
1603 class DRBD8(BaseDRBD):
1604 """DRBD v8.x block device.
1606 This implements the local host part of the DRBD device, i.e. it
1607 doesn't do anything to the supposed peer. If you need a fully
1608 connected DRBD pair, you need to use this class on both hosts.
1610 The unique_id for the drbd device is the (local_ip, local_port,
1611 remote_ip, remote_port) tuple, and it must have two children: the
1612 data device and the meta_device. The meta device is checked for
1613 valid size and is zeroed on create.
1619 def __init__(self, unique_id, children):
1620 if children and children.count(None) > 0:
1622 super(DRBD8, self).__init__(unique_id, children)
1623 self.major = self._DRBD_MAJOR
1624 [kmaj, kmin, kfix, api, proto] = self._GetVersion()
1626 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
1627 " requested ganeti usage: kernel is"
1628 " %s.%s, ganeti wants 8.x" % (kmaj, kmin))
1630 if len(children) not in (0, 2):
1631 raise ValueError("Invalid configuration data %s" % str(children))
1632 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
1633 raise ValueError("Invalid configuration data %s" % str(unique_id))
1634 self._lhost, self._lport, self._rhost, self._rport = unique_id
1638 def _InitMeta(cls, minor, dev_path):
1639 """Initialize a meta device.
1641 This will not work if the given minor is in use.
1644 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
1645 "v08", dev_path, "0", "create-md"])
1647 raise errors.BlockDeviceError("Can't initialize meta device: %s" %
1651 def _FindUnusedMinor(cls):
1652 """Find an unused DRBD device.
1654 This is specific to 8.x as the minors are allocated dynamically,
1655 so non-existing numbers up to a max minor count are actually free.
1658 data = cls._GetProcData()
1660 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
1661 used_line = re.compile("^ *([0-9]+): cs:")
1664 match = unused_line.match(line)
1666 return int(match.group(1))
1667 match = used_line.match(line)
1669 minor = int(match.group(1))
1670 highest = max(highest, minor)
1671 if highest is None: # there are no minors in use at all
1673 if highest >= cls._MAX_MINORS:
1674 logger.Error("Error: no free drbd minors!")
1675 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1679 def _IsValidMeta(cls, meta_device):
1680 """Check if the given meta device looks like a valid one.
1683 minor = cls._FindUnusedMinor()
1684 minor_path = cls._DevPath(minor)
1685 result = utils.RunCmd(["drbdmeta", minor_path,
1686 "v08", meta_device, "0",
1689 logger.Error("Invalid meta device %s: %s" % (meta_device, result.output))
1694 def _GetShowParser(cls):
1695 """Return a parser for `drbd show` output.
1697 This will either create or return an already-create parser for the
1698 output of the command `drbd show`.
1701 if cls._PARSE_SHOW is not None:
1702 return cls._PARSE_SHOW
1705 lbrace = pyp.Literal("{").suppress()
1706 rbrace = pyp.Literal("}").suppress()
1707 semi = pyp.Literal(";").suppress()
1708 # this also converts the value to an int
1709 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
1711 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
1712 defa = pyp.Literal("_is_default").suppress()
1713 dbl_quote = pyp.Literal('"').suppress()
1715 keyword = pyp.Word(pyp.alphanums + '-')
1718 value = pyp.Word(pyp.alphanums + '_-/.:')
1719 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
1720 addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() +
1722 # meta device, extended syntax
1723 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
1724 number + pyp.Word(']').suppress())
1727 stmt = (~rbrace + keyword + ~lbrace +
1728 (addr_port ^ value ^ quoted ^ meta_value) +
1729 pyp.Optional(defa) + semi +
1730 pyp.Optional(pyp.restOfLine).suppress())
1733 section_name = pyp.Word(pyp.alphas + '_')
1734 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
1736 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
1739 cls._PARSE_SHOW = bnf
1744 def _GetShowData(cls, minor):
1745 """Return the `drbdsetup show` data for a minor.
1748 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1750 logger.Error("Can't display the drbd config: %s" % result.fail_reason)
1752 return result.stdout
1755 def _GetDevInfo(cls, out):
1756 """Parse details about a given DRBD minor.
1758 This return, if available, the local backing device (as a path)
1759 and the local and remote (ip, port) information from a string
1760 containing the output of the `drbdsetup show` command as returned
1768 bnf = cls._GetShowParser()
1772 results = bnf.parseString(out)
1773 except pyp.ParseException, err:
1774 raise errors.BlockDeviceError("Can't parse drbdsetup show output: %s" %
1777 # and massage the results into our desired format
1778 for section in results:
1780 if sname == "_this_host":
1781 for lst in section[1:]:
1782 if lst[0] == "disk":
1783 data["local_dev"] = lst[1]
1784 elif lst[0] == "meta-disk":
1785 data["meta_dev"] = lst[1]
1786 data["meta_index"] = lst[2]
1787 elif lst[0] == "address":
1788 data["local_addr"] = tuple(lst[1:])
1789 elif sname == "_remote_host":
1790 for lst in section[1:]:
1791 if lst[0] == "address":
1792 data["remote_addr"] = tuple(lst[1:])
1795 def _MatchesLocal(self, info):
1796 """Test if our local config matches with an existing device.
1798 The parameter should be as returned from `_GetDevInfo()`. This
1799 method tests if our local backing device is the same as the one in
1800 the info parameter, in effect testing if we look like the given
1805 backend, meta = self._children
1807 backend = meta = None
1809 if backend is not None:
1810 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
1812 retval = ("local_dev" not in info)
1814 if meta is not None:
1815 retval = retval and ("meta_dev" in info and
1816 info["meta_dev"] == meta.dev_path)
1817 retval = retval and ("meta_index" in info and
1818 info["meta_index"] == 0)
1820 retval = retval and ("meta_dev" not in info and
1821 "meta_index" not in info)
1824 def _MatchesNet(self, info):
1825 """Test if our network config matches with an existing device.
1827 The parameter should be as returned from `_GetDevInfo()`. This
1828 method tests if our network configuration is the same as the one
1829 in the info parameter, in effect testing if we look like the given
1833 if (((self._lhost is None and not ("local_addr" in info)) and
1834 (self._rhost is None and not ("remote_addr" in info)))):
1837 if self._lhost is None:
1840 if not ("local_addr" in info and
1841 "remote_addr" in info):
1844 retval = (info["local_addr"] == (self._lhost, self._lport))
1845 retval = (retval and
1846 info["remote_addr"] == (self._rhost, self._rport))
1850 def _AssembleLocal(cls, minor, backend, meta):
1851 """Configure the local part of a DRBD device.
1853 This is the first thing that must be done on an unconfigured DRBD
1854 device. And it must be done only once.
1857 if not cls._IsValidMeta(meta):
1859 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1860 backend, meta, "0", "-e", "detach",
1863 logger.Error("Can't attach local disk: %s" % result.output)
1864 return not result.failed
1867 def _AssembleNet(cls, minor, net_info, protocol,
1868 dual_pri=False, hmac=None, secret=None):
1869 """Configure the network part of the device.
1872 lhost, lport, rhost, rport = net_info
1873 if None in net_info:
1874 # we don't want network connection and actually want to make
1876 return cls._ShutdownNet(minor)
1878 args = ["drbdsetup", cls._DevPath(minor), "net",
1879 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
1880 "-A", "discard-zero-changes",
1886 args.extend(["-a", hmac, "-x", secret])
1887 result = utils.RunCmd(args)
1889 logger.Error("Can't setup network for dbrd device: %s" %
1893 timeout = time.time() + 10
1895 while time.time() < timeout:
1896 info = cls._GetDevInfo(cls._GetShowData(minor))
1897 if not "local_addr" in info or not "remote_addr" in info:
1900 if (info["local_addr"] != (lhost, lport) or
1901 info["remote_addr"] != (rhost, rport)):
1907 logger.Error("Timeout while configuring network")
1911 def AddChildren(self, devices):
1912 """Add a disk to the DRBD device.
1915 if self.minor is None:
1916 raise errors.BlockDeviceError("Can't attach to dbrd8 during AddChildren")
1917 if len(devices) != 2:
1918 raise errors.BlockDeviceError("Need two devices for AddChildren")
1919 info = self._GetDevInfo(self._GetShowData(self.minor))
1920 if "local_dev" in info:
1921 raise errors.BlockDeviceError("DRBD8 already attached to a local disk")
1922 backend, meta = devices
1923 if backend.dev_path is None or meta.dev_path is None:
1924 raise errors.BlockDeviceError("Children not ready during AddChildren")
1927 if not self._CheckMetaSize(meta.dev_path):
1928 raise errors.BlockDeviceError("Invalid meta device size")
1929 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1930 if not self._IsValidMeta(meta.dev_path):
1931 raise errors.BlockDeviceError("Cannot initalize meta device")
1933 if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path):
1934 raise errors.BlockDeviceError("Can't attach to local storage")
1935 self._children = devices
1937 def RemoveChildren(self, devices):
1938 """Detach the drbd device from local storage.
1941 if self.minor is None:
1942 raise errors.BlockDeviceError("Can't attach to drbd8 during"
1944 # early return if we don't actually have backing storage
1945 info = self._GetDevInfo(self._GetShowData(self.minor))
1946 if "local_dev" not in info:
1948 if len(self._children) != 2:
1949 raise errors.BlockDeviceError("We don't have two children: %s" %
1951 if self._children.count(None) == 2: # we don't actually have children :)
1952 logger.Error("Requested detach while detached")
1954 if len(devices) != 2:
1955 raise errors.BlockDeviceError("We need two children in RemoveChildren")
1956 for child, dev in zip(self._children, devices):
1957 if dev != child.dev_path:
1958 raise errors.BlockDeviceError("Mismatch in local storage"
1959 " (%s != %s) in RemoveChildren" %
1960 (dev, child.dev_path))
1962 if not self._ShutdownLocal(self.minor):
1963 raise errors.BlockDeviceError("Can't detach from local storage")
1966 def SetSyncSpeed(self, kbytes):
1967 """Set the speed of the DRBD syncer.
1970 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1971 if self.minor is None:
1972 logger.Info("Instance not attached to a device")
1974 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1977 logger.Error("Can't change syncer rate: %s " % result.fail_reason)
1978 return not result.failed and children_result
1980 def GetSyncStatus(self):
1981 """Returns the sync status of the device.
1984 (sync_percent, estimated_time, is_degraded)
1986 If sync_percent is None, it means all is ok
1987 If estimated_time is None, it means we can't esimate
1988 the time needed, otherwise it's the time left in seconds.
1991 We set the is_degraded parameter to True on two conditions:
1992 network not connected or local disk missing.
1994 We compute the ldisk parameter based on wheter we have a local
1998 if self.minor is None and not self.Attach():
1999 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
2000 proc_info = self._MassageProcData(self._GetProcData())
2001 if self.minor not in proc_info:
2002 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
2004 line = proc_info[self.minor]
2005 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
2006 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
2008 sync_percent = float(match.group(1))
2009 hours = int(match.group(2))
2010 minutes = int(match.group(3))
2011 seconds = int(match.group(4))
2012 est_time = hours * 3600 + minutes * 60 + seconds
2016 match = re.match("^ *\d+: cs:(\w+).*ds:(\w+)/(\w+).*$", line)
2018 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
2020 client_state = match.group(1)
2021 local_disk_state = match.group(2)
2022 ldisk = local_disk_state != "UpToDate"
2023 is_degraded = client_state != "Connected"
2024 return sync_percent, est_time, is_degraded or ldisk, ldisk
2026 def GetStatus(self):
2027 """Compute the status of the DRBD device
2029 Note that DRBD devices don't have the STATUS_EXISTING state.
2032 if self.minor is None and not self.Attach():
2033 return self.STATUS_UNKNOWN
2035 data = self._GetProcData()
2036 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
2039 mresult = match.match(line)
2043 logger.Error("Can't find myself!")
2044 return self.STATUS_UNKNOWN
2046 state = mresult.group(2)
2047 if state == "Primary":
2048 result = self.STATUS_ONLINE
2050 result = self.STATUS_STANDBY
2054 def Open(self, force=False):
2055 """Make the local state primary.
2057 If the 'force' parameter is given, the '--do-what-I-say' parameter
2058 is given. Since this is a pottentialy dangerous operation, the
2059 force flag should be only given after creation, when it actually
2063 if self.minor is None and not self.Attach():
2064 logger.Error("DRBD cannot attach to a device during open")
2066 cmd = ["drbdsetup", self.dev_path, "primary"]
2069 result = utils.RunCmd(cmd)
2071 logger.Error("Can't make drbd device primary: %s" % result.output)
2076 """Make the local state secondary.
2078 This will, of course, fail if the device is in use.
2081 if self.minor is None and not self.Attach():
2082 logger.Info("Instance not attached to a device")
2083 raise errors.BlockDeviceError("Can't find device")
2084 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
2086 logger.Error("Can't switch drbd device to secondary: %s" % result.output)
2087 raise errors.BlockDeviceError("Can't switch drbd device to secondary")
2090 """Find a DRBD device which matches our config and attach to it.
2092 In case of partially attached (local device matches but no network
2093 setup), we perform the network attach. If successful, we re-test
2094 the attach if can return success.
2097 for minor in self._GetUsedDevs():
2098 info = self._GetDevInfo(self._GetShowData(minor))
2099 match_l = self._MatchesLocal(info)
2100 match_r = self._MatchesNet(info)
2101 if match_l and match_r:
2103 if match_l and not match_r and "local_addr" not in info:
2104 res_r = self._AssembleNet(minor,
2105 (self._lhost, self._lport,
2106 self._rhost, self._rport),
2109 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
2111 # the weakest case: we find something that is only net attached
2112 # even though we were passed some children at init time
2113 if match_r and "local_dev" not in info:
2115 if match_l and not match_r and "local_addr" in info:
2116 # strange case - the device network part points to somewhere
2117 # else, even though its local storage is ours; as we own the
2118 # drbd space, we try to disconnect from the remote peer and
2119 # reconnect to our correct one
2120 if not self._ShutdownNet(minor):
2121 raise errors.BlockDeviceError("Device has correct local storage,"
2122 " wrong remote peer and is unable to"
2123 " disconnect in order to attach to"
2124 " the correct peer")
2125 # note: _AssembleNet also handles the case when we don't want
2126 # local storage (i.e. one or more of the _[lr](host|port) is
2128 if (self._AssembleNet(minor, (self._lhost, self._lport,
2129 self._rhost, self._rport), "C") and
2130 self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))):
2136 self._SetFromMinor(minor)
2137 return minor is not None
2140 """Assemble the drbd.
2143 - if we have a local backing device, we bind to it by:
2144 - checking the list of used drbd devices
2145 - check if the local minor use of any of them is our own device
2148 - if we have a local/remote net info:
2149 - redo the local backing device step for the remote device
2150 - check if any drbd device is using the local port,
2152 - check if any remote drbd device is using the remote
2153 port, if yes abort (for now)
2155 - bind the remote net port
2159 if self.minor is not None:
2160 logger.Info("Already assembled")
2163 result = super(DRBD8, self).Assemble()
2167 minor = self._FindUnusedMinor()
2168 need_localdev_teardown = False
2169 if self._children and self._children[0] and self._children[1]:
2170 result = self._AssembleLocal(minor, self._children[0].dev_path,
2171 self._children[1].dev_path)
2174 need_localdev_teardown = True
2175 if self._lhost and self._lport and self._rhost and self._rport:
2176 result = self._AssembleNet(minor,
2177 (self._lhost, self._lport,
2178 self._rhost, self._rport),
2181 if need_localdev_teardown:
2182 # we will ignore failures from this
2183 logger.Error("net setup failed, tearing down local device")
2184 self._ShutdownAll(minor)
2186 self._SetFromMinor(minor)
2190 def _ShutdownLocal(cls, minor):
2191 """Detach from the local device.
2193 I/Os will continue to be served from the remote device. If we
2194 don't have a remote device, this operation will fail.
2197 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
2199 logger.Error("Can't detach local device: %s" % result.output)
2200 return not result.failed
2203 def _ShutdownNet(cls, minor):
2204 """Disconnect from the remote peer.
2206 This fails if we don't have a local device.
2209 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
2211 logger.Error("Can't shutdown network: %s" % result.output)
2212 return not result.failed
2215 def _ShutdownAll(cls, minor):
2216 """Deactivate the device.
2218 This will, of course, fail if the device is in use.
2221 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
2223 logger.Error("Can't shutdown drbd device: %s" % result.output)
2224 return not result.failed
2227 """Shutdown the DRBD device.
2230 if self.minor is None and not self.Attach():
2231 logger.Info("DRBD device not attached to a device during Shutdown")
2233 if not self._ShutdownAll(self.minor):
2236 self.dev_path = None
2240 """Stub remove for DRBD devices.
2243 return self.Shutdown()
2246 def Create(cls, unique_id, children, size):
2247 """Create a new DRBD8 device.
2249 Since DRBD devices are not created per se, just assembled, this
2250 function only initializes the metadata.
2253 if len(children) != 2:
2254 raise errors.ProgrammerError("Invalid setup for the drbd device")
2257 if not meta.Attach():
2258 raise errors.BlockDeviceError("Can't attach to meta device")
2259 if not cls._CheckMetaSize(meta.dev_path):
2260 raise errors.BlockDeviceError("Invalid meta device size")
2261 cls._InitMeta(cls._FindUnusedMinor(), meta.dev_path)
2262 if not cls._IsValidMeta(meta.dev_path):
2263 raise errors.BlockDeviceError("Cannot initalize meta device")
2264 return cls(unique_id, children)
2268 constants.LD_LV: LogicalVolume,
2269 constants.LD_MD_R1: MDRaid1,
2270 constants.LD_DRBD7: DRBDev,
2271 constants.LD_DRBD8: DRBD8,
2275 def FindDevice(dev_type, unique_id, children):
2276 """Search for an existing, assembled device.
2278 This will succeed only if the device exists and is assembled, but it
2279 does not do any actions in order to activate the device.
2282 if dev_type not in DEV_MAP:
2283 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2284 device = DEV_MAP[dev_type](unique_id, children)
2285 if not device.Attach():
2290 def AttachOrAssemble(dev_type, unique_id, children):
2291 """Try to attach or assemble an existing device.
2293 This will attach to an existing assembled device or will assemble
2294 the device, as needed, to bring it fully up.
2297 if dev_type not in DEV_MAP:
2298 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2299 device = DEV_MAP[dev_type](unique_id, children)
2300 if not device.Attach():
2302 if not device.Attach():
2303 raise errors.BlockDeviceError("Can't find a valid block device for"
2305 (dev_type, unique_id, children))
2309 def Create(dev_type, unique_id, children, size):
2313 if dev_type not in DEV_MAP:
2314 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2315 device = DEV_MAP[dev_type].Create(unique_id, children, size)