4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
28 from ganeti import utils
29 from ganeti import logger
30 from ganeti import errors
33 class BlockDev(object):
34 """Block device abstract class.
36 A block device can be in the following states:
37 - not existing on the system, and by `Create()` it goes into:
38 - existing but not setup/not active, and by `Assemble()` goes into:
39 - active read-write and by `Open()` it goes into
40 - online (=used, or ready for use)
42 A device can also be online but read-only, however we are not using
43 the readonly state (MD and LV have it, if needed in the future)
44 and we are usually looking at this like at a stack, so it's easier
45 to conceptualise the transition from not-existing to online and back
48 The many different states of the device are due to the fact that we
49 need to cover many device types:
50 - logical volumes are created, lvchange -a y $lv, and used
51 - md arrays are created or assembled and used
52 - drbd devices are attached to a local disk/remote peer and made primary
54 The status of the device can be examined by `GetStatus()`, which
55 returns a numerical value, depending on the position in the
56 transition stack of the device.
58 A block device is identified by three items:
59 - the /dev path of the device (dynamic)
60 - a unique ID of the device (static)
61 - it's major/minor pair (dynamic)
63 Not all devices implement both the first two as distinct items. LVM
64 logical volumes have their unique ID (the pair volume group, logical
65 volume name) in a 1-to-1 relation to the dev path. For MD devices,
66 the /dev path is dynamic and the unique ID is the UUID generated at
67 array creation plus the slave list. For DRBD devices, the /dev path
68 is again dynamic and the unique id is the pair (host1, dev1),
71 You can get to a device in two ways:
72 - creating the (real) device, which returns you
73 an attached instance (lvcreate, mdadm --create)
74 - attaching of a python instance to an existing (real) device
76 The second point, the attachement to a device, is different
77 depending on whether the device is assembled or not. At init() time,
78 we search for a device with the same unique_id as us. If found,
79 good. It also means that the device is already assembled. If not,
80 after assembly we'll have our correct major/minor.
89 STATUS_UNKNOWN: "unknown",
90 STATUS_EXISTING: "existing",
91 STATUS_STANDBY: "ready for use",
92 STATUS_ONLINE: "online",
96 def __init__(self, unique_id, children):
97 self._children = children
99 self.unique_id = unique_id
105 """Assemble the device from its components.
107 If this is a plain block device (e.g. LVM) than assemble does
108 nothing, as the LVM has no children and we don't put logical
111 One guarantee is that after the device has been assembled, it
112 knows its major/minor numbers. This allows other devices (usually
113 parents) to probe correctly for their children.
117 for child in self._children:
118 if not isinstance(child, BlockDev):
119 raise TypeError("Invalid child passed of type '%s'" % type(child))
122 status = status and child.Assemble()
125 status = status and child.Open()
128 for child in self._children:
134 """Find a device which matches our config and attach to it.
137 raise NotImplementedError
141 """Notifies that the device will no longer be used for I/O.
144 raise NotImplementedError
148 def Create(cls, unique_id, children, size):
149 """Create the device.
151 If the device cannot be created, it will return None
152 instead. Error messages go to the logging system.
154 Note that for some devices, the unique_id is used, and for other,
155 the children. The idea is that these two, taken together, are
156 enough for both creation and assembly (later).
159 raise NotImplementedError
163 """Remove this device.
165 This makes sense only for some of the device types: LV and to a
166 lesser degree, md devices. Also note that if the device can't
167 attach, the removal can't be completed.
170 raise NotImplementedError
174 """Return the status of the device.
177 raise NotImplementedError
180 def Open(self, force=False):
181 """Make the device ready for use.
183 This makes the device ready for I/O. For now, just the DRBD
186 The force parameter signifies that if the device has any kind of
187 --force thing, it should be used, we know what we are doing.
190 raise NotImplementedError
194 """Shut down the device, freeing its children.
196 This undoes the `Assemble()` work, except for the child
197 assembling; as such, the children on the device are still
198 assembled after this call.
201 raise NotImplementedError
204 def SetSyncSpeed(self, speed):
205 """Adjust the sync speed of the mirror.
207 In case this is not a mirroring device, this is no-op.
212 for child in self._children:
213 result = result and child.SetSyncSpeed(speed)
217 def GetSyncStatus(self):
218 """Returns the sync status of the device.
220 If this device is a mirroring device, this function returns the
221 status of the mirror.
224 (sync_percent, estimated_time, is_degraded)
226 If sync_percent is None, it means all is ok
227 If estimated_time is None, it means we can't estimate
228 the time needed, otherwise it's the time left in seconds
229 If is_degraded is True, it means the device is missing
230 redundancy. This is usually a sign that something went wrong in
231 the device setup, if sync_percent is None.
234 return None, None, False
237 def CombinedSyncStatus(self):
238 """Calculate the mirror status recursively for our children.
240 The return value is the same as for `GetSyncStatus()` except the
241 minimum percent and maximum time are calculated across our
245 min_percent, max_time, is_degraded = self.GetSyncStatus()
247 for child in self._children:
248 c_percent, c_time, c_degraded = child.GetSyncStatus()
249 if min_percent is None:
250 min_percent = c_percent
251 elif c_percent is not None:
252 min_percent = min(min_percent, c_percent)
255 elif c_time is not None:
256 max_time = max(max_time, c_time)
257 is_degraded = is_degraded or c_degraded
258 return min_percent, max_time, is_degraded
261 def SetInfo(self, text):
262 """Update metadata with info text.
264 Only supported for some device types.
267 for child in self._children:
272 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
273 (self.__class__, self.unique_id, self._children,
274 self.major, self.minor, self.dev_path))
277 class LogicalVolume(BlockDev):
278 """Logical Volume block device.
281 def __init__(self, unique_id, children):
282 """Attaches to a LV device.
284 The unique_id is a tuple (vg_name, lv_name)
287 super(LogicalVolume, self).__init__(unique_id, children)
288 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
289 raise ValueError("Invalid configuration data %s" % str(unique_id))
290 self._vg_name, self._lv_name = unique_id
291 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
296 def Create(cls, unique_id, children, size):
297 """Create a new logical volume.
300 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
301 raise ValueError("Invalid configuration data %s" % str(unique_id))
302 vg_name, lv_name = unique_id
303 pvs_info = cls.GetPVInfo(vg_name)
305 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
310 pvlist = [ pv[1] for pv in pvs_info ]
311 free_size = sum([ pv[0] for pv in pvs_info ])
313 # The size constraint should have been checked from the master before
314 # calling the create function.
316 raise errors.BlockDeviceError("Not enough free space: required %s,"
317 " available %s" % (size, free_size))
318 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
321 raise errors.BlockDeviceError(result.fail_reason)
322 return LogicalVolume(unique_id, children)
325 def GetPVInfo(vg_name):
326 """Get the free space info for PVs in a volume group.
329 vg_name: the volume group name
332 list of (free_space, name) with free_space in mebibytes
335 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
336 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
338 result = utils.RunCmd(command)
340 logger.Error("Can't get the PV information: %s" % result.fail_reason)
343 for line in result.stdout.splitlines():
344 fields = line.strip().split(':')
346 logger.Error("Can't parse pvs output: line '%s'" % line)
348 # skip over pvs from another vg or ones which are not allocatable
349 if fields[1] != vg_name or fields[3][0] != 'a':
351 data.append((float(fields[2]), fields[0]))
356 """Remove this logical volume.
359 if not self.minor and not self.Attach():
360 # the LV does not exist
362 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
363 (self._vg_name, self._lv_name)])
365 logger.Error("Can't lvremove: %s" % result.fail_reason)
367 return not result.failed
371 """Attach to an existing LV.
373 This method will try to see if an existing and active LV exists
374 which matches the our name. If so, its major/minor will be
378 result = utils.RunCmd(["lvdisplay", self.dev_path])
380 logger.Error("Can't find LV %s: %s" %
381 (self.dev_path, result.fail_reason))
383 match = re.compile("^ *Block device *([0-9]+):([0-9]+).*$")
384 for line in result.stdout.splitlines():
385 match_result = match.match(line)
387 self.major = int(match_result.group(1))
388 self.minor = int(match_result.group(2))
394 """Assemble the device.
396 This is a no-op for the LV device type. Eventually, we could
397 lvchange -ay here if we see that the LV is not active.
404 """Shutdown the device.
406 This is a no-op for the LV device type, as we don't deactivate the
414 """Return the status of the device.
416 Logical volumes will can be in all four states, although we don't
417 deactivate (lvchange -an) them when shutdown, so STATUS_EXISTING
418 should not be seen for our devices.
421 result = utils.RunCmd(["lvs", "--noheadings", "-olv_attr", self.dev_path])
423 logger.Error("Can't display lv: %s" % result.fail_reason)
424 return self.STATUS_UNKNOWN
425 out = result.stdout.strip()
426 # format: type/permissions/alloc/fixed_minor/state/open
428 return self.STATUS_UNKNOWN
429 #writable = (out[1] == "w")
430 active = (out[4] == "a")
431 online = (out[5] == "o")
433 retval = self.STATUS_ONLINE
435 retval = self.STATUS_STANDBY
437 retval = self.STATUS_EXISTING
442 def Open(self, force=False):
443 """Make the device ready for I/O.
445 This is a no-op for the LV device type.
452 """Notifies that the device will no longer be used for I/O.
454 This is a no-op for the LV device type.
460 def Snapshot(self, size):
461 """Create a snapshot copy of an lvm block device.
464 snap_name = self._lv_name + ".snap"
466 # remove existing snapshot if found
467 snap = LogicalVolume((self._vg_name, snap_name), None)
470 pvs_info = self.GetPVInfo(self._vg_name)
472 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
476 free_size, pv_name = pvs_info[0]
478 raise errors.BlockDeviceError("Not enough free space: required %s,"
479 " available %s" % (size, free_size))
481 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
482 "-n%s" % snap_name, self.dev_path])
484 raise errors.BlockDeviceError("command: %s error: %s" %
485 (result.cmd, result.fail_reason))
490 def SetInfo(self, text):
491 """Update metadata with info text.
494 BlockDev.SetInfo(self, text)
496 # Replace invalid characters
497 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
498 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
500 # Only up to 128 characters are allowed
503 result = utils.RunCmd(["lvchange", "--addtag", text,
506 raise errors.BlockDeviceError("Command: %s error: %s" %
507 (result.cmd, result.fail_reason))
510 class MDRaid1(BlockDev):
511 """raid1 device implemented via md.
514 def __init__(self, unique_id, children):
515 super(MDRaid1, self).__init__(unique_id, children)
521 """Find an array which matches our config and attach to it.
523 This tries to find a MD array which has the same UUID as our own.
526 minor = self._FindMDByUUID(self.unique_id)
527 if minor is not None:
528 self._SetFromMinor(minor)
533 return (minor is not None)
538 """Compute the list of in-use MD devices.
540 It doesn't matter if the used device have other raid level, just
541 that they are in use.
544 mdstat = open("/proc/mdstat", "r")
545 data = mdstat.readlines()
549 valid_line = re.compile("^md([0-9]+) : .*$")
551 match = valid_line.match(line)
553 md_no = int(match.group(1))
554 used_md[md_no] = line
560 def _GetDevInfo(minor):
561 """Get info about a MD device.
563 Currently only uuid is returned.
566 result = utils.RunCmd(["mdadm", "-D", "/dev/md%d" % minor])
568 logger.Error("Can't display md: %s" % result.fail_reason)
571 for line in result.stdout.splitlines():
573 kv = line.split(" : ", 1)
576 retval["uuid"] = kv[1]
577 elif kv[0] == "State":
578 retval["state"] = kv[1].split(", ")
583 def _FindUnusedMinor():
584 """Compute an unused MD minor.
586 This code assumes that there are 256 minors only.
589 used_md = MDRaid1._GetUsedDevs()
596 logger.Error("Critical: Out of md minor numbers.")
602 def _FindMDByUUID(cls, uuid):
603 """Find the minor of an MD array with a given UUID.
606 md_list = cls._GetUsedDevs()
607 for minor in md_list:
608 info = cls._GetDevInfo(minor)
609 if info and info["uuid"] == uuid:
615 def Create(cls, unique_id, children, size):
616 """Create a new MD raid1 array.
619 if not isinstance(children, (tuple, list)):
620 raise ValueError("Invalid setup data for MDRaid1 dev: %s" %
623 if not isinstance(i, BlockDev):
624 raise ValueError("Invalid member in MDRaid1 dev: %s" % type(i))
626 result = utils.RunCmd(["mdadm", "--zero-superblock", "--force",
629 logger.Error("Can't zero superblock: %s" % result.fail_reason)
631 minor = cls._FindUnusedMinor()
632 result = utils.RunCmd(["mdadm", "--create", "/dev/md%d" % minor,
633 "--auto=yes", "--force", "-l1",
634 "-n%d" % len(children)] +
635 [dev.dev_path for dev in children])
638 logger.Error("Can't create md: %s" % result.fail_reason)
640 info = cls._GetDevInfo(minor)
641 if not info or not "uuid" in info:
642 logger.Error("Wrong information returned from mdadm -D: %s" % str(info))
644 return MDRaid1(info["uuid"], children)
648 """Stub remove function for MD RAID 1 arrays.
650 We don't remove the superblock right now. Mark a to do.
653 #TODO: maybe zero superblock on child devices?
654 return self.Shutdown()
657 def AddChild(self, device):
658 """Add a new member to the md raid1.
661 if self.minor is None and not self.Attach():
662 raise errors.BlockDeviceError("Can't attach to device")
663 if device.dev_path is None:
664 raise errors.BlockDeviceError("New child is not initialised")
665 result = utils.RunCmd(["mdadm", "-a", self.dev_path, device.dev_path])
667 raise errors.BlockDeviceError("Failed to add new device to array: %s" %
669 new_len = len(self._children) + 1
670 result = utils.RunCmd(["mdadm", "--grow", self.dev_path, "-n", new_len])
672 raise errors.BlockDeviceError("Can't grow md array: %s" %
674 self._children.append(device)
677 def RemoveChild(self, dev_path):
678 """Remove member from the md raid1.
681 if self.minor is None and not self.Attach():
682 raise errors.BlockDeviceError("Can't attach to device")
683 if len(self._children) == 1:
684 raise errors.BlockDeviceError("Can't reduce member when only one"
686 for device in self._children:
687 if device.dev_path == dev_path:
690 raise errors.BlockDeviceError("Can't find child with this path")
691 new_len = len(self._children) - 1
692 result = utils.RunCmd(["mdadm", "-f", self.dev_path, dev_path])
694 raise errors.BlockDeviceError("Failed to mark device as failed: %s" %
697 # it seems here we need a short delay for MD to update its
700 result = utils.RunCmd(["mdadm", "-r", self.dev_path, dev_path])
702 raise errors.BlockDeviceError("Failed to remove device from array:"
703 " %s" % result.output)
704 result = utils.RunCmd(["mdadm", "--grow", "--force", self.dev_path,
707 raise errors.BlockDeviceError("Can't shrink md array: %s" %
709 self._children.remove(device)
713 """Return the status of the device.
717 if self.minor is None:
718 retval = self.STATUS_UNKNOWN
720 retval = self.STATUS_ONLINE
724 def _SetFromMinor(self, minor):
725 """Set our parameters based on the given minor.
727 This sets our minor variable and our dev_path.
731 self.dev_path = "/dev/md%d" % minor
735 """Assemble the MD device.
737 At this point we should have:
738 - list of children devices
742 result = super(MDRaid1, self).Assemble()
745 md_list = self._GetUsedDevs()
746 for minor in md_list:
747 info = self._GetDevInfo(minor)
748 if info and info["uuid"] == self.unique_id:
749 self._SetFromMinor(minor)
750 logger.Info("MD array %s already started" % str(self))
752 free_minor = self._FindUnusedMinor()
753 result = utils.RunCmd(["mdadm", "-A", "--auto=yes", "--uuid",
754 self.unique_id, "/dev/md%d" % free_minor] +
755 [bdev.dev_path for bdev in self._children])
757 logger.Error("Can't assemble MD array: %s" % result.fail_reason)
760 self.minor = free_minor
761 return not result.failed
765 """Tear down the MD array.
767 This does a 'mdadm --stop' so after this command, the array is no
771 if self.minor is None and not self.Attach():
772 logger.Info("MD object not attached to a device")
775 result = utils.RunCmd(["mdadm", "--stop", "/dev/md%d" % self.minor])
777 logger.Error("Can't stop MD array: %s" % result.fail_reason)
784 def SetSyncSpeed(self, kbytes):
785 """Set the maximum sync speed for the MD array.
788 result = super(MDRaid1, self).SetSyncSpeed(kbytes)
789 if self.minor is None:
790 logger.Error("MD array not attached to a device")
792 f = open("/sys/block/md%d/md/sync_speed_max" % self.minor, "w")
794 f.write("%d" % kbytes)
797 f = open("/sys/block/md%d/md/sync_speed_min" % self.minor, "w")
799 f.write("%d" % (kbytes/2))
805 def GetSyncStatus(self):
806 """Returns the sync status of the device.
809 (sync_percent, estimated_time)
811 If sync_percent is None, it means all is ok
812 If estimated_time is None, it means we can't esimate
813 the time needed, otherwise it's the time left in seconds
816 if self.minor is None and not self.Attach():
817 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
818 dev_info = self._GetDevInfo(self.minor)
819 is_clean = ("state" in dev_info and
820 len(dev_info["state"]) == 1 and
821 dev_info["state"][0] in ("clean", "active"))
822 sys_path = "/sys/block/md%s/md/" % self.minor
823 f = file(sys_path + "sync_action")
824 sync_status = f.readline().strip()
826 if sync_status == "idle":
827 return None, None, not is_clean
828 f = file(sys_path + "sync_completed")
829 sync_completed = f.readline().strip().split(" / ")
831 if len(sync_completed) != 2:
832 return 0, None, not is_clean
833 sync_done, sync_total = [float(i) for i in sync_completed]
834 sync_percent = 100.0*sync_done/sync_total
835 f = file(sys_path + "sync_speed")
836 sync_speed_k = int(f.readline().strip())
837 if sync_speed_k == 0:
840 time_est = (sync_total - sync_done) / 2 / sync_speed_k
841 return sync_percent, time_est, not is_clean
844 def Open(self, force=False):
845 """Make the device ready for I/O.
847 This is a no-op for the MDRaid1 device type, although we could use
848 the 2.6.18's new array_state thing.
855 """Notifies that the device will no longer be used for I/O.
857 This is a no-op for the MDRaid1 device type, but see comment for
865 class DRBDev(BlockDev):
866 """DRBD block device.
868 This implements the local host part of the DRBD device, i.e. it
869 doesn't do anything to the supposed peer. If you need a fully
870 connected DRBD pair, you need to use this class on both hosts.
872 The unique_id for the drbd device is the (local_ip, local_port,
873 remote_ip, remote_port) tuple, and it must have two children: the
874 data device and the meta_device. The meta device is checked for
875 valid size and is zeroed on create.
879 _ST_UNCONFIGURED = "Unconfigured"
880 _ST_WFCONNECTION = "WFConnection"
881 _ST_CONNECTED = "Connected"
883 def __init__(self, unique_id, children):
884 super(DRBDev, self).__init__(unique_id, children)
885 self.major = self._DRBD_MAJOR
886 if len(children) != 2:
887 raise ValueError("Invalid configuration data %s" % str(children))
888 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 4:
889 raise ValueError("Invalid configuration data %s" % str(unique_id))
890 self._lhost, self._lport, self._rhost, self._rport = unique_id
895 """Return the path to a drbd device for a given minor.
898 return "/dev/drbd%d" % minor
902 """Return data from /proc/drbd.
905 stat = open("/proc/drbd", "r")
906 data = stat.read().splitlines()
912 def _GetUsedDevs(cls):
913 """Compute the list of used DRBD devices.
916 data = cls._GetProcData()
919 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
921 match = valid_line.match(line)
924 minor = int(match.group(1))
925 state = match.group(2)
926 if state == cls._ST_UNCONFIGURED:
928 used_devs[minor] = state, line
934 def _FindUnusedMinor(cls):
935 """Find an unused DRBD device.
938 data = cls._GetProcData()
940 valid_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
942 match = valid_line.match(line)
944 return int(match.group(1))
945 logger.Error("Error: no free drbd minors!")
950 def _GetDevInfo(cls, minor):
951 """Get details about a given DRBD minor.
953 This return, if available, the local backing device in (major,
954 minor) formant and the local and remote (ip, port) information.
958 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
960 logger.Error("Can't display the drbd config: %s" % result.fail_reason)
963 if out == "Not configured\n":
965 for line in out.splitlines():
966 if "local_dev" not in data:
967 match = re.match("^Lower device: ([0-9]+):([0-9]+) .*$", line)
969 data["local_dev"] = (int(match.group(1)), int(match.group(2)))
971 if "meta_dev" not in data:
972 match = re.match("^Meta device: (([0-9]+):([0-9]+)|internal).*$", line)
974 if match.group(2) is not None and match.group(3) is not None:
975 # matched on the major/minor
976 data["meta_dev"] = (int(match.group(2)), int(match.group(3)))
978 # matched on the "internal" string
979 data["meta_dev"] = match.group(1)
980 # in this case, no meta_index is in the output
981 data["meta_index"] = -1
983 if "meta_index" not in data:
984 match = re.match("^Meta index: ([0-9]+).*$", line)
986 data["meta_index"] = int(match.group(1))
988 if "local_addr" not in data:
989 match = re.match("^Local address: ([0-9.]+):([0-9]+)$", line)
991 data["local_addr"] = (match.group(1), int(match.group(2)))
993 if "remote_addr" not in data:
994 match = re.match("^Remote address: ([0-9.]+):([0-9]+)$", line)
996 data["remote_addr"] = (match.group(1), int(match.group(2)))
1001 def _MatchesLocal(self, info):
1002 """Test if our local config matches with an existing device.
1004 The parameter should be as returned from `_GetDevInfo()`. This
1005 method tests if our local backing device is the same as the one in
1006 the info parameter, in effect testing if we look like the given
1010 if not ("local_dev" in info and "meta_dev" in info and
1011 "meta_index" in info):
1014 backend = self._children[0]
1015 if backend is not None:
1016 retval = (info["local_dev"] == (backend.major, backend.minor))
1018 retval = (info["local_dev"] == (0, 0))
1019 meta = self._children[1]
1020 if meta is not None:
1021 retval = retval and (info["meta_dev"] == (meta.major, meta.minor))
1022 retval = retval and (info["meta_index"] == 0)
1024 retval = retval and (info["meta_dev"] == "internal" and
1025 info["meta_index"] == -1)
1029 def _MatchesNet(self, info):
1030 """Test if our network config matches with an existing device.
1032 The parameter should be as returned from `_GetDevInfo()`. This
1033 method tests if our network configuration is the same as the one
1034 in the info parameter, in effect testing if we look like the given
1038 if (((self._lhost is None and not ("local_addr" in info)) and
1039 (self._rhost is None and not ("remote_addr" in info)))):
1042 if self._lhost is None:
1045 if not ("local_addr" in info and
1046 "remote_addr" in info):
1049 retval = (info["local_addr"] == (self._lhost, self._lport))
1050 retval = (retval and
1051 info["remote_addr"] == (self._rhost, self._rport))
1056 def _IsValidMeta(meta_device):
1057 """Check if the given meta device looks like a valid one.
1059 This currently only check the size, which must be around
1063 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1065 logger.Error("Failed to get device size: %s" % result.fail_reason)
1068 sectors = int(result.stdout)
1070 logger.Error("Invalid output from blockdev: '%s'" % result.stdout)
1072 bytes = sectors * 512
1073 if bytes < 128*1024*1024: # less than 128MiB
1074 logger.Error("Meta device too small (%.2fMib)" % (bytes/1024/1024))
1076 if bytes > (128+32)*1024*1024: # account for an extra (big) PE on LVM
1077 logger.Error("Meta device too big (%.2fMiB)" % (bytes/1024/1024))
1083 def _AssembleLocal(cls, minor, backend, meta):
1084 """Configure the local part of a DRBD device.
1086 This is the first thing that must be done on an unconfigured DRBD
1087 device. And it must be done only once.
1090 if not cls._IsValidMeta(meta):
1092 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disk",
1093 backend, meta, "0", "-e", "detach"])
1095 logger.Error("Can't attach local disk: %s" % result.output)
1096 return not result.failed
1100 def _ShutdownLocal(cls, minor):
1101 """Detach from the local device.
1103 I/Os will continue to be served from the remote device. If we
1104 don't have a remote device, this operation will fail.
1107 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1109 logger.Error("Can't detach local device: %s" % result.output)
1110 return not result.failed
1114 def _ShutdownAll(minor):
1115 """Deactivate the device.
1117 This will, of course, fail if the device is in use.
1120 result = utils.RunCmd(["drbdsetup", DRBDev._DevPath(minor), "down"])
1122 logger.Error("Can't shutdown drbd device: %s" % result.output)
1123 return not result.failed
1127 def _AssembleNet(cls, minor, net_info, protocol):
1128 """Configure the network part of the device.
1130 This operation can be, in theory, done multiple times, but there
1131 have been cases (in lab testing) in which the network part of the
1132 device had become stuck and couldn't be shut down because activity
1133 from the new peer (also stuck) triggered a timer re-init and
1134 needed remote peer interface shutdown in order to clear. So please
1135 don't change online the net config.
1138 lhost, lport, rhost, rport = net_info
1139 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "net",
1140 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport),
1143 logger.Error("Can't setup network for dbrd device: %s" %
1147 timeout = time.time() + 10
1149 while time.time() < timeout:
1150 info = cls._GetDevInfo(minor)
1151 if not "local_addr" in info or not "remote_addr" in info:
1154 if (info["local_addr"] != (lhost, lport) or
1155 info["remote_addr"] != (rhost, rport)):
1161 logger.Error("Timeout while configuring network")
1167 def _ShutdownNet(cls, minor):
1168 """Disconnect from the remote peer.
1170 This fails if we don't have a local device.
1173 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1174 logger.Error("Can't shutdown network: %s" % result.output)
1175 return not result.failed
1178 def _SetFromMinor(self, minor):
1179 """Set our parameters based on the given minor.
1181 This sets our minor variable and our dev_path.
1185 self.minor = self.dev_path = None
1188 self.dev_path = self._DevPath(minor)
1192 """Assemble the drbd.
1195 - if we have a local backing device, we bind to it by:
1196 - checking the list of used drbd devices
1197 - check if the local minor use of any of them is our own device
1200 - if we have a local/remote net info:
1201 - redo the local backing device step for the remote device
1202 - check if any drbd device is using the local port,
1204 - check if any remote drbd device is using the remote
1205 port, if yes abort (for now)
1207 - bind the remote net port
1211 if self.minor is not None:
1212 logger.Info("Already assembled")
1215 result = super(DRBDev, self).Assemble()
1219 minor = self._FindUnusedMinor()
1221 raise errors.BlockDeviceError("Not enough free minors for DRBD!")
1222 need_localdev_teardown = False
1223 if self._children[0]:
1224 result = self._AssembleLocal(minor, self._children[0].dev_path,
1225 self._children[1].dev_path)
1228 need_localdev_teardown = True
1229 if self._lhost and self._lport and self._rhost and self._rport:
1230 result = self._AssembleNet(minor,
1231 (self._lhost, self._lport,
1232 self._rhost, self._rport),
1235 if need_localdev_teardown:
1236 # we will ignore failures from this
1237 logger.Error("net setup failed, tearing down local device")
1238 self._ShutdownAll(minor)
1240 self._SetFromMinor(minor)
1245 """Shutdown the DRBD device.
1248 if self.minor is None and not self.Attach():
1249 logger.Info("DRBD device not attached to a device during Shutdown")
1251 if not self._ShutdownAll(self.minor):
1254 self.dev_path = None
1259 """Find a DRBD device which matches our config and attach to it.
1261 In case of partially attached (local device matches but no network
1262 setup), we perform the network attach. If successful, we re-test
1263 the attach if can return success.
1266 for minor in self._GetUsedDevs():
1267 info = self._GetDevInfo(minor)
1268 match_l = self._MatchesLocal(info)
1269 match_r = self._MatchesNet(info)
1270 if match_l and match_r:
1272 if match_l and not match_r and "local_addr" not in info:
1273 res_r = self._AssembleNet(minor,
1274 (self._lhost, self._lport,
1275 self._rhost, self._rport),
1277 if res_r and self._MatchesNet(self._GetDevInfo(minor)):
1282 self._SetFromMinor(minor)
1283 return minor is not None
1286 def Open(self, force=False):
1287 """Make the local state primary.
1289 If the 'force' parameter is given, the '--do-what-I-say' parameter
1290 is given. Since this is a pottentialy dangerous operation, the
1291 force flag should be only given after creation, when it actually
1295 if self.minor is None and not self.Attach():
1296 logger.Error("DRBD cannot attach to a device during open")
1298 cmd = ["drbdsetup", self.dev_path, "primary"]
1300 cmd.append("--do-what-I-say")
1301 result = utils.RunCmd(cmd)
1303 logger.Error("Can't make drbd device primary: %s" % result.output)
1309 """Make the local state secondary.
1311 This will, of course, fail if the device is in use.
1314 if self.minor is None and not self.Attach():
1315 logger.Info("Instance not attached to a device")
1316 raise errors.BlockDeviceError("Can't find device")
1317 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1319 logger.Error("Can't switch drbd device to secondary: %s" % result.output)
1320 raise errors.BlockDeviceError("Can't switch drbd device to secondary")
1323 def SetSyncSpeed(self, kbytes):
1324 """Set the speed of the DRBD syncer.
1327 children_result = super(DRBDev, self).SetSyncSpeed(kbytes)
1328 if self.minor is None:
1329 logger.Info("Instance not attached to a device")
1331 result = utils.RunCmd(["drbdsetup", self.dev_path, "syncer", "-r", "%d" %
1334 logger.Error("Can't change syncer rate: %s " % result.fail_reason)
1335 return not result.failed and children_result
1338 def GetSyncStatus(self):
1339 """Returns the sync status of the device.
1342 (sync_percent, estimated_time)
1344 If sync_percent is None, it means all is ok
1345 If estimated_time is None, it means we can't esimate
1346 the time needed, otherwise it's the time left in seconds
1349 if self.minor is None and not self.Attach():
1350 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1351 proc_info = self._MassageProcData(self._GetProcData())
1352 if self.minor not in proc_info:
1353 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1355 line = proc_info[self.minor]
1356 match = re.match("^.*sync'ed: *([0-9.]+)%.*"
1357 " finish: ([0-9]+):([0-9]+):([0-9]+) .*$", line)
1359 sync_percent = float(match.group(1))
1360 hours = int(match.group(2))
1361 minutes = int(match.group(3))
1362 seconds = int(match.group(4))
1363 est_time = hours * 3600 + minutes * 60 + seconds
1367 match = re.match("^ *[0-9]+: cs:([^ ]+).*$", line)
1369 raise errors.BlockDeviceError("Can't find my data in /proc (minor %d)" %
1371 client_state = match.group(1)
1372 is_degraded = client_state != "Connected"
1373 return sync_percent, est_time, is_degraded
1377 def _MassageProcData(data):
1378 """Transform the output of _GetProdData into a nicer form.
1381 a dictionary of minor: joined lines from /proc/drbd for that minor
1384 lmatch = re.compile("^ *([0-9]+):.*$")
1386 old_minor = old_line = None
1388 lresult = lmatch.match(line)
1389 if lresult is not None:
1390 if old_minor is not None:
1391 results[old_minor] = old_line
1392 old_minor = int(lresult.group(1))
1395 if old_minor is not None:
1396 old_line += " " + line.strip()
1398 if old_minor is not None:
1399 results[old_minor] = old_line
1403 def GetStatus(self):
1404 """Compute the status of the DRBD device
1406 Note that DRBD devices don't have the STATUS_EXISTING state.
1409 if self.minor is None and not self.Attach():
1410 return self.STATUS_UNKNOWN
1412 data = self._GetProcData()
1413 match = re.compile("^ *%d: cs:[^ ]+ st:(Primary|Secondary)/.*$" %
1416 mresult = match.match(line)
1420 logger.Error("Can't find myself!")
1421 return self.STATUS_UNKNOWN
1423 state = mresult.group(2)
1424 if state == "Primary":
1425 result = self.STATUS_ONLINE
1427 result = self.STATUS_STANDBY
1433 def _ZeroDevice(device):
1436 This writes until we get ENOSPC.
1439 f = open(device, "w")
1440 buf = "\0" * 1048576
1444 except IOError, err:
1445 if err.errno != errno.ENOSPC:
1450 def Create(cls, unique_id, children, size):
1451 """Create a new DRBD device.
1453 Since DRBD devices are not created per se, just assembled, this
1454 function just zeroes the meta device.
1457 if len(children) != 2:
1458 raise errors.ProgrammerError("Invalid setup for the drbd device")
1461 if not meta.Attach():
1462 raise errors.BlockDeviceError("Can't attach to meta device")
1463 if not cls._IsValidMeta(meta.dev_path):
1464 raise errors.BlockDeviceError("Invalid meta device")
1465 logger.Info("Started zeroing device %s" % meta.dev_path)
1466 cls._ZeroDevice(meta.dev_path)
1467 logger.Info("Done zeroing device %s" % meta.dev_path)
1468 return cls(unique_id, children)
1472 """Stub remove for DRBD devices.
1475 return self.Shutdown()
1479 "lvm": LogicalVolume,
1480 "md_raid1": MDRaid1,
1485 def FindDevice(dev_type, unique_id, children):
1486 """Search for an existing, assembled device.
1488 This will succeed only if the device exists and is assembled, but it
1489 does not do any actions in order to activate the device.
1492 if dev_type not in DEV_MAP:
1493 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1494 device = DEV_MAP[dev_type](unique_id, children)
1495 if not device.Attach():
1500 def AttachOrAssemble(dev_type, unique_id, children):
1501 """Try to attach or assemble an existing device.
1503 This will attach to an existing assembled device or will assemble
1504 the device, as needed, to bring it fully up.
1507 if dev_type not in DEV_MAP:
1508 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1509 device = DEV_MAP[dev_type](unique_id, children)
1510 if not device.Attach():
1512 if not device.Attach():
1513 raise errors.BlockDeviceError("Can't find a valid block device for"
1515 (dev_type, unique_id, children))
1519 def Create(dev_type, unique_id, children, size):
1523 if dev_type not in DEV_MAP:
1524 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1525 device = DEV_MAP[dev_type].Create(unique_id, children, size)