4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
31 from ganeti import utils
32 from ganeti import errors
33 from ganeti import constants
36 class BlockDev(object):
37 """Block device abstract class.
39 A block device can be in the following states:
40 - not existing on the system, and by `Create()` it goes into:
41 - existing but not setup/not active, and by `Assemble()` goes into:
42 - active read-write and by `Open()` it goes into
43 - online (=used, or ready for use)
45 A device can also be online but read-only, however we are not using
46 the readonly state (LV has it, if needed in the future) and we are
47 usually looking at this like at a stack, so it's easier to
48 conceptualise the transition from not-existing to online and back
51 The many different states of the device are due to the fact that we
52 need to cover many device types:
53 - logical volumes are created, lvchange -a y $lv, and used
54 - drbd devices are attached to a local disk/remote peer and made primary
56 A block device is identified by three items:
57 - the /dev path of the device (dynamic)
58 - a unique ID of the device (static)
59 - it's major/minor pair (dynamic)
61 Not all devices implement both the first two as distinct items. LVM
62 logical volumes have their unique ID (the pair volume group, logical
63 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
64 the /dev path is again dynamic and the unique id is the pair (host1,
67 You can get to a device in two ways:
68 - creating the (real) device, which returns you
69 an attached instance (lvcreate)
70 - attaching of a python instance to an existing (real) device
72 The second point, the attachement to a device, is different
73 depending on whether the device is assembled or not. At init() time,
74 we search for a device with the same unique_id as us. If found,
75 good. It also means that the device is already assembled. If not,
76 after assembly we'll have our correct major/minor.
79 def __init__(self, unique_id, children):
80 self._children = children
82 self.unique_id = unique_id
88 """Assemble the device from its components.
90 Implementations of this method by child classes must ensure that:
91 - after the device has been assembled, it knows its major/minor
92 numbers; this allows other devices (usually parents) to probe
93 correctly for their children
94 - calling this method on an existing, in-use device is safe
95 - if the device is already configured (and in an OK state),
96 this method is idempotent
102 """Find a device which matches our config and attach to it.
105 raise NotImplementedError
108 """Notifies that the device will no longer be used for I/O.
111 raise NotImplementedError
114 def Create(cls, unique_id, children, size):
115 """Create the device.
117 If the device cannot be created, it will return None
118 instead. Error messages go to the logging system.
120 Note that for some devices, the unique_id is used, and for other,
121 the children. The idea is that these two, taken together, are
122 enough for both creation and assembly (later).
125 raise NotImplementedError
128 """Remove this device.
130 This makes sense only for some of the device types: LV and file
131 storeage. Also note that if the device can't attach, the removal
135 raise NotImplementedError
137 def Rename(self, new_id):
138 """Rename this device.
140 This may or may not make sense for a given device type.
143 raise NotImplementedError
145 def Open(self, force=False):
146 """Make the device ready for use.
148 This makes the device ready for I/O. For now, just the DRBD
151 The force parameter signifies that if the device has any kind of
152 --force thing, it should be used, we know what we are doing.
155 raise NotImplementedError
158 """Shut down the device, freeing its children.
160 This undoes the `Assemble()` work, except for the child
161 assembling; as such, the children on the device are still
162 assembled after this call.
165 raise NotImplementedError
167 def SetSyncSpeed(self, speed):
168 """Adjust the sync speed of the mirror.
170 In case this is not a mirroring device, this is no-op.
175 for child in self._children:
176 result = result and child.SetSyncSpeed(speed)
179 def GetSyncStatus(self):
180 """Returns the sync status of the device.
182 If this device is a mirroring device, this function returns the
183 status of the mirror.
185 If sync_percent is None, it means the device is not syncing.
187 If estimated_time is None, it means we can't estimate
188 the time needed, otherwise it's the time left in seconds.
190 If is_degraded is True, it means the device is missing
191 redundancy. This is usually a sign that something went wrong in
192 the device setup, if sync_percent is None.
194 The ldisk parameter represents the degradation of the local
195 data. This is only valid for some devices, the rest will always
196 return False (not degraded).
199 @return: (sync_percent, estimated_time, is_degraded, ldisk)
202 return None, None, False, False
205 def CombinedSyncStatus(self):
206 """Calculate the mirror status recursively for our children.
208 The return value is the same as for `GetSyncStatus()` except the
209 minimum percent and maximum time are calculated across our
213 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
215 for child in self._children:
216 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
217 if min_percent is None:
218 min_percent = c_percent
219 elif c_percent is not None:
220 min_percent = min(min_percent, c_percent)
223 elif c_time is not None:
224 max_time = max(max_time, c_time)
225 is_degraded = is_degraded or c_degraded
226 ldisk = ldisk or c_ldisk
227 return min_percent, max_time, is_degraded, ldisk
230 def SetInfo(self, text):
231 """Update metadata with info text.
233 Only supported for some device types.
236 for child in self._children:
239 def Grow(self, amount):
240 """Grow the block device.
242 @param amount: the amount (in mebibytes) to grow with
245 raise NotImplementedError
248 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
249 (self.__class__, self.unique_id, self._children,
250 self.major, self.minor, self.dev_path))
253 class LogicalVolume(BlockDev):
254 """Logical Volume block device.
257 def __init__(self, unique_id, children):
258 """Attaches to a LV device.
260 The unique_id is a tuple (vg_name, lv_name)
263 super(LogicalVolume, self).__init__(unique_id, children)
264 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
265 raise ValueError("Invalid configuration data %s" % str(unique_id))
266 self._vg_name, self._lv_name = unique_id
267 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
268 self._degraded = True
269 self.major = self.minor = None
273 def Create(cls, unique_id, children, size):
274 """Create a new logical volume.
277 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
278 raise errors.ProgrammerError("Invalid configuration data %s" %
280 vg_name, lv_name = unique_id
281 pvs_info = cls.GetPVInfo(vg_name)
283 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
288 pvlist = [ pv[1] for pv in pvs_info ]
289 free_size = sum([ pv[0] for pv in pvs_info ])
291 # The size constraint should have been checked from the master before
292 # calling the create function.
294 raise errors.BlockDeviceError("Not enough free space: required %s,"
295 " available %s" % (size, free_size))
296 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
299 raise errors.BlockDeviceError("LV create failed (%s): %s" %
300 (result.fail_reason, result.output))
301 return LogicalVolume(unique_id, children)
304 def GetPVInfo(vg_name):
305 """Get the free space info for PVs in a volume group.
307 @param vg_name: the volume group name
310 @return: list of tuples (free_space, name) with free_space in mebibytes
313 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
314 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
316 result = utils.RunCmd(command)
318 logging.error("Can't get the PV information: %s - %s",
319 result.fail_reason, result.output)
322 for line in result.stdout.splitlines():
323 fields = line.strip().split(':')
325 logging.error("Can't parse pvs output: line '%s'", line)
327 # skip over pvs from another vg or ones which are not allocatable
328 if fields[1] != vg_name or fields[3][0] != 'a':
330 data.append((float(fields[2]), fields[0]))
335 """Remove this logical volume.
338 if not self.minor and not self.Attach():
339 # the LV does not exist
341 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
342 (self._vg_name, self._lv_name)])
344 logging.error("Can't lvremove: %s - %s",
345 result.fail_reason, result.output)
347 return not result.failed
349 def Rename(self, new_id):
350 """Rename this logical volume.
353 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
354 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
355 new_vg, new_name = new_id
356 if new_vg != self._vg_name:
357 raise errors.ProgrammerError("Can't move a logical volume across"
358 " volume groups (from %s to to %s)" %
359 (self._vg_name, new_vg))
360 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
362 raise errors.BlockDeviceError("Failed to rename the logical volume: %s" %
364 self._lv_name = new_name
365 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
368 """Attach to an existing LV.
370 This method will try to see if an existing and active LV exists
371 which matches our name. If so, its major/minor will be
375 self.attached = False
376 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
377 "-olv_attr,lv_kernel_major,lv_kernel_minor",
380 logging.error("Can't find LV %s: %s, %s",
381 self.dev_path, result.fail_reason, result.output)
383 out = result.stdout.strip().rstrip(',')
386 logging.error("Can't parse LVS output, len(%s) != 3", str(out))
389 status, major, minor = out[:3]
391 logging.error("lvs lv_attr is not 6 characters (%s)", status)
397 except ValueError, err:
398 logging.error("lvs major/minor cannot be parsed: %s", str(err))
402 self._degraded = status[0] == 'v' # virtual volume, i.e. doesn't backing
408 """Assemble the device.
410 We alway run `lvchange -ay` on the LV to ensure it's active before
411 use, as there were cases when xenvg was not active after boot
412 (also possibly after disk issues).
415 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
417 logging.error("Can't activate lv %s: %s", self.dev_path, result.output)
422 """Shutdown the device.
424 This is a no-op for the LV device type, as we don't deactivate the
430 def GetSyncStatus(self):
431 """Returns the sync status of the device.
433 If this device is a mirroring device, this function returns the
434 status of the mirror.
436 For logical volumes, sync_percent and estimated_time are always
437 None (no recovery in progress, as we don't handle the mirrored LV
438 case). The is_degraded parameter is the inverse of the ldisk
441 For the ldisk parameter, we check if the logical volume has the
442 'virtual' type, which means it's not backed by existing storage
443 anymore (read from it return I/O error). This happens after a
444 physical disk failure and subsequent 'vgreduce --removemissing' on
447 The status was already read in Attach, so we just return it.
450 @return: (sync_percent, estimated_time, is_degraded, ldisk)
453 return None, None, self._degraded, self._degraded
455 def Open(self, force=False):
456 """Make the device ready for I/O.
458 This is a no-op for the LV device type.
464 """Notifies that the device will no longer be used for I/O.
466 This is a no-op for the LV device type.
471 def Snapshot(self, size):
472 """Create a snapshot copy of an lvm block device.
475 snap_name = self._lv_name + ".snap"
477 # remove existing snapshot if found
478 snap = LogicalVolume((self._vg_name, snap_name), None)
481 pvs_info = self.GetPVInfo(self._vg_name)
483 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
487 free_size, pv_name = pvs_info[0]
489 raise errors.BlockDeviceError("Not enough free space: required %s,"
490 " available %s" % (size, free_size))
492 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
493 "-n%s" % snap_name, self.dev_path])
495 raise errors.BlockDeviceError("command: %s error: %s - %s" %
496 (result.cmd, result.fail_reason,
501 def SetInfo(self, text):
502 """Update metadata with info text.
505 BlockDev.SetInfo(self, text)
507 # Replace invalid characters
508 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
509 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
511 # Only up to 128 characters are allowed
514 result = utils.RunCmd(["lvchange", "--addtag", text,
517 raise errors.BlockDeviceError("Command: %s error: %s - %s" %
518 (result.cmd, result.fail_reason,
520 def Grow(self, amount):
521 """Grow the logical volume.
524 # we try multiple algorithms since the 'best' ones might not have
525 # space available in the right place, but later ones might (since
526 # they have less constraints); also note that only recent LVM
528 for alloc_policy in "contiguous", "cling", "normal":
529 result = utils.RunCmd(["lvextend", "--alloc", alloc_policy,
530 "-L", "+%dm" % amount, self.dev_path])
531 if not result.failed:
533 raise errors.BlockDeviceError("Can't grow LV %s: %s" %
534 (self.dev_path, result.output))
537 class DRBD8Status(object):
538 """A DRBD status representation class.
540 Note that this doesn't support unconfigured devices (cs:Unconfigured).
543 UNCONF_RE = re.compile(r"\s*[0-9]+:\s*cs:Unconfigured$")
544 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+st:([^/]+)/(\S+)"
545 "\s+ds:([^/]+)/(\S+)\s+.*$")
546 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
547 "\sfinish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
549 def __init__(self, procline):
550 u = self.UNCONF_RE.match(procline)
552 self.cstatus = "Unconfigured"
553 self.lrole = self.rrole = self.ldisk = self.rdisk = None
555 m = self.LINE_RE.match(procline)
557 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
558 self.cstatus = m.group(1)
559 self.lrole = m.group(2)
560 self.rrole = m.group(3)
561 self.ldisk = m.group(4)
562 self.rdisk = m.group(5)
564 # end reading of data from the LINE_RE or UNCONF_RE
566 self.is_standalone = self.cstatus == "StandAlone"
567 self.is_wfconn = self.cstatus == "WFConnection"
568 self.is_connected = self.cstatus == "Connected"
569 self.is_primary = self.lrole == "Primary"
570 self.is_secondary = self.lrole == "Secondary"
571 self.peer_primary = self.rrole == "Primary"
572 self.peer_secondary = self.rrole == "Secondary"
573 self.both_primary = self.is_primary and self.peer_primary
574 self.both_secondary = self.is_secondary and self.peer_secondary
576 self.is_diskless = self.ldisk == "Diskless"
577 self.is_disk_uptodate = self.ldisk == "UpToDate"
579 self.is_in_resync = self.cstatus in ("SyncSource", "SyncTarget")
580 self.is_in_use = self.cstatus != "Unconfigured"
582 m = self.SYNC_RE.match(procline)
584 self.sync_percent = float(m.group(1))
585 hours = int(m.group(2))
586 minutes = int(m.group(3))
587 seconds = int(m.group(4))
588 self.est_time = hours * 3600 + minutes * 60 + seconds
590 self.sync_percent = None
593 self.is_sync_target = self.peer_sync_source = self.cstatus == "SyncTarget"
594 self.peer_sync_target = self.is_sync_source = self.cstatus == "SyncSource"
595 self.is_resync = self.is_sync_target or self.is_sync_source
598 class BaseDRBD(BlockDev):
601 This class contains a few bits of common functionality between the
602 0.7 and 8.x versions of DRBD.
605 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
606 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
609 _ST_UNCONFIGURED = "Unconfigured"
610 _ST_WFCONNECTION = "WFConnection"
611 _ST_CONNECTED = "Connected"
613 _STATUS_FILE = "/proc/drbd"
616 def _GetProcData(filename=_STATUS_FILE):
617 """Return data from /proc/drbd.
620 stat = open(filename, "r")
622 data = stat.read().splitlines()
626 raise errors.BlockDeviceError("Can't read any data from %s" % filename)
630 def _MassageProcData(data):
631 """Transform the output of _GetProdData into a nicer form.
633 @return: a dictionary of minor: joined lines from /proc/drbd
637 lmatch = re.compile("^ *([0-9]+):.*$")
639 old_minor = old_line = None
641 lresult = lmatch.match(line)
642 if lresult is not None:
643 if old_minor is not None:
644 results[old_minor] = old_line
645 old_minor = int(lresult.group(1))
648 if old_minor is not None:
649 old_line += " " + line.strip()
651 if old_minor is not None:
652 results[old_minor] = old_line
656 def _GetVersion(cls):
657 """Return the DRBD version.
659 This will return a dict with keys:
665 - proto2 (only on drbd > 8.2.X)
668 proc_data = cls._GetProcData()
669 first_line = proc_data[0].strip()
670 version = cls._VERSION_RE.match(first_line)
672 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
675 values = version.groups()
676 retval = {'k_major': int(values[0]),
677 'k_minor': int(values[1]),
678 'k_point': int(values[2]),
679 'api': int(values[3]),
680 'proto': int(values[4]),
682 if values[5] is not None:
683 retval['proto2'] = values[5]
689 """Return the path to a drbd device for a given minor.
692 return "/dev/drbd%d" % minor
695 def GetUsedDevs(cls):
696 """Compute the list of used DRBD devices.
699 data = cls._GetProcData()
702 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
704 match = valid_line.match(line)
707 minor = int(match.group(1))
708 state = match.group(2)
709 if state == cls._ST_UNCONFIGURED:
711 used_devs[minor] = state, line
715 def _SetFromMinor(self, minor):
716 """Set our parameters based on the given minor.
718 This sets our minor variable and our dev_path.
722 self.minor = self.dev_path = None
723 self.attached = False
726 self.dev_path = self._DevPath(minor)
730 def _CheckMetaSize(meta_device):
731 """Check if the given meta device looks like a valid one.
733 This currently only check the size, which must be around
737 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
739 logging.error("Failed to get device size: %s - %s",
740 result.fail_reason, result.output)
743 sectors = int(result.stdout)
745 logging.error("Invalid output from blockdev: '%s'", result.stdout)
747 bytes = sectors * 512
748 if bytes < 128 * 1024 * 1024: # less than 128MiB
749 logging.error("Meta device too small (%.2fMib)", (bytes / 1024 / 1024))
751 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
752 logging.error("Meta device too big (%.2fMiB)", (bytes / 1024 / 1024))
756 def Rename(self, new_id):
759 This is not supported for drbd devices.
762 raise errors.ProgrammerError("Can't rename a drbd device")
765 class DRBD8(BaseDRBD):
766 """DRBD v8.x block device.
768 This implements the local host part of the DRBD device, i.e. it
769 doesn't do anything to the supposed peer. If you need a fully
770 connected DRBD pair, you need to use this class on both hosts.
772 The unique_id for the drbd device is the (local_ip, local_port,
773 remote_ip, remote_port) tuple, and it must have two children: the
774 data device and the meta_device. The meta device is checked for
775 valid size and is zeroed on create.
782 _NET_RECONFIG_TIMEOUT = 60
784 def __init__(self, unique_id, children):
785 if children and children.count(None) > 0:
787 super(DRBD8, self).__init__(unique_id, children)
788 self.major = self._DRBD_MAJOR
789 version = self._GetVersion()
790 if version['k_major'] != 8 :
791 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
792 " requested ganeti usage: kernel is"
793 " %s.%s, ganeti wants 8.x" %
794 (version['k_major'], version['k_minor']))
796 if len(children) not in (0, 2):
797 raise ValueError("Invalid configuration data %s" % str(children))
798 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
799 raise ValueError("Invalid configuration data %s" % str(unique_id))
800 (self._lhost, self._lport,
801 self._rhost, self._rport,
802 self._aminor, self._secret) = unique_id
803 if (self._lhost is not None and self._lhost == self._rhost and
804 self._lport == self._rport):
805 raise ValueError("Invalid configuration data, same local/remote %s" %
810 def _InitMeta(cls, minor, dev_path):
811 """Initialize a meta device.
813 This will not work if the given minor is in use.
816 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
817 "v08", dev_path, "0", "create-md"])
819 raise errors.BlockDeviceError("Can't initialize meta device: %s" %
823 def _FindUnusedMinor(cls):
824 """Find an unused DRBD device.
826 This is specific to 8.x as the minors are allocated dynamically,
827 so non-existing numbers up to a max minor count are actually free.
830 data = cls._GetProcData()
832 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
833 used_line = re.compile("^ *([0-9]+): cs:")
836 match = unused_line.match(line)
838 return int(match.group(1))
839 match = used_line.match(line)
841 minor = int(match.group(1))
842 highest = max(highest, minor)
843 if highest is None: # there are no minors in use at all
845 if highest >= cls._MAX_MINORS:
846 logging.error("Error: no free drbd minors!")
847 raise errors.BlockDeviceError("Can't find a free DRBD minor")
851 def _IsValidMeta(cls, meta_device):
852 """Check if the given meta device looks like a valid one.
855 minor = cls._FindUnusedMinor()
856 minor_path = cls._DevPath(minor)
857 result = utils.RunCmd(["drbdmeta", minor_path,
858 "v08", meta_device, "0",
861 logging.error("Invalid meta device %s: %s", meta_device, result.output)
866 def _GetShowParser(cls):
867 """Return a parser for `drbd show` output.
869 This will either create or return an already-create parser for the
870 output of the command `drbd show`.
873 if cls._PARSE_SHOW is not None:
874 return cls._PARSE_SHOW
877 lbrace = pyp.Literal("{").suppress()
878 rbrace = pyp.Literal("}").suppress()
879 semi = pyp.Literal(";").suppress()
880 # this also converts the value to an int
881 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
883 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
884 defa = pyp.Literal("_is_default").suppress()
885 dbl_quote = pyp.Literal('"').suppress()
887 keyword = pyp.Word(pyp.alphanums + '-')
890 value = pyp.Word(pyp.alphanums + '_-/.:')
891 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
892 addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() +
894 # meta device, extended syntax
895 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
896 number + pyp.Word(']').suppress())
899 stmt = (~rbrace + keyword + ~lbrace +
900 pyp.Optional(addr_port ^ value ^ quoted ^ meta_value) +
901 pyp.Optional(defa) + semi +
902 pyp.Optional(pyp.restOfLine).suppress())
905 section_name = pyp.Word(pyp.alphas + '_')
906 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
908 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
911 cls._PARSE_SHOW = bnf
916 def _GetShowData(cls, minor):
917 """Return the `drbdsetup show` data for a minor.
920 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
922 logging.error("Can't display the drbd config: %s - %s",
923 result.fail_reason, result.output)
928 def _GetDevInfo(cls, out):
929 """Parse details about a given DRBD minor.
931 This return, if available, the local backing device (as a path)
932 and the local and remote (ip, port) information from a string
933 containing the output of the `drbdsetup show` command as returned
941 bnf = cls._GetShowParser()
945 results = bnf.parseString(out)
946 except pyp.ParseException, err:
947 raise errors.BlockDeviceError("Can't parse drbdsetup show output: %s" %
950 # and massage the results into our desired format
951 for section in results:
953 if sname == "_this_host":
954 for lst in section[1:]:
956 data["local_dev"] = lst[1]
957 elif lst[0] == "meta-disk":
958 data["meta_dev"] = lst[1]
959 data["meta_index"] = lst[2]
960 elif lst[0] == "address":
961 data["local_addr"] = tuple(lst[1:])
962 elif sname == "_remote_host":
963 for lst in section[1:]:
964 if lst[0] == "address":
965 data["remote_addr"] = tuple(lst[1:])
968 def _MatchesLocal(self, info):
969 """Test if our local config matches with an existing device.
971 The parameter should be as returned from `_GetDevInfo()`. This
972 method tests if our local backing device is the same as the one in
973 the info parameter, in effect testing if we look like the given
978 backend, meta = self._children
980 backend = meta = None
982 if backend is not None:
983 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
985 retval = ("local_dev" not in info)
988 retval = retval and ("meta_dev" in info and
989 info["meta_dev"] == meta.dev_path)
990 retval = retval and ("meta_index" in info and
991 info["meta_index"] == 0)
993 retval = retval and ("meta_dev" not in info and
994 "meta_index" not in info)
997 def _MatchesNet(self, info):
998 """Test if our network config matches with an existing device.
1000 The parameter should be as returned from `_GetDevInfo()`. This
1001 method tests if our network configuration is the same as the one
1002 in the info parameter, in effect testing if we look like the given
1006 if (((self._lhost is None and not ("local_addr" in info)) and
1007 (self._rhost is None and not ("remote_addr" in info)))):
1010 if self._lhost is None:
1013 if not ("local_addr" in info and
1014 "remote_addr" in info):
1017 retval = (info["local_addr"] == (self._lhost, self._lport))
1018 retval = (retval and
1019 info["remote_addr"] == (self._rhost, self._rport))
1023 def _AssembleLocal(cls, minor, backend, meta):
1024 """Configure the local part of a DRBD device.
1026 This is the first thing that must be done on an unconfigured DRBD
1027 device. And it must be done only once.
1030 if not cls._IsValidMeta(meta):
1032 args = ["drbdsetup", cls._DevPath(minor), "disk",
1033 backend, meta, "0", "-e", "detach", "--create-device"]
1034 result = utils.RunCmd(args)
1036 logging.error("Can't attach local disk: %s", result.output)
1037 return not result.failed
1040 def _AssembleNet(cls, minor, net_info, protocol,
1041 dual_pri=False, hmac=None, secret=None):
1042 """Configure the network part of the device.
1045 lhost, lport, rhost, rport = net_info
1046 if None in net_info:
1047 # we don't want network connection and actually want to make
1049 return cls._ShutdownNet(minor)
1051 # Workaround for a race condition. When DRBD is doing its dance to
1052 # establish a connection with its peer, it also sends the
1053 # synchronization speed over the wire. In some cases setting the
1054 # sync speed only after setting up both sides can race with DRBD
1055 # connecting, hence we set it here before telling DRBD anything
1057 cls._SetMinorSyncSpeed(minor, constants.SYNC_SPEED)
1059 args = ["drbdsetup", cls._DevPath(minor), "net",
1060 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
1061 "-A", "discard-zero-changes",
1068 args.extend(["-a", hmac, "-x", secret])
1069 result = utils.RunCmd(args)
1071 logging.error("Can't setup network for dbrd device: %s - %s",
1072 result.fail_reason, result.output)
1075 timeout = time.time() + 10
1077 while time.time() < timeout:
1078 info = cls._GetDevInfo(cls._GetShowData(minor))
1079 if not "local_addr" in info or not "remote_addr" in info:
1082 if (info["local_addr"] != (lhost, lport) or
1083 info["remote_addr"] != (rhost, rport)):
1089 logging.error("Timeout while configuring network")
1093 def AddChildren(self, devices):
1094 """Add a disk to the DRBD device.
1097 if self.minor is None:
1098 raise errors.BlockDeviceError("Can't attach to dbrd8 during AddChildren")
1099 if len(devices) != 2:
1100 raise errors.BlockDeviceError("Need two devices for AddChildren")
1101 info = self._GetDevInfo(self._GetShowData(self.minor))
1102 if "local_dev" in info:
1103 raise errors.BlockDeviceError("DRBD8 already attached to a local disk")
1104 backend, meta = devices
1105 if backend.dev_path is None or meta.dev_path is None:
1106 raise errors.BlockDeviceError("Children not ready during AddChildren")
1109 if not self._CheckMetaSize(meta.dev_path):
1110 raise errors.BlockDeviceError("Invalid meta device size")
1111 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1112 if not self._IsValidMeta(meta.dev_path):
1113 raise errors.BlockDeviceError("Cannot initalize meta device")
1115 if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path):
1116 raise errors.BlockDeviceError("Can't attach to local storage")
1117 self._children = devices
1119 def RemoveChildren(self, devices):
1120 """Detach the drbd device from local storage.
1123 if self.minor is None:
1124 raise errors.BlockDeviceError("Can't attach to drbd8 during"
1126 # early return if we don't actually have backing storage
1127 info = self._GetDevInfo(self._GetShowData(self.minor))
1128 if "local_dev" not in info:
1130 if len(self._children) != 2:
1131 raise errors.BlockDeviceError("We don't have two children: %s" %
1133 if self._children.count(None) == 2: # we don't actually have children :)
1134 logging.error("Requested detach while detached")
1136 if len(devices) != 2:
1137 raise errors.BlockDeviceError("We need two children in RemoveChildren")
1138 for child, dev in zip(self._children, devices):
1139 if dev != child.dev_path:
1140 raise errors.BlockDeviceError("Mismatch in local storage"
1141 " (%s != %s) in RemoveChildren" %
1142 (dev, child.dev_path))
1144 if not self._ShutdownLocal(self.minor):
1145 raise errors.BlockDeviceError("Can't detach from local storage")
1149 def _SetMinorSyncSpeed(cls, minor, kbytes):
1150 """Set the speed of the DRBD syncer.
1152 This is the low-level implementation.
1155 @param minor: the drbd minor whose settings we change
1157 @param kbytes: the speed in kbytes/second
1159 @return: the success of the operation
1162 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "syncer",
1163 "-r", "%d" % kbytes, "--create-device"])
1165 logging.error("Can't change syncer rate: %s - %s",
1166 result.fail_reason, result.output)
1167 return not result.failed
1169 def SetSyncSpeed(self, kbytes):
1170 """Set the speed of the DRBD syncer.
1173 @param kbytes: the speed in kbytes/second
1175 @return: the success of the operation
1178 if self.minor is None:
1179 logging.info("Not attached during SetSyncSpeed")
1181 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1182 return self._SetMinorSyncSpeed(self.minor, kbytes) and children_result
1184 def GetProcStatus(self):
1185 """Return device data from /proc.
1188 if self.minor is None:
1189 raise errors.BlockDeviceError("GetStats() called while not attached")
1190 proc_info = self._MassageProcData(self._GetProcData())
1191 if self.minor not in proc_info:
1192 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1194 return DRBD8Status(proc_info[self.minor])
1196 def GetSyncStatus(self):
1197 """Returns the sync status of the device.
1200 If sync_percent is None, it means all is ok
1201 If estimated_time is None, it means we can't esimate
1202 the time needed, otherwise it's the time left in seconds.
1205 We set the is_degraded parameter to True on two conditions:
1206 network not connected or local disk missing.
1208 We compute the ldisk parameter based on wheter we have a local
1212 @return: (sync_percent, estimated_time, is_degraded, ldisk)
1215 if self.minor is None and not self.Attach():
1216 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1217 stats = self.GetProcStatus()
1218 ldisk = not stats.is_disk_uptodate
1219 is_degraded = not stats.is_connected
1220 return stats.sync_percent, stats.est_time, is_degraded or ldisk, ldisk
1222 def Open(self, force=False):
1223 """Make the local state primary.
1225 If the 'force' parameter is given, the '-o' option is passed to
1226 drbdsetup. Since this is a potentially dangerous operation, the
1227 force flag should be only given after creation, when it actually
1231 if self.minor is None and not self.Attach():
1232 logging.error("DRBD cannot attach to a device during open")
1234 cmd = ["drbdsetup", self.dev_path, "primary"]
1237 result = utils.RunCmd(cmd)
1239 msg = ("Can't make drbd device primary: %s" % result.output)
1241 raise errors.BlockDeviceError(msg)
1244 """Make the local state secondary.
1246 This will, of course, fail if the device is in use.
1249 if self.minor is None and not self.Attach():
1250 logging.info("Instance not attached to a device")
1251 raise errors.BlockDeviceError("Can't find device")
1252 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1254 msg = ("Can't switch drbd device to"
1255 " secondary: %s" % result.output)
1257 raise errors.BlockDeviceError(msg)
1259 def DisconnectNet(self):
1260 """Removes network configuration.
1262 This method shutdowns the network side of the device.
1264 The method will wait up to a hardcoded timeout for the device to
1265 go into standalone after the 'disconnect' command before
1266 re-configuring it, as sometimes it takes a while for the
1267 disconnect to actually propagate and thus we might issue a 'net'
1268 command while the device is still connected. If the device will
1269 still be attached to the network and we time out, we raise an
1273 if self.minor is None:
1274 raise errors.BlockDeviceError("DRBD disk not attached in re-attach net")
1276 if None in (self._lhost, self._lport, self._rhost, self._rport):
1277 raise errors.BlockDeviceError("DRBD disk missing network info in"
1280 ever_disconnected = self._ShutdownNet(self.minor)
1281 timeout_limit = time.time() + self._NET_RECONFIG_TIMEOUT
1282 sleep_time = 0.100 # we start the retry time at 100 miliseconds
1283 while time.time() < timeout_limit:
1284 status = self.GetProcStatus()
1285 if status.is_standalone:
1287 # retry the disconnect, it seems possible that due to a
1288 # well-time disconnect on the peer, my disconnect command might
1289 # be ingored and forgotten
1290 ever_disconnected = self._ShutdownNet(self.minor) or ever_disconnected
1291 time.sleep(sleep_time)
1292 sleep_time = min(2, sleep_time * 1.5)
1294 if not status.is_standalone:
1295 if ever_disconnected:
1296 msg = ("Device did not react to the"
1297 " 'disconnect' command in a timely manner")
1299 msg = ("Can't shutdown network, even after multiple retries")
1300 raise errors.BlockDeviceError(msg)
1302 reconfig_time = time.time() - timeout_limit + self._NET_RECONFIG_TIMEOUT
1303 if reconfig_time > 15: # hardcoded alert limit
1304 logging.debug("DRBD8.DisconnectNet: detach took %.3f seconds",
1307 def AttachNet(self, multimaster):
1308 """Reconnects the network.
1310 This method connects the network side of the device with a
1311 specified multi-master flag. The device needs to be 'Standalone'
1312 but have valid network configuration data.
1315 - multimaster: init the network in dual-primary mode
1318 if self.minor is None:
1319 raise errors.BlockDeviceError("DRBD disk not attached in AttachNet")
1321 if None in (self._lhost, self._lport, self._rhost, self._rport):
1322 raise errors.BlockDeviceError("DRBD disk missing network info in"
1325 status = self.GetProcStatus()
1327 if not status.is_standalone:
1328 raise errors.BlockDeviceError("Device is not standalone in AttachNet")
1330 return self._AssembleNet(self.minor,
1331 (self._lhost, self._lport,
1332 self._rhost, self._rport),
1333 "C", dual_pri=multimaster)
1336 """Check if our minor is configured.
1338 This doesn't do any device configurations - it only checks if the
1339 minor is in a state different from Unconfigured.
1341 Note that this function will not change the state of the system in
1342 any way (except in case of side-effects caused by reading from
1346 used_devs = self.GetUsedDevs()
1347 if self._aminor in used_devs:
1348 minor = self._aminor
1352 self._SetFromMinor(minor)
1353 return minor is not None
1356 """Assemble the drbd.
1359 - if we have a configured device, we try to ensure that it matches
1361 - if not, we create it from zero
1364 result = super(DRBD8, self).Assemble()
1369 if self.minor is None:
1370 # local device completely unconfigured
1371 return self._FastAssemble()
1373 # we have to recheck the local and network status and try to fix
1375 return self._SlowAssemble()
1377 def _SlowAssemble(self):
1378 """Assembles the DRBD device from a (partially) configured device.
1380 In case of partially attached (local device matches but no network
1381 setup), we perform the network attach. If successful, we re-test
1382 the attach if can return success.
1385 for minor in (self._aminor,):
1386 info = self._GetDevInfo(self._GetShowData(minor))
1387 match_l = self._MatchesLocal(info)
1388 match_r = self._MatchesNet(info)
1389 if match_l and match_r:
1391 if match_l and not match_r and "local_addr" not in info:
1392 res_r = self._AssembleNet(minor,
1393 (self._lhost, self._lport,
1394 self._rhost, self._rport),
1395 constants.DRBD_NET_PROTOCOL,
1396 hmac=constants.DRBD_HMAC_ALG,
1400 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1402 # the weakest case: we find something that is only net attached
1403 # even though we were passed some children at init time
1404 if match_r and "local_dev" not in info:
1407 # this case must be considered only if we actually have local
1408 # storage, i.e. not in diskless mode, because all diskless
1409 # devices are equal from the point of view of local
1411 if (match_l and "local_dev" in info and
1412 not match_r and "local_addr" in info):
1413 # strange case - the device network part points to somewhere
1414 # else, even though its local storage is ours; as we own the
1415 # drbd space, we try to disconnect from the remote peer and
1416 # reconnect to our correct one
1417 if not self._ShutdownNet(minor):
1418 raise errors.BlockDeviceError("Device has correct local storage,"
1419 " wrong remote peer and is unable to"
1420 " disconnect in order to attach to"
1421 " the correct peer")
1422 # note: _AssembleNet also handles the case when we don't want
1423 # local storage (i.e. one or more of the _[lr](host|port) is
1425 if (self._AssembleNet(minor, (self._lhost, self._lport,
1426 self._rhost, self._rport),
1427 constants.DRBD_NET_PROTOCOL,
1428 hmac=constants.DRBD_HMAC_ALG,
1429 secret=self._secret) and
1430 self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))):
1436 self._SetFromMinor(minor)
1437 return minor is not None
1439 def _FastAssemble(self):
1440 """Assemble the drbd device from zero.
1442 This is run when in Assemble we detect our minor is unused.
1445 # TODO: maybe completely tear-down the minor (drbdsetup ... down)
1446 # before attaching our own?
1447 minor = self._aminor
1448 need_localdev_teardown = False
1449 if self._children and self._children[0] and self._children[1]:
1450 result = self._AssembleLocal(minor, self._children[0].dev_path,
1451 self._children[1].dev_path)
1454 need_localdev_teardown = True
1455 if self._lhost and self._lport and self._rhost and self._rport:
1456 result = self._AssembleNet(minor,
1457 (self._lhost, self._lport,
1458 self._rhost, self._rport),
1459 constants.DRBD_NET_PROTOCOL,
1460 hmac=constants.DRBD_HMAC_ALG,
1461 secret=self._secret)
1463 if need_localdev_teardown:
1464 # we will ignore failures from this
1465 logging.error("net setup failed, tearing down local device")
1466 self._ShutdownAll(minor)
1468 self._SetFromMinor(minor)
1472 def _ShutdownLocal(cls, minor):
1473 """Detach from the local device.
1475 I/Os will continue to be served from the remote device. If we
1476 don't have a remote device, this operation will fail.
1479 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1481 logging.error("Can't detach local device: %s", result.output)
1482 return not result.failed
1485 def _ShutdownNet(cls, minor):
1486 """Disconnect from the remote peer.
1488 This fails if we don't have a local device.
1491 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1493 logging.error("Can't shutdown network: %s", result.output)
1494 return not result.failed
1497 def _ShutdownAll(cls, minor):
1498 """Deactivate the device.
1500 This will, of course, fail if the device is in use.
1503 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1505 logging.error("Can't shutdown drbd device: %s", result.output)
1506 return not result.failed
1509 """Shutdown the DRBD device.
1512 if self.minor is None and not self.Attach():
1513 logging.info("DRBD device not attached to a device during Shutdown")
1515 if not self._ShutdownAll(self.minor):
1518 self.dev_path = None
1522 """Stub remove for DRBD devices.
1525 return self.Shutdown()
1528 def Create(cls, unique_id, children, size):
1529 """Create a new DRBD8 device.
1531 Since DRBD devices are not created per se, just assembled, this
1532 function only initializes the metadata.
1535 if len(children) != 2:
1536 raise errors.ProgrammerError("Invalid setup for the drbd device")
1537 # check that the minor is unused
1538 aminor = unique_id[4]
1539 proc_info = cls._MassageProcData(cls._GetProcData())
1540 if aminor in proc_info:
1541 status = DRBD8Status(proc_info[aminor])
1542 in_use = status.is_in_use
1546 raise errors.BlockDeviceError("DRBD minor %d already in use at"
1547 " Create() time" % aminor)
1550 if not meta.Attach():
1551 raise errors.BlockDeviceError("Can't attach to meta device")
1552 if not cls._CheckMetaSize(meta.dev_path):
1553 raise errors.BlockDeviceError("Invalid meta device size")
1554 cls._InitMeta(cls._FindUnusedMinor(), meta.dev_path)
1555 if not cls._IsValidMeta(meta.dev_path):
1556 raise errors.BlockDeviceError("Cannot initalize meta device")
1557 return cls(unique_id, children)
1559 def Grow(self, amount):
1560 """Resize the DRBD device and its backing storage.
1563 if self.minor is None:
1564 raise errors.ProgrammerError("drbd8: Grow called while not attached")
1565 if len(self._children) != 2 or None in self._children:
1566 raise errors.BlockDeviceError("Cannot grow diskless DRBD8 device")
1567 self._children[0].Grow(amount)
1568 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize"])
1570 raise errors.BlockDeviceError("resize failed for %s: %s" %
1571 (self.dev_path, result.output))
1575 class FileStorage(BlockDev):
1578 This class represents the a file storage backend device.
1580 The unique_id for the file device is a (file_driver, file_path) tuple.
1583 def __init__(self, unique_id, children):
1584 """Initalizes a file device backend.
1588 raise errors.BlockDeviceError("Invalid setup for file device")
1589 super(FileStorage, self).__init__(unique_id, children)
1590 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1591 raise ValueError("Invalid configuration data %s" % str(unique_id))
1592 self.driver = unique_id[0]
1593 self.dev_path = unique_id[1]
1597 """Assemble the device.
1599 Checks whether the file device exists, raises BlockDeviceError otherwise.
1602 if not os.path.exists(self.dev_path):
1603 raise errors.BlockDeviceError("File device '%s' does not exist." %
1608 """Shutdown the device.
1610 This is a no-op for the file type, as we don't deacivate
1611 the file on shutdown.
1616 def Open(self, force=False):
1617 """Make the device ready for I/O.
1619 This is a no-op for the file type.
1625 """Notifies that the device will no longer be used for I/O.
1627 This is a no-op for the file type.
1633 """Remove the file backing the block device.
1636 @return: True if the removal was successful
1639 if not os.path.exists(self.dev_path):
1642 os.remove(self.dev_path)
1644 except OSError, err:
1645 logging.error("Can't remove file '%s': %s", self.dev_path, err)
1649 """Attach to an existing file.
1651 Check if this file already exists.
1654 @return: True if file exists
1657 self.attached = os.path.exists(self.dev_path)
1658 return self.attached
1661 def Create(cls, unique_id, children, size):
1662 """Create a new file.
1664 @param size: the size of file in MiB
1666 @rtype: L{bdev.FileStorage}
1667 @return: an instance of FileStorage
1670 # TODO: decide whether we should check for existing files and
1672 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1673 raise ValueError("Invalid configuration data %s" % str(unique_id))
1674 dev_path = unique_id[1]
1676 f = open(dev_path, 'w')
1677 f.truncate(size * 1024 * 1024)
1679 except IOError, err:
1680 raise errors.BlockDeviceError("Error in file creation: %" % str(err))
1682 return FileStorage(unique_id, children)
1686 constants.LD_LV: LogicalVolume,
1687 constants.LD_DRBD8: DRBD8,
1688 constants.LD_FILE: FileStorage,
1692 def FindDevice(dev_type, unique_id, children):
1693 """Search for an existing, assembled device.
1695 This will succeed only if the device exists and is assembled, but it
1696 does not do any actions in order to activate the device.
1699 if dev_type not in DEV_MAP:
1700 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1701 device = DEV_MAP[dev_type](unique_id, children)
1702 if not device.attached:
1707 def Assemble(dev_type, unique_id, children):
1708 """Try to attach or assemble an existing device.
1710 This will attach to assemble the device, as needed, to bring it
1711 fully up. It must be safe to run on already-assembled devices.
1714 if dev_type not in DEV_MAP:
1715 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1716 device = DEV_MAP[dev_type](unique_id, children)
1717 if not device.Assemble():
1718 raise errors.BlockDeviceError("Can't find a valid block device for"
1720 (dev_type, unique_id, children))
1724 def Create(dev_type, unique_id, children, size):
1728 if dev_type not in DEV_MAP:
1729 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1730 device = DEV_MAP[dev_type].Create(unique_id, children, size)