4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
31 from ganeti import utils
32 from ganeti import errors
33 from ganeti import constants
36 class BlockDev(object):
37 """Block device abstract class.
39 A block device can be in the following states:
40 - not existing on the system, and by `Create()` it goes into:
41 - existing but not setup/not active, and by `Assemble()` goes into:
42 - active read-write and by `Open()` it goes into
43 - online (=used, or ready for use)
45 A device can also be online but read-only, however we are not using
46 the readonly state (LV has it, if needed in the future) and we are
47 usually looking at this like at a stack, so it's easier to
48 conceptualise the transition from not-existing to online and back
51 The many different states of the device are due to the fact that we
52 need to cover many device types:
53 - logical volumes are created, lvchange -a y $lv, and used
54 - drbd devices are attached to a local disk/remote peer and made primary
56 A block device is identified by three items:
57 - the /dev path of the device (dynamic)
58 - a unique ID of the device (static)
59 - it's major/minor pair (dynamic)
61 Not all devices implement both the first two as distinct items. LVM
62 logical volumes have their unique ID (the pair volume group, logical
63 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
64 the /dev path is again dynamic and the unique id is the pair (host1,
67 You can get to a device in two ways:
68 - creating the (real) device, which returns you
69 an attached instance (lvcreate)
70 - attaching of a python instance to an existing (real) device
72 The second point, the attachement to a device, is different
73 depending on whether the device is assembled or not. At init() time,
74 we search for a device with the same unique_id as us. If found,
75 good. It also means that the device is already assembled. If not,
76 after assembly we'll have our correct major/minor.
79 def __init__(self, unique_id, children):
80 self._children = children
82 self.unique_id = unique_id
88 """Assemble the device from its components.
90 Implementations of this method by child classes must ensure that:
91 - after the device has been assembled, it knows its major/minor
92 numbers; this allows other devices (usually parents) to probe
93 correctly for their children
94 - calling this method on an existing, in-use device is safe
95 - if the device is already configured (and in an OK state),
96 this method is idempotent
102 """Find a device which matches our config and attach to it.
105 raise NotImplementedError
108 """Notifies that the device will no longer be used for I/O.
111 raise NotImplementedError
114 def Create(cls, unique_id, children, size):
115 """Create the device.
117 If the device cannot be created, it will return None
118 instead. Error messages go to the logging system.
120 Note that for some devices, the unique_id is used, and for other,
121 the children. The idea is that these two, taken together, are
122 enough for both creation and assembly (later).
125 raise NotImplementedError
128 """Remove this device.
130 This makes sense only for some of the device types: LV and file
131 storeage. Also note that if the device can't attach, the removal
135 raise NotImplementedError
137 def Rename(self, new_id):
138 """Rename this device.
140 This may or may not make sense for a given device type.
143 raise NotImplementedError
145 def Open(self, force=False):
146 """Make the device ready for use.
148 This makes the device ready for I/O. For now, just the DRBD
151 The force parameter signifies that if the device has any kind of
152 --force thing, it should be used, we know what we are doing.
155 raise NotImplementedError
158 """Shut down the device, freeing its children.
160 This undoes the `Assemble()` work, except for the child
161 assembling; as such, the children on the device are still
162 assembled after this call.
165 raise NotImplementedError
167 def SetSyncSpeed(self, speed):
168 """Adjust the sync speed of the mirror.
170 In case this is not a mirroring device, this is no-op.
175 for child in self._children:
176 result = result and child.SetSyncSpeed(speed)
179 def GetSyncStatus(self):
180 """Returns the sync status of the device.
182 If this device is a mirroring device, this function returns the
183 status of the mirror.
185 If sync_percent is None, it means the device is not syncing.
187 If estimated_time is None, it means we can't estimate
188 the time needed, otherwise it's the time left in seconds.
190 If is_degraded is True, it means the device is missing
191 redundancy. This is usually a sign that something went wrong in
192 the device setup, if sync_percent is None.
194 The ldisk parameter represents the degradation of the local
195 data. This is only valid for some devices, the rest will always
196 return False (not degraded).
199 @return: (sync_percent, estimated_time, is_degraded, ldisk)
202 return None, None, False, False
205 def CombinedSyncStatus(self):
206 """Calculate the mirror status recursively for our children.
208 The return value is the same as for `GetSyncStatus()` except the
209 minimum percent and maximum time are calculated across our
213 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
215 for child in self._children:
216 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
217 if min_percent is None:
218 min_percent = c_percent
219 elif c_percent is not None:
220 min_percent = min(min_percent, c_percent)
223 elif c_time is not None:
224 max_time = max(max_time, c_time)
225 is_degraded = is_degraded or c_degraded
226 ldisk = ldisk or c_ldisk
227 return min_percent, max_time, is_degraded, ldisk
230 def SetInfo(self, text):
231 """Update metadata with info text.
233 Only supported for some device types.
236 for child in self._children:
239 def Grow(self, amount):
240 """Grow the block device.
242 @param amount: the amount (in mebibytes) to grow with
245 raise NotImplementedError
248 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
249 (self.__class__, self.unique_id, self._children,
250 self.major, self.minor, self.dev_path))
253 class LogicalVolume(BlockDev):
254 """Logical Volume block device.
257 def __init__(self, unique_id, children):
258 """Attaches to a LV device.
260 The unique_id is a tuple (vg_name, lv_name)
263 super(LogicalVolume, self).__init__(unique_id, children)
264 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
265 raise ValueError("Invalid configuration data %s" % str(unique_id))
266 self._vg_name, self._lv_name = unique_id
267 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
268 self._degraded = True
269 self.major = self.minor = None
273 def Create(cls, unique_id, children, size):
274 """Create a new logical volume.
277 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
278 raise errors.ProgrammerError("Invalid configuration data %s" %
280 vg_name, lv_name = unique_id
281 pvs_info = cls.GetPVInfo(vg_name)
283 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
288 pvlist = [ pv[1] for pv in pvs_info ]
289 free_size = sum([ pv[0] for pv in pvs_info ])
291 # The size constraint should have been checked from the master before
292 # calling the create function.
294 raise errors.BlockDeviceError("Not enough free space: required %s,"
295 " available %s" % (size, free_size))
296 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
299 raise errors.BlockDeviceError("LV create failed (%s): %s" %
300 (result.fail_reason, result.output))
301 return LogicalVolume(unique_id, children)
304 def GetPVInfo(vg_name):
305 """Get the free space info for PVs in a volume group.
307 @param vg_name: the volume group name
310 @return: list of tuples (free_space, name) with free_space in mebibytes
313 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
314 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
316 result = utils.RunCmd(command)
318 logging.error("Can't get the PV information: %s - %s",
319 result.fail_reason, result.output)
322 for line in result.stdout.splitlines():
323 fields = line.strip().split(':')
325 logging.error("Can't parse pvs output: line '%s'", line)
327 # skip over pvs from another vg or ones which are not allocatable
328 if fields[1] != vg_name or fields[3][0] != 'a':
330 data.append((float(fields[2]), fields[0]))
335 """Remove this logical volume.
338 if not self.minor and not self.Attach():
339 # the LV does not exist
341 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
342 (self._vg_name, self._lv_name)])
344 logging.error("Can't lvremove: %s - %s",
345 result.fail_reason, result.output)
347 return not result.failed
349 def Rename(self, new_id):
350 """Rename this logical volume.
353 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
354 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
355 new_vg, new_name = new_id
356 if new_vg != self._vg_name:
357 raise errors.ProgrammerError("Can't move a logical volume across"
358 " volume groups (from %s to to %s)" %
359 (self._vg_name, new_vg))
360 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
362 raise errors.BlockDeviceError("Failed to rename the logical volume: %s" %
364 self._lv_name = new_name
365 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
368 """Attach to an existing LV.
370 This method will try to see if an existing and active LV exists
371 which matches our name. If so, its major/minor will be
375 self.attached = False
376 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
377 "-olv_attr,lv_kernel_major,lv_kernel_minor",
380 logging.error("Can't find LV %s: %s, %s",
381 self.dev_path, result.fail_reason, result.output)
383 out = result.stdout.strip().rstrip(',')
386 logging.error("Can't parse LVS output, len(%s) != 3", str(out))
389 status, major, minor = out[:3]
391 logging.error("lvs lv_attr is not 6 characters (%s)", status)
397 except ValueError, err:
398 logging.error("lvs major/minor cannot be parsed: %s", str(err))
402 self._degraded = status[0] == 'v' # virtual volume, i.e. doesn't backing
408 """Assemble the device.
410 We alway run `lvchange -ay` on the LV to ensure it's active before
411 use, as there were cases when xenvg was not active after boot
412 (also possibly after disk issues).
415 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
417 logging.error("Can't activate lv %s: %s", self.dev_path, result.output)
422 """Shutdown the device.
424 This is a no-op for the LV device type, as we don't deactivate the
430 def GetSyncStatus(self):
431 """Returns the sync status of the device.
433 If this device is a mirroring device, this function returns the
434 status of the mirror.
436 For logical volumes, sync_percent and estimated_time are always
437 None (no recovery in progress, as we don't handle the mirrored LV
438 case). The is_degraded parameter is the inverse of the ldisk
441 For the ldisk parameter, we check if the logical volume has the
442 'virtual' type, which means it's not backed by existing storage
443 anymore (read from it return I/O error). This happens after a
444 physical disk failure and subsequent 'vgreduce --removemissing' on
447 The status was already read in Attach, so we just return it.
450 @return: (sync_percent, estimated_time, is_degraded, ldisk)
453 return None, None, self._degraded, self._degraded
455 def Open(self, force=False):
456 """Make the device ready for I/O.
458 This is a no-op for the LV device type.
464 """Notifies that the device will no longer be used for I/O.
466 This is a no-op for the LV device type.
471 def Snapshot(self, size):
472 """Create a snapshot copy of an lvm block device.
475 snap_name = self._lv_name + ".snap"
477 # remove existing snapshot if found
478 snap = LogicalVolume((self._vg_name, snap_name), None)
481 pvs_info = self.GetPVInfo(self._vg_name)
483 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
487 free_size, pv_name = pvs_info[0]
489 raise errors.BlockDeviceError("Not enough free space: required %s,"
490 " available %s" % (size, free_size))
492 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
493 "-n%s" % snap_name, self.dev_path])
495 raise errors.BlockDeviceError("command: %s error: %s - %s" %
496 (result.cmd, result.fail_reason,
501 def SetInfo(self, text):
502 """Update metadata with info text.
505 BlockDev.SetInfo(self, text)
507 # Replace invalid characters
508 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
509 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
511 # Only up to 128 characters are allowed
514 result = utils.RunCmd(["lvchange", "--addtag", text,
517 raise errors.BlockDeviceError("Command: %s error: %s - %s" %
518 (result.cmd, result.fail_reason,
520 def Grow(self, amount):
521 """Grow the logical volume.
524 # we try multiple algorithms since the 'best' ones might not have
525 # space available in the right place, but later ones might (since
526 # they have less constraints); also note that only recent LVM
528 for alloc_policy in "contiguous", "cling", "normal":
529 result = utils.RunCmd(["lvextend", "--alloc", alloc_policy,
530 "-L", "+%dm" % amount, self.dev_path])
531 if not result.failed:
533 raise errors.BlockDeviceError("Can't grow LV %s: %s" %
534 (self.dev_path, result.output))
537 class DRBD8Status(object):
538 """A DRBD status representation class.
540 Note that this doesn't support unconfigured devices (cs:Unconfigured).
543 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+st:([^/]+)/(\S+)"
544 "\s+ds:([^/]+)/(\S+)\s+.*$")
545 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
546 "\sfinish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
548 def __init__(self, procline):
549 m = self.LINE_RE.match(procline)
551 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
552 self.cstatus = m.group(1)
553 self.lrole = m.group(2)
554 self.rrole = m.group(3)
555 self.ldisk = m.group(4)
556 self.rdisk = m.group(5)
558 self.is_standalone = self.cstatus == "StandAlone"
559 self.is_wfconn = self.cstatus == "WFConnection"
560 self.is_connected = self.cstatus == "Connected"
561 self.is_primary = self.lrole == "Primary"
562 self.is_secondary = self.lrole == "Secondary"
563 self.peer_primary = self.rrole == "Primary"
564 self.peer_secondary = self.rrole == "Secondary"
565 self.both_primary = self.is_primary and self.peer_primary
566 self.both_secondary = self.is_secondary and self.peer_secondary
568 self.is_diskless = self.ldisk == "Diskless"
569 self.is_disk_uptodate = self.ldisk == "UpToDate"
571 self.is_in_resync = self.cstatus in ('SyncSource', 'SyncTarget')
573 m = self.SYNC_RE.match(procline)
575 self.sync_percent = float(m.group(1))
576 hours = int(m.group(2))
577 minutes = int(m.group(3))
578 seconds = int(m.group(4))
579 self.est_time = hours * 3600 + minutes * 60 + seconds
581 self.sync_percent = None
584 self.is_sync_target = self.peer_sync_source = self.cstatus == "SyncTarget"
585 self.peer_sync_target = self.is_sync_source = self.cstatus == "SyncSource"
586 self.is_resync = self.is_sync_target or self.is_sync_source
589 class BaseDRBD(BlockDev):
592 This class contains a few bits of common functionality between the
593 0.7 and 8.x versions of DRBD.
596 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
597 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
600 _ST_UNCONFIGURED = "Unconfigured"
601 _ST_WFCONNECTION = "WFConnection"
602 _ST_CONNECTED = "Connected"
604 _STATUS_FILE = "/proc/drbd"
607 def _GetProcData(filename=_STATUS_FILE):
608 """Return data from /proc/drbd.
611 stat = open(filename, "r")
613 data = stat.read().splitlines()
617 raise errors.BlockDeviceError("Can't read any data from %s" % filename)
621 def _MassageProcData(data):
622 """Transform the output of _GetProdData into a nicer form.
624 @return: a dictionary of minor: joined lines from /proc/drbd
628 lmatch = re.compile("^ *([0-9]+):.*$")
630 old_minor = old_line = None
632 lresult = lmatch.match(line)
633 if lresult is not None:
634 if old_minor is not None:
635 results[old_minor] = old_line
636 old_minor = int(lresult.group(1))
639 if old_minor is not None:
640 old_line += " " + line.strip()
642 if old_minor is not None:
643 results[old_minor] = old_line
647 def _GetVersion(cls):
648 """Return the DRBD version.
650 This will return a dict with keys:
656 - proto2 (only on drbd > 8.2.X)
659 proc_data = cls._GetProcData()
660 first_line = proc_data[0].strip()
661 version = cls._VERSION_RE.match(first_line)
663 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
666 values = version.groups()
667 retval = {'k_major': int(values[0]),
668 'k_minor': int(values[1]),
669 'k_point': int(values[2]),
670 'api': int(values[3]),
671 'proto': int(values[4]),
673 if values[5] is not None:
674 retval['proto2'] = values[5]
680 """Return the path to a drbd device for a given minor.
683 return "/dev/drbd%d" % minor
686 def _GetUsedDevs(cls):
687 """Compute the list of used DRBD devices.
690 data = cls._GetProcData()
693 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
695 match = valid_line.match(line)
698 minor = int(match.group(1))
699 state = match.group(2)
700 if state == cls._ST_UNCONFIGURED:
702 used_devs[minor] = state, line
706 def _SetFromMinor(self, minor):
707 """Set our parameters based on the given minor.
709 This sets our minor variable and our dev_path.
713 self.minor = self.dev_path = None
714 self.attached = False
717 self.dev_path = self._DevPath(minor)
721 def _CheckMetaSize(meta_device):
722 """Check if the given meta device looks like a valid one.
724 This currently only check the size, which must be around
728 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
730 logging.error("Failed to get device size: %s - %s",
731 result.fail_reason, result.output)
734 sectors = int(result.stdout)
736 logging.error("Invalid output from blockdev: '%s'", result.stdout)
738 bytes = sectors * 512
739 if bytes < 128 * 1024 * 1024: # less than 128MiB
740 logging.error("Meta device too small (%.2fMib)", (bytes / 1024 / 1024))
742 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
743 logging.error("Meta device too big (%.2fMiB)", (bytes / 1024 / 1024))
747 def Rename(self, new_id):
750 This is not supported for drbd devices.
753 raise errors.ProgrammerError("Can't rename a drbd device")
756 class DRBD8(BaseDRBD):
757 """DRBD v8.x block device.
759 This implements the local host part of the DRBD device, i.e. it
760 doesn't do anything to the supposed peer. If you need a fully
761 connected DRBD pair, you need to use this class on both hosts.
763 The unique_id for the drbd device is the (local_ip, local_port,
764 remote_ip, remote_port) tuple, and it must have two children: the
765 data device and the meta_device. The meta device is checked for
766 valid size and is zeroed on create.
773 _NET_RECONFIG_TIMEOUT = 60
775 def __init__(self, unique_id, children):
776 if children and children.count(None) > 0:
778 super(DRBD8, self).__init__(unique_id, children)
779 self.major = self._DRBD_MAJOR
780 version = self._GetVersion()
781 if version['k_major'] != 8 :
782 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
783 " requested ganeti usage: kernel is"
784 " %s.%s, ganeti wants 8.x" %
785 (version['k_major'], version['k_minor']))
787 if len(children) not in (0, 2):
788 raise ValueError("Invalid configuration data %s" % str(children))
789 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
790 raise ValueError("Invalid configuration data %s" % str(unique_id))
791 (self._lhost, self._lport,
792 self._rhost, self._rport,
793 self._aminor, self._secret) = unique_id
794 if (self._lhost is not None and self._lhost == self._rhost and
795 self._lport == self._rport):
796 raise ValueError("Invalid configuration data, same local/remote %s" %
801 def _InitMeta(cls, minor, dev_path):
802 """Initialize a meta device.
804 This will not work if the given minor is in use.
807 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
808 "v08", dev_path, "0", "create-md"])
810 raise errors.BlockDeviceError("Can't initialize meta device: %s" %
814 def _FindUnusedMinor(cls):
815 """Find an unused DRBD device.
817 This is specific to 8.x as the minors are allocated dynamically,
818 so non-existing numbers up to a max minor count are actually free.
821 data = cls._GetProcData()
823 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
824 used_line = re.compile("^ *([0-9]+): cs:")
827 match = unused_line.match(line)
829 return int(match.group(1))
830 match = used_line.match(line)
832 minor = int(match.group(1))
833 highest = max(highest, minor)
834 if highest is None: # there are no minors in use at all
836 if highest >= cls._MAX_MINORS:
837 logging.error("Error: no free drbd minors!")
838 raise errors.BlockDeviceError("Can't find a free DRBD minor")
842 def _IsValidMeta(cls, meta_device):
843 """Check if the given meta device looks like a valid one.
846 minor = cls._FindUnusedMinor()
847 minor_path = cls._DevPath(minor)
848 result = utils.RunCmd(["drbdmeta", minor_path,
849 "v08", meta_device, "0",
852 logging.error("Invalid meta device %s: %s", meta_device, result.output)
857 def _GetShowParser(cls):
858 """Return a parser for `drbd show` output.
860 This will either create or return an already-create parser for the
861 output of the command `drbd show`.
864 if cls._PARSE_SHOW is not None:
865 return cls._PARSE_SHOW
868 lbrace = pyp.Literal("{").suppress()
869 rbrace = pyp.Literal("}").suppress()
870 semi = pyp.Literal(";").suppress()
871 # this also converts the value to an int
872 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
874 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
875 defa = pyp.Literal("_is_default").suppress()
876 dbl_quote = pyp.Literal('"').suppress()
878 keyword = pyp.Word(pyp.alphanums + '-')
881 value = pyp.Word(pyp.alphanums + '_-/.:')
882 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
883 addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() +
885 # meta device, extended syntax
886 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
887 number + pyp.Word(']').suppress())
890 stmt = (~rbrace + keyword + ~lbrace +
891 pyp.Optional(addr_port ^ value ^ quoted ^ meta_value) +
892 pyp.Optional(defa) + semi +
893 pyp.Optional(pyp.restOfLine).suppress())
896 section_name = pyp.Word(pyp.alphas + '_')
897 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
899 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
902 cls._PARSE_SHOW = bnf
907 def _GetShowData(cls, minor):
908 """Return the `drbdsetup show` data for a minor.
911 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
913 logging.error("Can't display the drbd config: %s - %s",
914 result.fail_reason, result.output)
919 def _GetDevInfo(cls, out):
920 """Parse details about a given DRBD minor.
922 This return, if available, the local backing device (as a path)
923 and the local and remote (ip, port) information from a string
924 containing the output of the `drbdsetup show` command as returned
932 bnf = cls._GetShowParser()
936 results = bnf.parseString(out)
937 except pyp.ParseException, err:
938 raise errors.BlockDeviceError("Can't parse drbdsetup show output: %s" %
941 # and massage the results into our desired format
942 for section in results:
944 if sname == "_this_host":
945 for lst in section[1:]:
947 data["local_dev"] = lst[1]
948 elif lst[0] == "meta-disk":
949 data["meta_dev"] = lst[1]
950 data["meta_index"] = lst[2]
951 elif lst[0] == "address":
952 data["local_addr"] = tuple(lst[1:])
953 elif sname == "_remote_host":
954 for lst in section[1:]:
955 if lst[0] == "address":
956 data["remote_addr"] = tuple(lst[1:])
959 def _MatchesLocal(self, info):
960 """Test if our local config matches with an existing device.
962 The parameter should be as returned from `_GetDevInfo()`. This
963 method tests if our local backing device is the same as the one in
964 the info parameter, in effect testing if we look like the given
969 backend, meta = self._children
971 backend = meta = None
973 if backend is not None:
974 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
976 retval = ("local_dev" not in info)
979 retval = retval and ("meta_dev" in info and
980 info["meta_dev"] == meta.dev_path)
981 retval = retval and ("meta_index" in info and
982 info["meta_index"] == 0)
984 retval = retval and ("meta_dev" not in info and
985 "meta_index" not in info)
988 def _MatchesNet(self, info):
989 """Test if our network config matches with an existing device.
991 The parameter should be as returned from `_GetDevInfo()`. This
992 method tests if our network configuration is the same as the one
993 in the info parameter, in effect testing if we look like the given
997 if (((self._lhost is None and not ("local_addr" in info)) and
998 (self._rhost is None and not ("remote_addr" in info)))):
1001 if self._lhost is None:
1004 if not ("local_addr" in info and
1005 "remote_addr" in info):
1008 retval = (info["local_addr"] == (self._lhost, self._lport))
1009 retval = (retval and
1010 info["remote_addr"] == (self._rhost, self._rport))
1014 def _AssembleLocal(cls, minor, backend, meta):
1015 """Configure the local part of a DRBD device.
1017 This is the first thing that must be done on an unconfigured DRBD
1018 device. And it must be done only once.
1021 if not cls._IsValidMeta(meta):
1023 args = ["drbdsetup", cls._DevPath(minor), "disk",
1024 backend, meta, "0", "-e", "detach", "--create-device"]
1025 result = utils.RunCmd(args)
1027 logging.error("Can't attach local disk: %s", result.output)
1028 return not result.failed
1031 def _AssembleNet(cls, minor, net_info, protocol,
1032 dual_pri=False, hmac=None, secret=None):
1033 """Configure the network part of the device.
1036 lhost, lport, rhost, rport = net_info
1037 if None in net_info:
1038 # we don't want network connection and actually want to make
1040 return cls._ShutdownNet(minor)
1042 # Workaround for a race condition. When DRBD is doing its dance to
1043 # establish a connection with its peer, it also sends the
1044 # synchronization speed over the wire. In some cases setting the
1045 # sync speed only after setting up both sides can race with DRBD
1046 # connecting, hence we set it here before telling DRBD anything
1048 cls._SetMinorSyncSpeed(minor, constants.SYNC_SPEED)
1050 args = ["drbdsetup", cls._DevPath(minor), "net",
1051 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
1052 "-A", "discard-zero-changes",
1059 args.extend(["-a", hmac, "-x", secret])
1060 result = utils.RunCmd(args)
1062 logging.error("Can't setup network for dbrd device: %s - %s",
1063 result.fail_reason, result.output)
1066 timeout = time.time() + 10
1068 while time.time() < timeout:
1069 info = cls._GetDevInfo(cls._GetShowData(minor))
1070 if not "local_addr" in info or not "remote_addr" in info:
1073 if (info["local_addr"] != (lhost, lport) or
1074 info["remote_addr"] != (rhost, rport)):
1080 logging.error("Timeout while configuring network")
1084 def AddChildren(self, devices):
1085 """Add a disk to the DRBD device.
1088 if self.minor is None:
1089 raise errors.BlockDeviceError("Can't attach to dbrd8 during AddChildren")
1090 if len(devices) != 2:
1091 raise errors.BlockDeviceError("Need two devices for AddChildren")
1092 info = self._GetDevInfo(self._GetShowData(self.minor))
1093 if "local_dev" in info:
1094 raise errors.BlockDeviceError("DRBD8 already attached to a local disk")
1095 backend, meta = devices
1096 if backend.dev_path is None or meta.dev_path is None:
1097 raise errors.BlockDeviceError("Children not ready during AddChildren")
1100 if not self._CheckMetaSize(meta.dev_path):
1101 raise errors.BlockDeviceError("Invalid meta device size")
1102 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1103 if not self._IsValidMeta(meta.dev_path):
1104 raise errors.BlockDeviceError("Cannot initalize meta device")
1106 if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path):
1107 raise errors.BlockDeviceError("Can't attach to local storage")
1108 self._children = devices
1110 def RemoveChildren(self, devices):
1111 """Detach the drbd device from local storage.
1114 if self.minor is None:
1115 raise errors.BlockDeviceError("Can't attach to drbd8 during"
1117 # early return if we don't actually have backing storage
1118 info = self._GetDevInfo(self._GetShowData(self.minor))
1119 if "local_dev" not in info:
1121 if len(self._children) != 2:
1122 raise errors.BlockDeviceError("We don't have two children: %s" %
1124 if self._children.count(None) == 2: # we don't actually have children :)
1125 logging.error("Requested detach while detached")
1127 if len(devices) != 2:
1128 raise errors.BlockDeviceError("We need two children in RemoveChildren")
1129 for child, dev in zip(self._children, devices):
1130 if dev != child.dev_path:
1131 raise errors.BlockDeviceError("Mismatch in local storage"
1132 " (%s != %s) in RemoveChildren" %
1133 (dev, child.dev_path))
1135 if not self._ShutdownLocal(self.minor):
1136 raise errors.BlockDeviceError("Can't detach from local storage")
1140 def _SetMinorSyncSpeed(cls, minor, kbytes):
1141 """Set the speed of the DRBD syncer.
1143 This is the low-level implementation.
1146 @param minor: the drbd minor whose settings we change
1148 @param kbytes: the speed in kbytes/second
1150 @return: the success of the operation
1153 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "syncer",
1154 "-r", "%d" % kbytes, "--create-device"])
1156 logging.error("Can't change syncer rate: %s - %s",
1157 result.fail_reason, result.output)
1158 return not result.failed
1160 def SetSyncSpeed(self, kbytes):
1161 """Set the speed of the DRBD syncer.
1164 @param kbytes: the speed in kbytes/second
1166 @return: the success of the operation
1169 if self.minor is None:
1170 logging.info("Not attached during SetSyncSpeed")
1172 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1173 return self._SetMinorSyncSpeed(self.minor, kbytes) and children_result
1175 def GetProcStatus(self):
1176 """Return device data from /proc.
1179 if self.minor is None:
1180 raise errors.BlockDeviceError("GetStats() called while not attached")
1181 proc_info = self._MassageProcData(self._GetProcData())
1182 if self.minor not in proc_info:
1183 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1185 return DRBD8Status(proc_info[self.minor])
1187 def GetSyncStatus(self):
1188 """Returns the sync status of the device.
1191 If sync_percent is None, it means all is ok
1192 If estimated_time is None, it means we can't esimate
1193 the time needed, otherwise it's the time left in seconds.
1196 We set the is_degraded parameter to True on two conditions:
1197 network not connected or local disk missing.
1199 We compute the ldisk parameter based on wheter we have a local
1203 @return: (sync_percent, estimated_time, is_degraded, ldisk)
1206 if self.minor is None and not self.Attach():
1207 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1208 stats = self.GetProcStatus()
1209 ldisk = not stats.is_disk_uptodate
1210 is_degraded = not stats.is_connected
1211 return stats.sync_percent, stats.est_time, is_degraded or ldisk, ldisk
1213 def Open(self, force=False):
1214 """Make the local state primary.
1216 If the 'force' parameter is given, the '-o' option is passed to
1217 drbdsetup. Since this is a potentially dangerous operation, the
1218 force flag should be only given after creation, when it actually
1222 if self.minor is None and not self.Attach():
1223 logging.error("DRBD cannot attach to a device during open")
1225 cmd = ["drbdsetup", self.dev_path, "primary"]
1228 result = utils.RunCmd(cmd)
1230 msg = ("Can't make drbd device primary: %s" % result.output)
1232 raise errors.BlockDeviceError(msg)
1235 """Make the local state secondary.
1237 This will, of course, fail if the device is in use.
1240 if self.minor is None and not self.Attach():
1241 logging.info("Instance not attached to a device")
1242 raise errors.BlockDeviceError("Can't find device")
1243 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1245 msg = ("Can't switch drbd device to"
1246 " secondary: %s" % result.output)
1248 raise errors.BlockDeviceError(msg)
1250 def DisconnectNet(self):
1251 """Removes network configuration.
1253 This method shutdowns the network side of the device.
1255 The method will wait up to a hardcoded timeout for the device to
1256 go into standalone after the 'disconnect' command before
1257 re-configuring it, as sometimes it takes a while for the
1258 disconnect to actually propagate and thus we might issue a 'net'
1259 command while the device is still connected. If the device will
1260 still be attached to the network and we time out, we raise an
1264 if self.minor is None:
1265 raise errors.BlockDeviceError("DRBD disk not attached in re-attach net")
1267 if None in (self._lhost, self._lport, self._rhost, self._rport):
1268 raise errors.BlockDeviceError("DRBD disk missing network info in"
1271 ever_disconnected = self._ShutdownNet(self.minor)
1272 timeout_limit = time.time() + self._NET_RECONFIG_TIMEOUT
1273 sleep_time = 0.100 # we start the retry time at 100 miliseconds
1274 while time.time() < timeout_limit:
1275 status = self.GetProcStatus()
1276 if status.is_standalone:
1278 # retry the disconnect, it seems possible that due to a
1279 # well-time disconnect on the peer, my disconnect command might
1280 # be ingored and forgotten
1281 ever_disconnected = self._ShutdownNet(self.minor) or ever_disconnected
1282 time.sleep(sleep_time)
1283 sleep_time = min(2, sleep_time * 1.5)
1285 if not status.is_standalone:
1286 if ever_disconnected:
1287 msg = ("Device did not react to the"
1288 " 'disconnect' command in a timely manner")
1290 msg = ("Can't shutdown network, even after multiple retries")
1291 raise errors.BlockDeviceError(msg)
1293 reconfig_time = time.time() - timeout_limit + self._NET_RECONFIG_TIMEOUT
1294 if reconfig_time > 15: # hardcoded alert limit
1295 logging.debug("DRBD8.DisconnectNet: detach took %.3f seconds",
1298 def AttachNet(self, multimaster):
1299 """Reconnects the network.
1301 This method connects the network side of the device with a
1302 specified multi-master flag. The device needs to be 'Standalone'
1303 but have valid network configuration data.
1306 - multimaster: init the network in dual-primary mode
1309 if self.minor is None:
1310 raise errors.BlockDeviceError("DRBD disk not attached in AttachNet")
1312 if None in (self._lhost, self._lport, self._rhost, self._rport):
1313 raise errors.BlockDeviceError("DRBD disk missing network info in"
1316 status = self.GetProcStatus()
1318 if not status.is_standalone:
1319 raise errors.BlockDeviceError("Device is not standalone in AttachNet")
1321 return self._AssembleNet(self.minor,
1322 (self._lhost, self._lport,
1323 self._rhost, self._rport),
1324 "C", dual_pri=multimaster)
1327 """Check if our minor is configured.
1329 This doesn't do any device configurations - it only checks if the
1330 minor is in a state different from Unconfigured.
1332 Note that this function will not change the state of the system in
1333 any way (except in case of side-effects caused by reading from
1337 used_devs = self._GetUsedDevs()
1338 if self._aminor in used_devs:
1339 minor = self._aminor
1343 self._SetFromMinor(minor)
1344 return minor is not None
1347 """Assemble the drbd.
1350 - if we have a configured device, we try to ensure that it matches
1352 - if not, we create it from zero
1355 result = super(DRBD8, self).Assemble()
1360 if self.minor is None:
1361 # local device completely unconfigured
1362 return self._FastAssemble()
1364 # we have to recheck the local and network status and try to fix
1366 return self._SlowAssemble()
1368 def _SlowAssemble(self):
1369 """Assembles the DRBD device from a (partially) configured device.
1371 In case of partially attached (local device matches but no network
1372 setup), we perform the network attach. If successful, we re-test
1373 the attach if can return success.
1376 for minor in (self._aminor,):
1377 info = self._GetDevInfo(self._GetShowData(minor))
1378 match_l = self._MatchesLocal(info)
1379 match_r = self._MatchesNet(info)
1380 if match_l and match_r:
1382 if match_l and not match_r and "local_addr" not in info:
1383 res_r = self._AssembleNet(minor,
1384 (self._lhost, self._lport,
1385 self._rhost, self._rport),
1386 constants.DRBD_NET_PROTOCOL,
1387 hmac=constants.DRBD_HMAC_ALG,
1391 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1393 # the weakest case: we find something that is only net attached
1394 # even though we were passed some children at init time
1395 if match_r and "local_dev" not in info:
1398 # this case must be considered only if we actually have local
1399 # storage, i.e. not in diskless mode, because all diskless
1400 # devices are equal from the point of view of local
1402 if (match_l and "local_dev" in info and
1403 not match_r and "local_addr" in info):
1404 # strange case - the device network part points to somewhere
1405 # else, even though its local storage is ours; as we own the
1406 # drbd space, we try to disconnect from the remote peer and
1407 # reconnect to our correct one
1408 if not self._ShutdownNet(minor):
1409 raise errors.BlockDeviceError("Device has correct local storage,"
1410 " wrong remote peer and is unable to"
1411 " disconnect in order to attach to"
1412 " the correct peer")
1413 # note: _AssembleNet also handles the case when we don't want
1414 # local storage (i.e. one or more of the _[lr](host|port) is
1416 if (self._AssembleNet(minor, (self._lhost, self._lport,
1417 self._rhost, self._rport),
1418 constants.DRBD_NET_PROTOCOL,
1419 hmac=constants.DRBD_HMAC_ALG,
1420 secret=self._secret) and
1421 self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))):
1427 self._SetFromMinor(minor)
1428 return minor is not None
1430 def _FastAssemble(self):
1431 """Assemble the drbd device from zero.
1433 This is run when in Assemble we detect our minor is unused.
1436 # TODO: maybe completely tear-down the minor (drbdsetup ... down)
1437 # before attaching our own?
1438 minor = self._aminor
1439 need_localdev_teardown = False
1440 if self._children and self._children[0] and self._children[1]:
1441 result = self._AssembleLocal(minor, self._children[0].dev_path,
1442 self._children[1].dev_path)
1445 need_localdev_teardown = True
1446 if self._lhost and self._lport and self._rhost and self._rport:
1447 result = self._AssembleNet(minor,
1448 (self._lhost, self._lport,
1449 self._rhost, self._rport),
1450 constants.DRBD_NET_PROTOCOL,
1451 hmac=constants.DRBD_HMAC_ALG,
1452 secret=self._secret)
1454 if need_localdev_teardown:
1455 # we will ignore failures from this
1456 logging.error("net setup failed, tearing down local device")
1457 self._ShutdownAll(minor)
1459 self._SetFromMinor(minor)
1463 def _ShutdownLocal(cls, minor):
1464 """Detach from the local device.
1466 I/Os will continue to be served from the remote device. If we
1467 don't have a remote device, this operation will fail.
1470 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1472 logging.error("Can't detach local device: %s", result.output)
1473 return not result.failed
1476 def _ShutdownNet(cls, minor):
1477 """Disconnect from the remote peer.
1479 This fails if we don't have a local device.
1482 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1484 logging.error("Can't shutdown network: %s", result.output)
1485 return not result.failed
1488 def _ShutdownAll(cls, minor):
1489 """Deactivate the device.
1491 This will, of course, fail if the device is in use.
1494 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1496 logging.error("Can't shutdown drbd device: %s", result.output)
1497 return not result.failed
1500 """Shutdown the DRBD device.
1503 if self.minor is None and not self.Attach():
1504 logging.info("DRBD device not attached to a device during Shutdown")
1506 if not self._ShutdownAll(self.minor):
1509 self.dev_path = None
1513 """Stub remove for DRBD devices.
1516 return self.Shutdown()
1519 def Create(cls, unique_id, children, size):
1520 """Create a new DRBD8 device.
1522 Since DRBD devices are not created per se, just assembled, this
1523 function only initializes the metadata.
1526 if len(children) != 2:
1527 raise errors.ProgrammerError("Invalid setup for the drbd device")
1530 if not meta.Attach():
1531 raise errors.BlockDeviceError("Can't attach to meta device")
1532 if not cls._CheckMetaSize(meta.dev_path):
1533 raise errors.BlockDeviceError("Invalid meta device size")
1534 cls._InitMeta(cls._FindUnusedMinor(), meta.dev_path)
1535 if not cls._IsValidMeta(meta.dev_path):
1536 raise errors.BlockDeviceError("Cannot initalize meta device")
1537 return cls(unique_id, children)
1539 def Grow(self, amount):
1540 """Resize the DRBD device and its backing storage.
1543 if self.minor is None:
1544 raise errors.ProgrammerError("drbd8: Grow called while not attached")
1545 if len(self._children) != 2 or None in self._children:
1546 raise errors.BlockDeviceError("Cannot grow diskless DRBD8 device")
1547 self._children[0].Grow(amount)
1548 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize"])
1550 raise errors.BlockDeviceError("resize failed for %s: %s" %
1551 (self.dev_path, result.output))
1555 class FileStorage(BlockDev):
1558 This class represents the a file storage backend device.
1560 The unique_id for the file device is a (file_driver, file_path) tuple.
1563 def __init__(self, unique_id, children):
1564 """Initalizes a file device backend.
1568 raise errors.BlockDeviceError("Invalid setup for file device")
1569 super(FileStorage, self).__init__(unique_id, children)
1570 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1571 raise ValueError("Invalid configuration data %s" % str(unique_id))
1572 self.driver = unique_id[0]
1573 self.dev_path = unique_id[1]
1577 """Assemble the device.
1579 Checks whether the file device exists, raises BlockDeviceError otherwise.
1582 if not os.path.exists(self.dev_path):
1583 raise errors.BlockDeviceError("File device '%s' does not exist." %
1588 """Shutdown the device.
1590 This is a no-op for the file type, as we don't deacivate
1591 the file on shutdown.
1596 def Open(self, force=False):
1597 """Make the device ready for I/O.
1599 This is a no-op for the file type.
1605 """Notifies that the device will no longer be used for I/O.
1607 This is a no-op for the file type.
1613 """Remove the file backing the block device.
1616 @return: True if the removal was successful
1619 if not os.path.exists(self.dev_path):
1622 os.remove(self.dev_path)
1624 except OSError, err:
1625 logging.error("Can't remove file '%s': %s", self.dev_path, err)
1629 """Attach to an existing file.
1631 Check if this file already exists.
1634 @return: True if file exists
1637 self.attached = os.path.exists(self.dev_path)
1638 return self.attached
1641 def Create(cls, unique_id, children, size):
1642 """Create a new file.
1644 @param size: the size of file in MiB
1646 @rtype: L{bdev.FileStorage}
1647 @return: an instance of FileStorage
1650 # TODO: decide whether we should check for existing files and
1652 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1653 raise ValueError("Invalid configuration data %s" % str(unique_id))
1654 dev_path = unique_id[1]
1656 f = open(dev_path, 'w')
1657 f.truncate(size * 1024 * 1024)
1659 except IOError, err:
1660 raise errors.BlockDeviceError("Error in file creation: %" % str(err))
1662 return FileStorage(unique_id, children)
1666 constants.LD_LV: LogicalVolume,
1667 constants.LD_DRBD8: DRBD8,
1668 constants.LD_FILE: FileStorage,
1672 def FindDevice(dev_type, unique_id, children):
1673 """Search for an existing, assembled device.
1675 This will succeed only if the device exists and is assembled, but it
1676 does not do any actions in order to activate the device.
1679 if dev_type not in DEV_MAP:
1680 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1681 device = DEV_MAP[dev_type](unique_id, children)
1682 if not device.attached:
1687 def Assemble(dev_type, unique_id, children):
1688 """Try to attach or assemble an existing device.
1690 This will attach to assemble the device, as needed, to bring it
1691 fully up. It must be safe to run on already-assembled devices.
1694 if dev_type not in DEV_MAP:
1695 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1696 device = DEV_MAP[dev_type](unique_id, children)
1697 if not device.Assemble():
1698 raise errors.BlockDeviceError("Can't find a valid block device for"
1700 (dev_type, unique_id, children))
1704 def Create(dev_type, unique_id, children, size):
1708 if dev_type not in DEV_MAP:
1709 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1710 device = DEV_MAP[dev_type].Create(unique_id, children, size)