4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
31 from ganeti import utils
32 from ganeti import errors
33 from ganeti import constants
36 def _IgnoreError(fn, *args, **kwargs):
37 """Executes the given function, ignoring BlockDeviceErrors.
39 This is used in order to simplify the execution of cleanup or
43 @return: True when fn didn't raise an exception, False otherwise
49 except errors.BlockDeviceError, err:
50 logging.warning("Caught BlockDeviceError but ignoring: %s" % str(err))
54 def _ThrowError(msg, *args):
55 """Log an error to the node daemon and the raise an exception.
58 @param msg: the text of the exception
59 @raise errors.BlockDeviceError
65 raise errors.BlockDeviceError(msg)
68 class BlockDev(object):
69 """Block device abstract class.
71 A block device can be in the following states:
72 - not existing on the system, and by `Create()` it goes into:
73 - existing but not setup/not active, and by `Assemble()` goes into:
74 - active read-write and by `Open()` it goes into
75 - online (=used, or ready for use)
77 A device can also be online but read-only, however we are not using
78 the readonly state (LV has it, if needed in the future) and we are
79 usually looking at this like at a stack, so it's easier to
80 conceptualise the transition from not-existing to online and back
83 The many different states of the device are due to the fact that we
84 need to cover many device types:
85 - logical volumes are created, lvchange -a y $lv, and used
86 - drbd devices are attached to a local disk/remote peer and made primary
88 A block device is identified by three items:
89 - the /dev path of the device (dynamic)
90 - a unique ID of the device (static)
91 - it's major/minor pair (dynamic)
93 Not all devices implement both the first two as distinct items. LVM
94 logical volumes have their unique ID (the pair volume group, logical
95 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
96 the /dev path is again dynamic and the unique id is the pair (host1,
99 You can get to a device in two ways:
100 - creating the (real) device, which returns you
101 an attached instance (lvcreate)
102 - attaching of a python instance to an existing (real) device
104 The second point, the attachement to a device, is different
105 depending on whether the device is assembled or not. At init() time,
106 we search for a device with the same unique_id as us. If found,
107 good. It also means that the device is already assembled. If not,
108 after assembly we'll have our correct major/minor.
111 def __init__(self, unique_id, children):
112 self._children = children
114 self.unique_id = unique_id
117 self.attached = False
120 """Assemble the device from its components.
122 Implementations of this method by child classes must ensure that:
123 - after the device has been assembled, it knows its major/minor
124 numbers; this allows other devices (usually parents) to probe
125 correctly for their children
126 - calling this method on an existing, in-use device is safe
127 - if the device is already configured (and in an OK state),
128 this method is idempotent
134 """Find a device which matches our config and attach to it.
137 raise NotImplementedError
140 """Notifies that the device will no longer be used for I/O.
143 raise NotImplementedError
146 def Create(cls, unique_id, children, size):
147 """Create the device.
149 If the device cannot be created, it will return None
150 instead. Error messages go to the logging system.
152 Note that for some devices, the unique_id is used, and for other,
153 the children. The idea is that these two, taken together, are
154 enough for both creation and assembly (later).
157 raise NotImplementedError
160 """Remove this device.
162 This makes sense only for some of the device types: LV and file
163 storeage. Also note that if the device can't attach, the removal
167 raise NotImplementedError
169 def Rename(self, new_id):
170 """Rename this device.
172 This may or may not make sense for a given device type.
175 raise NotImplementedError
177 def Open(self, force=False):
178 """Make the device ready for use.
180 This makes the device ready for I/O. For now, just the DRBD
183 The force parameter signifies that if the device has any kind of
184 --force thing, it should be used, we know what we are doing.
187 raise NotImplementedError
190 """Shut down the device, freeing its children.
192 This undoes the `Assemble()` work, except for the child
193 assembling; as such, the children on the device are still
194 assembled after this call.
197 raise NotImplementedError
199 def SetSyncSpeed(self, speed):
200 """Adjust the sync speed of the mirror.
202 In case this is not a mirroring device, this is no-op.
207 for child in self._children:
208 result = result and child.SetSyncSpeed(speed)
211 def GetSyncStatus(self):
212 """Returns the sync status of the device.
214 If this device is a mirroring device, this function returns the
215 status of the mirror.
217 If sync_percent is None, it means the device is not syncing.
219 If estimated_time is None, it means we can't estimate
220 the time needed, otherwise it's the time left in seconds.
222 If is_degraded is True, it means the device is missing
223 redundancy. This is usually a sign that something went wrong in
224 the device setup, if sync_percent is None.
226 The ldisk parameter represents the degradation of the local
227 data. This is only valid for some devices, the rest will always
228 return False (not degraded).
231 @return: (sync_percent, estimated_time, is_degraded, ldisk)
234 return None, None, False, False
237 def CombinedSyncStatus(self):
238 """Calculate the mirror status recursively for our children.
240 The return value is the same as for `GetSyncStatus()` except the
241 minimum percent and maximum time are calculated across our
245 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
247 for child in self._children:
248 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
249 if min_percent is None:
250 min_percent = c_percent
251 elif c_percent is not None:
252 min_percent = min(min_percent, c_percent)
255 elif c_time is not None:
256 max_time = max(max_time, c_time)
257 is_degraded = is_degraded or c_degraded
258 ldisk = ldisk or c_ldisk
259 return min_percent, max_time, is_degraded, ldisk
262 def SetInfo(self, text):
263 """Update metadata with info text.
265 Only supported for some device types.
268 for child in self._children:
271 def Grow(self, amount):
272 """Grow the block device.
274 @param amount: the amount (in mebibytes) to grow with
277 raise NotImplementedError
280 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
281 (self.__class__, self.unique_id, self._children,
282 self.major, self.minor, self.dev_path))
285 class LogicalVolume(BlockDev):
286 """Logical Volume block device.
289 def __init__(self, unique_id, children):
290 """Attaches to a LV device.
292 The unique_id is a tuple (vg_name, lv_name)
295 super(LogicalVolume, self).__init__(unique_id, children)
296 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
297 raise ValueError("Invalid configuration data %s" % str(unique_id))
298 self._vg_name, self._lv_name = unique_id
299 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
300 self._degraded = True
301 self.major = self.minor = None
305 def Create(cls, unique_id, children, size):
306 """Create a new logical volume.
309 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
310 raise errors.ProgrammerError("Invalid configuration data %s" %
312 vg_name, lv_name = unique_id
313 pvs_info = cls.GetPVInfo(vg_name)
315 _ThrowError("Can't compute PV info for vg %s", vg_name)
319 pvlist = [ pv[1] for pv in pvs_info ]
320 free_size = sum([ pv[0] for pv in pvs_info ])
322 # The size constraint should have been checked from the master before
323 # calling the create function.
325 _ThrowError("Not enough free space: required %s,"
326 " available %s", size, free_size)
327 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
330 _ThrowError("LV create failed (%s): %s",
331 result.fail_reason, result.output)
332 return LogicalVolume(unique_id, children)
335 def GetPVInfo(vg_name):
336 """Get the free space info for PVs in a volume group.
338 @param vg_name: the volume group name
341 @return: list of tuples (free_space, name) with free_space in mebibytes
344 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
345 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
347 result = utils.RunCmd(command)
349 logging.error("Can't get the PV information: %s - %s",
350 result.fail_reason, result.output)
353 for line in result.stdout.splitlines():
354 fields = line.strip().split(':')
356 logging.error("Can't parse pvs output: line '%s'", line)
358 # skip over pvs from another vg or ones which are not allocatable
359 if fields[1] != vg_name or fields[3][0] != 'a':
361 data.append((float(fields[2]), fields[0]))
366 """Remove this logical volume.
369 if not self.minor and not self.Attach():
370 # the LV does not exist
372 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
373 (self._vg_name, self._lv_name)])
375 logging.error("Can't lvremove: %s - %s",
376 result.fail_reason, result.output)
378 return not result.failed
380 def Rename(self, new_id):
381 """Rename this logical volume.
384 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
385 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
386 new_vg, new_name = new_id
387 if new_vg != self._vg_name:
388 raise errors.ProgrammerError("Can't move a logical volume across"
389 " volume groups (from %s to to %s)" %
390 (self._vg_name, new_vg))
391 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
393 _ThrowError("Failed to rename the logical volume: %s", result.output)
394 self._lv_name = new_name
395 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
398 """Attach to an existing LV.
400 This method will try to see if an existing and active LV exists
401 which matches our name. If so, its major/minor will be
405 self.attached = False
406 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
407 "-olv_attr,lv_kernel_major,lv_kernel_minor",
410 logging.error("Can't find LV %s: %s, %s",
411 self.dev_path, result.fail_reason, result.output)
413 out = result.stdout.strip().rstrip(',')
416 logging.error("Can't parse LVS output, len(%s) != 3", str(out))
419 status, major, minor = out[:3]
421 logging.error("lvs lv_attr is not 6 characters (%s)", status)
427 except ValueError, err:
428 logging.error("lvs major/minor cannot be parsed: %s", str(err))
432 self._degraded = status[0] == 'v' # virtual volume, i.e. doesn't backing
438 """Assemble the device.
440 We alway run `lvchange -ay` on the LV to ensure it's active before
441 use, as there were cases when xenvg was not active after boot
442 (also possibly after disk issues).
445 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
447 logging.error("Can't activate lv %s: %s", self.dev_path, result.output)
452 """Shutdown the device.
454 This is a no-op for the LV device type, as we don't deactivate the
460 def GetSyncStatus(self):
461 """Returns the sync status of the device.
463 If this device is a mirroring device, this function returns the
464 status of the mirror.
466 For logical volumes, sync_percent and estimated_time are always
467 None (no recovery in progress, as we don't handle the mirrored LV
468 case). The is_degraded parameter is the inverse of the ldisk
471 For the ldisk parameter, we check if the logical volume has the
472 'virtual' type, which means it's not backed by existing storage
473 anymore (read from it return I/O error). This happens after a
474 physical disk failure and subsequent 'vgreduce --removemissing' on
477 The status was already read in Attach, so we just return it.
480 @return: (sync_percent, estimated_time, is_degraded, ldisk)
483 return None, None, self._degraded, self._degraded
485 def Open(self, force=False):
486 """Make the device ready for I/O.
488 This is a no-op for the LV device type.
494 """Notifies that the device will no longer be used for I/O.
496 This is a no-op for the LV device type.
501 def Snapshot(self, size):
502 """Create a snapshot copy of an lvm block device.
505 snap_name = self._lv_name + ".snap"
507 # remove existing snapshot if found
508 snap = LogicalVolume((self._vg_name, snap_name), None)
511 pvs_info = self.GetPVInfo(self._vg_name)
513 _ThrowError("Can't compute PV info for vg %s", self._vg_name)
516 free_size, pv_name = pvs_info[0]
518 _ThrowError("Not enough free space: required %s,"
519 " available %s", size, free_size)
521 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
522 "-n%s" % snap_name, self.dev_path])
524 _ThrowError("command: %s error: %s - %s",
525 result.cmd, result.fail_reason, result.output)
529 def SetInfo(self, text):
530 """Update metadata with info text.
533 BlockDev.SetInfo(self, text)
535 # Replace invalid characters
536 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
537 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
539 # Only up to 128 characters are allowed
542 result = utils.RunCmd(["lvchange", "--addtag", text,
545 _ThrowError("Command: %s error: %s - %s", result.cmd, result.fail_reason,
548 def Grow(self, amount):
549 """Grow the logical volume.
552 # we try multiple algorithms since the 'best' ones might not have
553 # space available in the right place, but later ones might (since
554 # they have less constraints); also note that only recent LVM
556 for alloc_policy in "contiguous", "cling", "normal":
557 result = utils.RunCmd(["lvextend", "--alloc", alloc_policy,
558 "-L", "+%dm" % amount, self.dev_path])
559 if not result.failed:
561 _ThrowError("Can't grow LV %s: %s", self.dev_path, result.output)
564 class DRBD8Status(object):
565 """A DRBD status representation class.
567 Note that this doesn't support unconfigured devices (cs:Unconfigured).
570 UNCONF_RE = re.compile(r"\s*[0-9]+:\s*cs:Unconfigured$")
571 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+st:([^/]+)/(\S+)"
572 "\s+ds:([^/]+)/(\S+)\s+.*$")
573 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
574 "\sfinish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
576 def __init__(self, procline):
577 u = self.UNCONF_RE.match(procline)
579 self.cstatus = "Unconfigured"
580 self.lrole = self.rrole = self.ldisk = self.rdisk = None
582 m = self.LINE_RE.match(procline)
584 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
585 self.cstatus = m.group(1)
586 self.lrole = m.group(2)
587 self.rrole = m.group(3)
588 self.ldisk = m.group(4)
589 self.rdisk = m.group(5)
591 # end reading of data from the LINE_RE or UNCONF_RE
593 self.is_standalone = self.cstatus == "StandAlone"
594 self.is_wfconn = self.cstatus == "WFConnection"
595 self.is_connected = self.cstatus == "Connected"
596 self.is_primary = self.lrole == "Primary"
597 self.is_secondary = self.lrole == "Secondary"
598 self.peer_primary = self.rrole == "Primary"
599 self.peer_secondary = self.rrole == "Secondary"
600 self.both_primary = self.is_primary and self.peer_primary
601 self.both_secondary = self.is_secondary and self.peer_secondary
603 self.is_diskless = self.ldisk == "Diskless"
604 self.is_disk_uptodate = self.ldisk == "UpToDate"
606 self.is_in_resync = self.cstatus in ("SyncSource", "SyncTarget")
607 self.is_in_use = self.cstatus != "Unconfigured"
609 m = self.SYNC_RE.match(procline)
611 self.sync_percent = float(m.group(1))
612 hours = int(m.group(2))
613 minutes = int(m.group(3))
614 seconds = int(m.group(4))
615 self.est_time = hours * 3600 + minutes * 60 + seconds
617 self.sync_percent = None
620 self.is_sync_target = self.peer_sync_source = self.cstatus == "SyncTarget"
621 self.peer_sync_target = self.is_sync_source = self.cstatus == "SyncSource"
622 self.is_resync = self.is_sync_target or self.is_sync_source
625 class BaseDRBD(BlockDev):
628 This class contains a few bits of common functionality between the
629 0.7 and 8.x versions of DRBD.
632 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
633 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
636 _ST_UNCONFIGURED = "Unconfigured"
637 _ST_WFCONNECTION = "WFConnection"
638 _ST_CONNECTED = "Connected"
640 _STATUS_FILE = "/proc/drbd"
643 def _GetProcData(filename=_STATUS_FILE):
644 """Return data from /proc/drbd.
647 stat = open(filename, "r")
649 data = stat.read().splitlines()
653 _ThrowError("Can't read any data from %s", filename)
657 def _MassageProcData(data):
658 """Transform the output of _GetProdData into a nicer form.
660 @return: a dictionary of minor: joined lines from /proc/drbd
664 lmatch = re.compile("^ *([0-9]+):.*$")
666 old_minor = old_line = None
668 lresult = lmatch.match(line)
669 if lresult is not None:
670 if old_minor is not None:
671 results[old_minor] = old_line
672 old_minor = int(lresult.group(1))
675 if old_minor is not None:
676 old_line += " " + line.strip()
678 if old_minor is not None:
679 results[old_minor] = old_line
683 def _GetVersion(cls):
684 """Return the DRBD version.
686 This will return a dict with keys:
692 - proto2 (only on drbd > 8.2.X)
695 proc_data = cls._GetProcData()
696 first_line = proc_data[0].strip()
697 version = cls._VERSION_RE.match(first_line)
699 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
702 values = version.groups()
703 retval = {'k_major': int(values[0]),
704 'k_minor': int(values[1]),
705 'k_point': int(values[2]),
706 'api': int(values[3]),
707 'proto': int(values[4]),
709 if values[5] is not None:
710 retval['proto2'] = values[5]
716 """Return the path to a drbd device for a given minor.
719 return "/dev/drbd%d" % minor
722 def GetUsedDevs(cls):
723 """Compute the list of used DRBD devices.
726 data = cls._GetProcData()
729 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
731 match = valid_line.match(line)
734 minor = int(match.group(1))
735 state = match.group(2)
736 if state == cls._ST_UNCONFIGURED:
738 used_devs[minor] = state, line
742 def _SetFromMinor(self, minor):
743 """Set our parameters based on the given minor.
745 This sets our minor variable and our dev_path.
749 self.minor = self.dev_path = None
750 self.attached = False
753 self.dev_path = self._DevPath(minor)
757 def _CheckMetaSize(meta_device):
758 """Check if the given meta device looks like a valid one.
760 This currently only check the size, which must be around
764 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
766 logging.error("Failed to get device size: %s - %s",
767 result.fail_reason, result.output)
770 sectors = int(result.stdout)
772 logging.error("Invalid output from blockdev: '%s'", result.stdout)
774 bytes = sectors * 512
775 if bytes < 128 * 1024 * 1024: # less than 128MiB
776 logging.error("Meta device too small (%.2fMib)", (bytes / 1024 / 1024))
778 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
779 logging.error("Meta device too big (%.2fMiB)", (bytes / 1024 / 1024))
783 def Rename(self, new_id):
786 This is not supported for drbd devices.
789 raise errors.ProgrammerError("Can't rename a drbd device")
792 class DRBD8(BaseDRBD):
793 """DRBD v8.x block device.
795 This implements the local host part of the DRBD device, i.e. it
796 doesn't do anything to the supposed peer. If you need a fully
797 connected DRBD pair, you need to use this class on both hosts.
799 The unique_id for the drbd device is the (local_ip, local_port,
800 remote_ip, remote_port) tuple, and it must have two children: the
801 data device and the meta_device. The meta device is checked for
802 valid size and is zeroed on create.
809 _NET_RECONFIG_TIMEOUT = 60
811 def __init__(self, unique_id, children):
812 if children and children.count(None) > 0:
814 super(DRBD8, self).__init__(unique_id, children)
815 self.major = self._DRBD_MAJOR
816 version = self._GetVersion()
817 if version['k_major'] != 8 :
818 _ThrowError("Mismatch in DRBD kernel version and requested ganeti"
819 " usage: kernel is %s.%s, ganeti wants 8.x",
820 version['k_major'], version['k_minor'])
822 if len(children) not in (0, 2):
823 raise ValueError("Invalid configuration data %s" % str(children))
824 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
825 raise ValueError("Invalid configuration data %s" % str(unique_id))
826 (self._lhost, self._lport,
827 self._rhost, self._rport,
828 self._aminor, self._secret) = unique_id
829 if (self._lhost is not None and self._lhost == self._rhost and
830 self._lport == self._rport):
831 raise ValueError("Invalid configuration data, same local/remote %s" %
836 def _InitMeta(cls, minor, dev_path):
837 """Initialize a meta device.
839 This will not work if the given minor is in use.
842 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
843 "v08", dev_path, "0", "create-md"])
845 _ThrowError("Can't initialize meta device: %s", result.output)
848 def _FindUnusedMinor(cls):
849 """Find an unused DRBD device.
851 This is specific to 8.x as the minors are allocated dynamically,
852 so non-existing numbers up to a max minor count are actually free.
855 data = cls._GetProcData()
857 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
858 used_line = re.compile("^ *([0-9]+): cs:")
861 match = unused_line.match(line)
863 return int(match.group(1))
864 match = used_line.match(line)
866 minor = int(match.group(1))
867 highest = max(highest, minor)
868 if highest is None: # there are no minors in use at all
870 if highest >= cls._MAX_MINORS:
871 logging.error("Error: no free drbd minors!")
872 raise errors.BlockDeviceError("Can't find a free DRBD minor")
876 def _GetShowParser(cls):
877 """Return a parser for `drbd show` output.
879 This will either create or return an already-create parser for the
880 output of the command `drbd show`.
883 if cls._PARSE_SHOW is not None:
884 return cls._PARSE_SHOW
887 lbrace = pyp.Literal("{").suppress()
888 rbrace = pyp.Literal("}").suppress()
889 semi = pyp.Literal(";").suppress()
890 # this also converts the value to an int
891 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
893 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
894 defa = pyp.Literal("_is_default").suppress()
895 dbl_quote = pyp.Literal('"').suppress()
897 keyword = pyp.Word(pyp.alphanums + '-')
900 value = pyp.Word(pyp.alphanums + '_-/.:')
901 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
902 addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() +
904 # meta device, extended syntax
905 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
906 number + pyp.Word(']').suppress())
909 stmt = (~rbrace + keyword + ~lbrace +
910 pyp.Optional(addr_port ^ value ^ quoted ^ meta_value) +
911 pyp.Optional(defa) + semi +
912 pyp.Optional(pyp.restOfLine).suppress())
915 section_name = pyp.Word(pyp.alphas + '_')
916 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
918 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
921 cls._PARSE_SHOW = bnf
926 def _GetShowData(cls, minor):
927 """Return the `drbdsetup show` data for a minor.
930 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
932 logging.error("Can't display the drbd config: %s - %s",
933 result.fail_reason, result.output)
938 def _GetDevInfo(cls, out):
939 """Parse details about a given DRBD minor.
941 This return, if available, the local backing device (as a path)
942 and the local and remote (ip, port) information from a string
943 containing the output of the `drbdsetup show` command as returned
951 bnf = cls._GetShowParser()
955 results = bnf.parseString(out)
956 except pyp.ParseException, err:
957 _ThrowError("Can't parse drbdsetup show output: %s", str(err))
959 # and massage the results into our desired format
960 for section in results:
962 if sname == "_this_host":
963 for lst in section[1:]:
965 data["local_dev"] = lst[1]
966 elif lst[0] == "meta-disk":
967 data["meta_dev"] = lst[1]
968 data["meta_index"] = lst[2]
969 elif lst[0] == "address":
970 data["local_addr"] = tuple(lst[1:])
971 elif sname == "_remote_host":
972 for lst in section[1:]:
973 if lst[0] == "address":
974 data["remote_addr"] = tuple(lst[1:])
977 def _MatchesLocal(self, info):
978 """Test if our local config matches with an existing device.
980 The parameter should be as returned from `_GetDevInfo()`. This
981 method tests if our local backing device is the same as the one in
982 the info parameter, in effect testing if we look like the given
987 backend, meta = self._children
989 backend = meta = None
991 if backend is not None:
992 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
994 retval = ("local_dev" not in info)
997 retval = retval and ("meta_dev" in info and
998 info["meta_dev"] == meta.dev_path)
999 retval = retval and ("meta_index" in info and
1000 info["meta_index"] == 0)
1002 retval = retval and ("meta_dev" not in info and
1003 "meta_index" not in info)
1006 def _MatchesNet(self, info):
1007 """Test if our network config matches with an existing device.
1009 The parameter should be as returned from `_GetDevInfo()`. This
1010 method tests if our network configuration is the same as the one
1011 in the info parameter, in effect testing if we look like the given
1015 if (((self._lhost is None and not ("local_addr" in info)) and
1016 (self._rhost is None and not ("remote_addr" in info)))):
1019 if self._lhost is None:
1022 if not ("local_addr" in info and
1023 "remote_addr" in info):
1026 retval = (info["local_addr"] == (self._lhost, self._lport))
1027 retval = (retval and
1028 info["remote_addr"] == (self._rhost, self._rport))
1032 def _AssembleLocal(cls, minor, backend, meta):
1033 """Configure the local part of a DRBD device.
1036 args = ["drbdsetup", cls._DevPath(minor), "disk",
1037 backend, meta, "0", "-e", "detach", "--create-device"]
1038 result = utils.RunCmd(args)
1040 logging.error("Can't attach local disk: %s", result.output)
1041 return not result.failed
1044 def _AssembleNet(cls, minor, net_info, protocol,
1045 dual_pri=False, hmac=None, secret=None):
1046 """Configure the network part of the device.
1049 lhost, lport, rhost, rport = net_info
1050 if None in net_info:
1051 # we don't want network connection and actually want to make
1053 return cls._ShutdownNet(minor)
1055 # Workaround for a race condition. When DRBD is doing its dance to
1056 # establish a connection with its peer, it also sends the
1057 # synchronization speed over the wire. In some cases setting the
1058 # sync speed only after setting up both sides can race with DRBD
1059 # connecting, hence we set it here before telling DRBD anything
1061 cls._SetMinorSyncSpeed(minor, constants.SYNC_SPEED)
1063 args = ["drbdsetup", cls._DevPath(minor), "net",
1064 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
1065 "-A", "discard-zero-changes",
1072 args.extend(["-a", hmac, "-x", secret])
1073 result = utils.RunCmd(args)
1075 logging.error("Can't setup network for dbrd device: %s - %s",
1076 result.fail_reason, result.output)
1079 timeout = time.time() + 10
1081 while time.time() < timeout:
1082 info = cls._GetDevInfo(cls._GetShowData(minor))
1083 if not "local_addr" in info or not "remote_addr" in info:
1086 if (info["local_addr"] != (lhost, lport) or
1087 info["remote_addr"] != (rhost, rport)):
1093 logging.error("Timeout while configuring network")
1097 def AddChildren(self, devices):
1098 """Add a disk to the DRBD device.
1101 if self.minor is None:
1102 _ThrowError("drbd%d: can't attach to dbrd8 during AddChildren",
1104 if len(devices) != 2:
1105 _ThrowError("drbd%d: need two devices for AddChildren", self.minor)
1106 info = self._GetDevInfo(self._GetShowData(self.minor))
1107 if "local_dev" in info:
1108 _ThrowError("drbd%d: already attached to a local disk", self.minor)
1109 backend, meta = devices
1110 if backend.dev_path is None or meta.dev_path is None:
1111 _ThrowError("drbd%d: children not ready during AddChildren", self.minor)
1114 if not self._CheckMetaSize(meta.dev_path):
1115 raise errors.BlockDeviceError("Invalid meta device size")
1116 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1118 if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path):
1119 raise errors.BlockDeviceError("Can't attach to local storage")
1120 self._children = devices
1122 def RemoveChildren(self, devices):
1123 """Detach the drbd device from local storage.
1126 if self.minor is None:
1127 _ThrowError("drbd%d: can't attach to drbd8 during RemoveChildren",
1129 # early return if we don't actually have backing storage
1130 info = self._GetDevInfo(self._GetShowData(self.minor))
1131 if "local_dev" not in info:
1133 if len(self._children) != 2:
1134 _ThrowError("drbd%d: we don't have two children: %s", self.minor,
1136 if self._children.count(None) == 2: # we don't actually have children :)
1137 logging.warning("drbd%d: requested detach while detached", self.minor)
1139 if len(devices) != 2:
1140 _ThrowError("drbd%d: we need two children in RemoveChildren", self.minor)
1141 for child, dev in zip(self._children, devices):
1142 if dev != child.dev_path:
1143 _ThrowError("drbd%d: mismatch in local storage (%s != %s) in"
1144 " RemoveChildren", self.minor, dev, child.dev_path)
1146 if not self._ShutdownLocal(self.minor):
1147 raise errors.BlockDeviceError("Can't detach from local storage")
1151 def _SetMinorSyncSpeed(cls, minor, kbytes):
1152 """Set the speed of the DRBD syncer.
1154 This is the low-level implementation.
1157 @param minor: the drbd minor whose settings we change
1159 @param kbytes: the speed in kbytes/second
1161 @return: the success of the operation
1164 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "syncer",
1165 "-r", "%d" % kbytes, "--create-device"])
1167 logging.error("Can't change syncer rate: %s - %s",
1168 result.fail_reason, result.output)
1169 return not result.failed
1171 def SetSyncSpeed(self, kbytes):
1172 """Set the speed of the DRBD syncer.
1175 @param kbytes: the speed in kbytes/second
1177 @return: the success of the operation
1180 if self.minor is None:
1181 logging.info("Not attached during SetSyncSpeed")
1183 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1184 return self._SetMinorSyncSpeed(self.minor, kbytes) and children_result
1186 def GetProcStatus(self):
1187 """Return device data from /proc.
1190 if self.minor is None:
1191 _ThrowError("drbd%d: GetStats() called while not attached", self._aminor)
1192 proc_info = self._MassageProcData(self._GetProcData())
1193 if self.minor not in proc_info:
1194 _ThrowError("drbd%d: can't find myself in /proc", self.minor)
1195 return DRBD8Status(proc_info[self.minor])
1197 def GetSyncStatus(self):
1198 """Returns the sync status of the device.
1201 If sync_percent is None, it means all is ok
1202 If estimated_time is None, it means we can't esimate
1203 the time needed, otherwise it's the time left in seconds.
1206 We set the is_degraded parameter to True on two conditions:
1207 network not connected or local disk missing.
1209 We compute the ldisk parameter based on wheter we have a local
1213 @return: (sync_percent, estimated_time, is_degraded, ldisk)
1216 if self.minor is None and not self.Attach():
1217 _ThrowError("drbd%d: can't Attach() in GetSyncStatus", self._aminor)
1218 stats = self.GetProcStatus()
1219 ldisk = not stats.is_disk_uptodate
1220 is_degraded = not stats.is_connected
1221 return stats.sync_percent, stats.est_time, is_degraded or ldisk, ldisk
1223 def Open(self, force=False):
1224 """Make the local state primary.
1226 If the 'force' parameter is given, the '-o' option is passed to
1227 drbdsetup. Since this is a potentially dangerous operation, the
1228 force flag should be only given after creation, when it actually
1232 if self.minor is None and not self.Attach():
1233 logging.error("DRBD cannot attach to a device during open")
1235 cmd = ["drbdsetup", self.dev_path, "primary"]
1238 result = utils.RunCmd(cmd)
1240 _ThrowError("drbd%d: can't make drbd device primary: %s", self.minor,
1244 """Make the local state secondary.
1246 This will, of course, fail if the device is in use.
1249 if self.minor is None and not self.Attach():
1250 _ThrowError("drbd%d: can't Attach() in Close()", self._aminor)
1251 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1253 _ThrowError("drbd%d: can't switch drbd device to secondary: %s",
1254 self.minor, result.output)
1256 def DisconnectNet(self):
1257 """Removes network configuration.
1259 This method shutdowns the network side of the device.
1261 The method will wait up to a hardcoded timeout for the device to
1262 go into standalone after the 'disconnect' command before
1263 re-configuring it, as sometimes it takes a while for the
1264 disconnect to actually propagate and thus we might issue a 'net'
1265 command while the device is still connected. If the device will
1266 still be attached to the network and we time out, we raise an
1270 if self.minor is None:
1271 _ThrowError("drbd%d: disk not attached in re-attach net", self._aminor)
1273 if None in (self._lhost, self._lport, self._rhost, self._rport):
1274 _ThrowError("drbd%d: DRBD disk missing network info in"
1275 " DisconnectNet()", self.minor)
1277 ever_disconnected = self._ShutdownNet(self.minor)
1278 timeout_limit = time.time() + self._NET_RECONFIG_TIMEOUT
1279 sleep_time = 0.100 # we start the retry time at 100 miliseconds
1280 while time.time() < timeout_limit:
1281 status = self.GetProcStatus()
1282 if status.is_standalone:
1284 # retry the disconnect, it seems possible that due to a
1285 # well-time disconnect on the peer, my disconnect command might
1286 # be ingored and forgotten
1287 ever_disconnected = self._ShutdownNet(self.minor) or ever_disconnected
1288 time.sleep(sleep_time)
1289 sleep_time = min(2, sleep_time * 1.5)
1291 if not status.is_standalone:
1292 if ever_disconnected:
1293 msg = ("drbd%d: device did not react to the"
1294 " 'disconnect' command in a timely manner")
1296 msg = "drbd%d: can't shutdown network, even after multiple retries"
1297 _ThrowError(msg, self.minor)
1299 reconfig_time = time.time() - timeout_limit + self._NET_RECONFIG_TIMEOUT
1300 if reconfig_time > 15: # hardcoded alert limit
1301 logging.info("drbd%d: DisconnectNet: detach took %.3f seconds",
1302 self.minor, reconfig_time)
1304 def AttachNet(self, multimaster):
1305 """Reconnects the network.
1307 This method connects the network side of the device with a
1308 specified multi-master flag. The device needs to be 'Standalone'
1309 but have valid network configuration data.
1312 - multimaster: init the network in dual-primary mode
1315 if self.minor is None:
1316 _ThrowError("drbd%d: device not attached in AttachNet", self._aminor)
1318 if None in (self._lhost, self._lport, self._rhost, self._rport):
1319 _ThrowError("drbd%d: missing network info in AttachNet()", self.minor)
1321 status = self.GetProcStatus()
1323 if not status.is_standalone:
1324 _ThrowError("drbd%d: device is not standalone in AttachNet", self.minor)
1326 return self._AssembleNet(self.minor,
1327 (self._lhost, self._lport,
1328 self._rhost, self._rport),
1329 "C", dual_pri=multimaster)
1332 """Check if our minor is configured.
1334 This doesn't do any device configurations - it only checks if the
1335 minor is in a state different from Unconfigured.
1337 Note that this function will not change the state of the system in
1338 any way (except in case of side-effects caused by reading from
1342 used_devs = self.GetUsedDevs()
1343 if self._aminor in used_devs:
1344 minor = self._aminor
1348 self._SetFromMinor(minor)
1349 return minor is not None
1352 """Assemble the drbd.
1355 - if we have a configured device, we try to ensure that it matches
1357 - if not, we create it from zero
1360 result = super(DRBD8, self).Assemble()
1365 if self.minor is None:
1366 # local device completely unconfigured
1367 return self._FastAssemble()
1369 # we have to recheck the local and network status and try to fix
1371 return self._SlowAssemble()
1373 def _SlowAssemble(self):
1374 """Assembles the DRBD device from a (partially) configured device.
1376 In case of partially attached (local device matches but no network
1377 setup), we perform the network attach. If successful, we re-test
1378 the attach if can return success.
1381 for minor in (self._aminor,):
1382 info = self._GetDevInfo(self._GetShowData(minor))
1383 match_l = self._MatchesLocal(info)
1384 match_r = self._MatchesNet(info)
1385 if match_l and match_r:
1387 if match_l and not match_r and "local_addr" not in info:
1388 res_r = self._AssembleNet(minor,
1389 (self._lhost, self._lport,
1390 self._rhost, self._rport),
1391 constants.DRBD_NET_PROTOCOL,
1392 hmac=constants.DRBD_HMAC_ALG,
1396 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1398 # the weakest case: we find something that is only net attached
1399 # even though we were passed some children at init time
1400 if match_r and "local_dev" not in info:
1403 # this case must be considered only if we actually have local
1404 # storage, i.e. not in diskless mode, because all diskless
1405 # devices are equal from the point of view of local
1407 if (match_l and "local_dev" in info and
1408 not match_r and "local_addr" in info):
1409 # strange case - the device network part points to somewhere
1410 # else, even though its local storage is ours; as we own the
1411 # drbd space, we try to disconnect from the remote peer and
1412 # reconnect to our correct one
1413 if not self._ShutdownNet(minor):
1414 raise errors.BlockDeviceError("Device has correct local storage,"
1415 " wrong remote peer and is unable to"
1416 " disconnect in order to attach to"
1417 " the correct peer")
1418 # note: _AssembleNet also handles the case when we don't want
1419 # local storage (i.e. one or more of the _[lr](host|port) is
1421 if (self._AssembleNet(minor, (self._lhost, self._lport,
1422 self._rhost, self._rport),
1423 constants.DRBD_NET_PROTOCOL,
1424 hmac=constants.DRBD_HMAC_ALG,
1425 secret=self._secret) and
1426 self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))):
1432 self._SetFromMinor(minor)
1433 return minor is not None
1435 def _FastAssemble(self):
1436 """Assemble the drbd device from zero.
1438 This is run when in Assemble we detect our minor is unused.
1441 # TODO: maybe completely tear-down the minor (drbdsetup ... down)
1442 # before attaching our own?
1443 minor = self._aminor
1444 need_localdev_teardown = False
1445 if self._children and self._children[0] and self._children[1]:
1446 result = self._AssembleLocal(minor, self._children[0].dev_path,
1447 self._children[1].dev_path)
1450 need_localdev_teardown = True
1451 if self._lhost and self._lport and self._rhost and self._rport:
1452 result = self._AssembleNet(minor,
1453 (self._lhost, self._lport,
1454 self._rhost, self._rport),
1455 constants.DRBD_NET_PROTOCOL,
1456 hmac=constants.DRBD_HMAC_ALG,
1457 secret=self._secret)
1459 if need_localdev_teardown:
1460 # we will ignore failures from this
1461 logging.error("net setup failed, tearing down local device")
1462 self._ShutdownAll(minor)
1464 self._SetFromMinor(minor)
1468 def _ShutdownLocal(cls, minor):
1469 """Detach from the local device.
1471 I/Os will continue to be served from the remote device. If we
1472 don't have a remote device, this operation will fail.
1475 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1477 logging.error("Can't detach local device: %s", result.output)
1478 return not result.failed
1481 def _ShutdownNet(cls, minor):
1482 """Disconnect from the remote peer.
1484 This fails if we don't have a local device.
1487 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1489 logging.error("Can't shutdown network: %s", result.output)
1490 return not result.failed
1493 def _ShutdownAll(cls, minor):
1494 """Deactivate the device.
1496 This will, of course, fail if the device is in use.
1499 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1501 logging.error("Can't shutdown drbd device: %s", result.output)
1502 return not result.failed
1505 """Shutdown the DRBD device.
1508 if self.minor is None and not self.Attach():
1509 logging.info("DRBD device not attached to a device during Shutdown")
1511 if not self._ShutdownAll(self.minor):
1514 self.dev_path = None
1518 """Stub remove for DRBD devices.
1521 return self.Shutdown()
1524 def Create(cls, unique_id, children, size):
1525 """Create a new DRBD8 device.
1527 Since DRBD devices are not created per se, just assembled, this
1528 function only initializes the metadata.
1531 if len(children) != 2:
1532 raise errors.ProgrammerError("Invalid setup for the drbd device")
1533 # check that the minor is unused
1534 aminor = unique_id[4]
1535 proc_info = cls._MassageProcData(cls._GetProcData())
1536 if aminor in proc_info:
1537 status = DRBD8Status(proc_info[aminor])
1538 in_use = status.is_in_use
1542 _ThrowError("DRBD minor %d already in use at Create() time", aminor)
1545 if not meta.Attach():
1546 raise errors.BlockDeviceError("Can't attach to meta device")
1547 if not cls._CheckMetaSize(meta.dev_path):
1548 raise errors.BlockDeviceError("Invalid meta device size")
1549 cls._InitMeta(aminor, meta.dev_path)
1550 return cls(unique_id, children)
1552 def Grow(self, amount):
1553 """Resize the DRBD device and its backing storage.
1556 if self.minor is None:
1557 _ThrowError("drbd%d: Grow called while not attached", self._aminor)
1558 if len(self._children) != 2 or None in self._children:
1559 _ThrowError("drbd%d: cannot grow diskless device", self.minor)
1560 self._children[0].Grow(amount)
1561 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize"])
1563 _ThrowError("drbd%d: resize failed: %s", self.minor, result.output)
1566 class FileStorage(BlockDev):
1569 This class represents the a file storage backend device.
1571 The unique_id for the file device is a (file_driver, file_path) tuple.
1574 def __init__(self, unique_id, children):
1575 """Initalizes a file device backend.
1579 raise errors.BlockDeviceError("Invalid setup for file device")
1580 super(FileStorage, self).__init__(unique_id, children)
1581 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1582 raise ValueError("Invalid configuration data %s" % str(unique_id))
1583 self.driver = unique_id[0]
1584 self.dev_path = unique_id[1]
1588 """Assemble the device.
1590 Checks whether the file device exists, raises BlockDeviceError otherwise.
1593 if not os.path.exists(self.dev_path):
1594 raise errors.BlockDeviceError("File device '%s' does not exist." %
1599 """Shutdown the device.
1601 This is a no-op for the file type, as we don't deacivate
1602 the file on shutdown.
1607 def Open(self, force=False):
1608 """Make the device ready for I/O.
1610 This is a no-op for the file type.
1616 """Notifies that the device will no longer be used for I/O.
1618 This is a no-op for the file type.
1624 """Remove the file backing the block device.
1627 @return: True if the removal was successful
1630 if not os.path.exists(self.dev_path):
1633 os.remove(self.dev_path)
1635 except OSError, err:
1636 logging.error("Can't remove file '%s': %s", self.dev_path, err)
1640 """Attach to an existing file.
1642 Check if this file already exists.
1645 @return: True if file exists
1648 self.attached = os.path.exists(self.dev_path)
1649 return self.attached
1652 def Create(cls, unique_id, children, size):
1653 """Create a new file.
1655 @param size: the size of file in MiB
1657 @rtype: L{bdev.FileStorage}
1658 @return: an instance of FileStorage
1661 # TODO: decide whether we should check for existing files and
1663 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1664 raise ValueError("Invalid configuration data %s" % str(unique_id))
1665 dev_path = unique_id[1]
1667 f = open(dev_path, 'w')
1668 f.truncate(size * 1024 * 1024)
1670 except IOError, err:
1671 _ThrowError("Error in file creation: %", str(err))
1673 return FileStorage(unique_id, children)
1677 constants.LD_LV: LogicalVolume,
1678 constants.LD_DRBD8: DRBD8,
1679 constants.LD_FILE: FileStorage,
1683 def FindDevice(dev_type, unique_id, children):
1684 """Search for an existing, assembled device.
1686 This will succeed only if the device exists and is assembled, but it
1687 does not do any actions in order to activate the device.
1690 if dev_type not in DEV_MAP:
1691 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1692 device = DEV_MAP[dev_type](unique_id, children)
1693 if not device.attached:
1698 def Assemble(dev_type, unique_id, children):
1699 """Try to attach or assemble an existing device.
1701 This will attach to assemble the device, as needed, to bring it
1702 fully up. It must be safe to run on already-assembled devices.
1705 if dev_type not in DEV_MAP:
1706 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1707 device = DEV_MAP[dev_type](unique_id, children)
1708 if not device.Assemble():
1709 raise errors.BlockDeviceError("Can't find a valid block device for"
1711 (dev_type, unique_id, children))
1715 def Create(dev_type, unique_id, children, size):
1719 if dev_type not in DEV_MAP:
1720 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1721 device = DEV_MAP[dev_type].Create(unique_id, children, size)