4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
31 from ganeti import utils
32 from ganeti import errors
33 from ganeti import constants
36 def _IgnoreError(fn, *args, **kwargs):
37 """Executes the given function, ignoring BlockDeviceErrors.
39 This is used in order to simplify the execution of cleanup or
43 @return: True when fn didn't raise an exception, False otherwise
49 except errors.BlockDeviceError, err:
50 logging.warning("Caught BlockDeviceError but ignoring: %s" % str(err))
54 def _ThrowError(msg, *args):
55 """Log an error to the node daemon and the raise an exception.
58 @param msg: the text of the exception
59 @raise errors.BlockDeviceError
65 raise errors.BlockDeviceError(msg)
68 class BlockDev(object):
69 """Block device abstract class.
71 A block device can be in the following states:
72 - not existing on the system, and by `Create()` it goes into:
73 - existing but not setup/not active, and by `Assemble()` goes into:
74 - active read-write and by `Open()` it goes into
75 - online (=used, or ready for use)
77 A device can also be online but read-only, however we are not using
78 the readonly state (LV has it, if needed in the future) and we are
79 usually looking at this like at a stack, so it's easier to
80 conceptualise the transition from not-existing to online and back
83 The many different states of the device are due to the fact that we
84 need to cover many device types:
85 - logical volumes are created, lvchange -a y $lv, and used
86 - drbd devices are attached to a local disk/remote peer and made primary
88 A block device is identified by three items:
89 - the /dev path of the device (dynamic)
90 - a unique ID of the device (static)
91 - it's major/minor pair (dynamic)
93 Not all devices implement both the first two as distinct items. LVM
94 logical volumes have their unique ID (the pair volume group, logical
95 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
96 the /dev path is again dynamic and the unique id is the pair (host1,
99 You can get to a device in two ways:
100 - creating the (real) device, which returns you
101 an attached instance (lvcreate)
102 - attaching of a python instance to an existing (real) device
104 The second point, the attachement to a device, is different
105 depending on whether the device is assembled or not. At init() time,
106 we search for a device with the same unique_id as us. If found,
107 good. It also means that the device is already assembled. If not,
108 after assembly we'll have our correct major/minor.
111 def __init__(self, unique_id, children):
112 self._children = children
114 self.unique_id = unique_id
117 self.attached = False
120 """Assemble the device from its components.
122 Implementations of this method by child classes must ensure that:
123 - after the device has been assembled, it knows its major/minor
124 numbers; this allows other devices (usually parents) to probe
125 correctly for their children
126 - calling this method on an existing, in-use device is safe
127 - if the device is already configured (and in an OK state),
128 this method is idempotent
134 """Find a device which matches our config and attach to it.
137 raise NotImplementedError
140 """Notifies that the device will no longer be used for I/O.
143 raise NotImplementedError
146 def Create(cls, unique_id, children, size):
147 """Create the device.
149 If the device cannot be created, it will return None
150 instead. Error messages go to the logging system.
152 Note that for some devices, the unique_id is used, and for other,
153 the children. The idea is that these two, taken together, are
154 enough for both creation and assembly (later).
157 raise NotImplementedError
160 """Remove this device.
162 This makes sense only for some of the device types: LV and file
163 storeage. Also note that if the device can't attach, the removal
167 raise NotImplementedError
169 def Rename(self, new_id):
170 """Rename this device.
172 This may or may not make sense for a given device type.
175 raise NotImplementedError
177 def Open(self, force=False):
178 """Make the device ready for use.
180 This makes the device ready for I/O. For now, just the DRBD
183 The force parameter signifies that if the device has any kind of
184 --force thing, it should be used, we know what we are doing.
187 raise NotImplementedError
190 """Shut down the device, freeing its children.
192 This undoes the `Assemble()` work, except for the child
193 assembling; as such, the children on the device are still
194 assembled after this call.
197 raise NotImplementedError
199 def SetSyncSpeed(self, speed):
200 """Adjust the sync speed of the mirror.
202 In case this is not a mirroring device, this is no-op.
207 for child in self._children:
208 result = result and child.SetSyncSpeed(speed)
211 def GetSyncStatus(self):
212 """Returns the sync status of the device.
214 If this device is a mirroring device, this function returns the
215 status of the mirror.
217 If sync_percent is None, it means the device is not syncing.
219 If estimated_time is None, it means we can't estimate
220 the time needed, otherwise it's the time left in seconds.
222 If is_degraded is True, it means the device is missing
223 redundancy. This is usually a sign that something went wrong in
224 the device setup, if sync_percent is None.
226 The ldisk parameter represents the degradation of the local
227 data. This is only valid for some devices, the rest will always
228 return False (not degraded).
231 @return: (sync_percent, estimated_time, is_degraded, ldisk)
234 return None, None, False, False
237 def CombinedSyncStatus(self):
238 """Calculate the mirror status recursively for our children.
240 The return value is the same as for `GetSyncStatus()` except the
241 minimum percent and maximum time are calculated across our
245 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
247 for child in self._children:
248 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
249 if min_percent is None:
250 min_percent = c_percent
251 elif c_percent is not None:
252 min_percent = min(min_percent, c_percent)
255 elif c_time is not None:
256 max_time = max(max_time, c_time)
257 is_degraded = is_degraded or c_degraded
258 ldisk = ldisk or c_ldisk
259 return min_percent, max_time, is_degraded, ldisk
262 def SetInfo(self, text):
263 """Update metadata with info text.
265 Only supported for some device types.
268 for child in self._children:
271 def Grow(self, amount):
272 """Grow the block device.
274 @param amount: the amount (in mebibytes) to grow with
277 raise NotImplementedError
280 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
281 (self.__class__, self.unique_id, self._children,
282 self.major, self.minor, self.dev_path))
285 class LogicalVolume(BlockDev):
286 """Logical Volume block device.
289 def __init__(self, unique_id, children):
290 """Attaches to a LV device.
292 The unique_id is a tuple (vg_name, lv_name)
295 super(LogicalVolume, self).__init__(unique_id, children)
296 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
297 raise ValueError("Invalid configuration data %s" % str(unique_id))
298 self._vg_name, self._lv_name = unique_id
299 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
300 self._degraded = True
301 self.major = self.minor = None
305 def Create(cls, unique_id, children, size):
306 """Create a new logical volume.
309 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
310 raise errors.ProgrammerError("Invalid configuration data %s" %
312 vg_name, lv_name = unique_id
313 pvs_info = cls.GetPVInfo(vg_name)
315 _ThrowError("Can't compute PV info for vg %s", vg_name)
319 pvlist = [ pv[1] for pv in pvs_info ]
320 free_size = sum([ pv[0] for pv in pvs_info ])
322 # The size constraint should have been checked from the master before
323 # calling the create function.
325 _ThrowError("Not enough free space: required %s,"
326 " available %s", size, free_size)
327 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
330 _ThrowError("LV create failed (%s): %s",
331 result.fail_reason, result.output)
332 return LogicalVolume(unique_id, children)
335 def GetPVInfo(vg_name):
336 """Get the free space info for PVs in a volume group.
338 @param vg_name: the volume group name
341 @return: list of tuples (free_space, name) with free_space in mebibytes
344 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
345 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
347 result = utils.RunCmd(command)
349 logging.error("Can't get the PV information: %s - %s",
350 result.fail_reason, result.output)
353 for line in result.stdout.splitlines():
354 fields = line.strip().split(':')
356 logging.error("Can't parse pvs output: line '%s'", line)
358 # skip over pvs from another vg or ones which are not allocatable
359 if fields[1] != vg_name or fields[3][0] != 'a':
361 data.append((float(fields[2]), fields[0]))
366 """Remove this logical volume.
369 if not self.minor and not self.Attach():
370 # the LV does not exist
372 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
373 (self._vg_name, self._lv_name)])
375 _ThrowError("Can't lvremove: %s - %s", result.fail_reason, result.output)
377 def Rename(self, new_id):
378 """Rename this logical volume.
381 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
382 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
383 new_vg, new_name = new_id
384 if new_vg != self._vg_name:
385 raise errors.ProgrammerError("Can't move a logical volume across"
386 " volume groups (from %s to to %s)" %
387 (self._vg_name, new_vg))
388 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
390 _ThrowError("Failed to rename the logical volume: %s", result.output)
391 self._lv_name = new_name
392 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
395 """Attach to an existing LV.
397 This method will try to see if an existing and active LV exists
398 which matches our name. If so, its major/minor will be
402 self.attached = False
403 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
404 "-olv_attr,lv_kernel_major,lv_kernel_minor",
407 logging.error("Can't find LV %s: %s, %s",
408 self.dev_path, result.fail_reason, result.output)
410 out = result.stdout.strip().rstrip(',')
413 logging.error("Can't parse LVS output, len(%s) != 3", str(out))
416 status, major, minor = out[:3]
418 logging.error("lvs lv_attr is not 6 characters (%s)", status)
424 except ValueError, err:
425 logging.error("lvs major/minor cannot be parsed: %s", str(err))
429 self._degraded = status[0] == 'v' # virtual volume, i.e. doesn't backing
435 """Assemble the device.
437 We alway run `lvchange -ay` on the LV to ensure it's active before
438 use, as there were cases when xenvg was not active after boot
439 (also possibly after disk issues).
442 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
444 _ThrowError("Can't activate lv %s: %s", self.dev_path, result.output)
447 """Shutdown the device.
449 This is a no-op for the LV device type, as we don't deactivate the
455 def GetSyncStatus(self):
456 """Returns the sync status of the device.
458 If this device is a mirroring device, this function returns the
459 status of the mirror.
461 For logical volumes, sync_percent and estimated_time are always
462 None (no recovery in progress, as we don't handle the mirrored LV
463 case). The is_degraded parameter is the inverse of the ldisk
466 For the ldisk parameter, we check if the logical volume has the
467 'virtual' type, which means it's not backed by existing storage
468 anymore (read from it return I/O error). This happens after a
469 physical disk failure and subsequent 'vgreduce --removemissing' on
472 The status was already read in Attach, so we just return it.
475 @return: (sync_percent, estimated_time, is_degraded, ldisk)
478 return None, None, self._degraded, self._degraded
480 def Open(self, force=False):
481 """Make the device ready for I/O.
483 This is a no-op for the LV device type.
489 """Notifies that the device will no longer be used for I/O.
491 This is a no-op for the LV device type.
496 def Snapshot(self, size):
497 """Create a snapshot copy of an lvm block device.
500 snap_name = self._lv_name + ".snap"
502 # remove existing snapshot if found
503 snap = LogicalVolume((self._vg_name, snap_name), None)
504 _IgnoreError(snap.Remove)
506 pvs_info = self.GetPVInfo(self._vg_name)
508 _ThrowError("Can't compute PV info for vg %s", self._vg_name)
511 free_size, pv_name = pvs_info[0]
513 _ThrowError("Not enough free space: required %s,"
514 " available %s", size, free_size)
516 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
517 "-n%s" % snap_name, self.dev_path])
519 _ThrowError("command: %s error: %s - %s",
520 result.cmd, result.fail_reason, result.output)
524 def SetInfo(self, text):
525 """Update metadata with info text.
528 BlockDev.SetInfo(self, text)
530 # Replace invalid characters
531 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
532 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
534 # Only up to 128 characters are allowed
537 result = utils.RunCmd(["lvchange", "--addtag", text,
540 _ThrowError("Command: %s error: %s - %s", result.cmd, result.fail_reason,
543 def Grow(self, amount):
544 """Grow the logical volume.
547 # we try multiple algorithms since the 'best' ones might not have
548 # space available in the right place, but later ones might (since
549 # they have less constraints); also note that only recent LVM
551 for alloc_policy in "contiguous", "cling", "normal":
552 result = utils.RunCmd(["lvextend", "--alloc", alloc_policy,
553 "-L", "+%dm" % amount, self.dev_path])
554 if not result.failed:
556 _ThrowError("Can't grow LV %s: %s", self.dev_path, result.output)
559 class DRBD8Status(object):
560 """A DRBD status representation class.
562 Note that this doesn't support unconfigured devices (cs:Unconfigured).
565 UNCONF_RE = re.compile(r"\s*[0-9]+:\s*cs:Unconfigured$")
566 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+(?:st|ro):([^/]+)/(\S+)"
567 "\s+ds:([^/]+)/(\S+)\s+.*$")
568 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
569 "\sfinish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
571 def __init__(self, procline):
572 u = self.UNCONF_RE.match(procline)
574 self.cstatus = "Unconfigured"
575 self.lrole = self.rrole = self.ldisk = self.rdisk = None
577 m = self.LINE_RE.match(procline)
579 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
580 self.cstatus = m.group(1)
581 self.lrole = m.group(2)
582 self.rrole = m.group(3)
583 self.ldisk = m.group(4)
584 self.rdisk = m.group(5)
586 # end reading of data from the LINE_RE or UNCONF_RE
588 self.is_standalone = self.cstatus == "StandAlone"
589 self.is_wfconn = self.cstatus == "WFConnection"
590 self.is_connected = self.cstatus == "Connected"
591 self.is_primary = self.lrole == "Primary"
592 self.is_secondary = self.lrole == "Secondary"
593 self.peer_primary = self.rrole == "Primary"
594 self.peer_secondary = self.rrole == "Secondary"
595 self.both_primary = self.is_primary and self.peer_primary
596 self.both_secondary = self.is_secondary and self.peer_secondary
598 self.is_diskless = self.ldisk == "Diskless"
599 self.is_disk_uptodate = self.ldisk == "UpToDate"
601 self.is_in_resync = self.cstatus in ("SyncSource", "SyncTarget")
602 self.is_in_use = self.cstatus != "Unconfigured"
604 m = self.SYNC_RE.match(procline)
606 self.sync_percent = float(m.group(1))
607 hours = int(m.group(2))
608 minutes = int(m.group(3))
609 seconds = int(m.group(4))
610 self.est_time = hours * 3600 + minutes * 60 + seconds
612 self.sync_percent = None
615 self.is_sync_target = self.peer_sync_source = self.cstatus == "SyncTarget"
616 self.peer_sync_target = self.is_sync_source = self.cstatus == "SyncSource"
617 self.is_resync = self.is_sync_target or self.is_sync_source
620 class BaseDRBD(BlockDev):
623 This class contains a few bits of common functionality between the
624 0.7 and 8.x versions of DRBD.
627 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
628 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
631 _ST_UNCONFIGURED = "Unconfigured"
632 _ST_WFCONNECTION = "WFConnection"
633 _ST_CONNECTED = "Connected"
635 _STATUS_FILE = "/proc/drbd"
638 def _GetProcData(filename=_STATUS_FILE):
639 """Return data from /proc/drbd.
643 stat = open(filename, "r")
645 data = stat.read().splitlines()
648 except EnvironmentError, err:
649 if err.errno == errno.ENOENT:
650 _ThrowError("The file %s cannot be opened, check if the module"
651 " is loaded (%s)", filename, str(err))
653 _ThrowError("Can't read the DRBD proc file %s: %s", filename, str(err))
655 _ThrowError("Can't read any data from %s", filename)
659 def _MassageProcData(data):
660 """Transform the output of _GetProdData into a nicer form.
662 @return: a dictionary of minor: joined lines from /proc/drbd
666 lmatch = re.compile("^ *([0-9]+):.*$")
668 old_minor = old_line = None
670 lresult = lmatch.match(line)
671 if lresult is not None:
672 if old_minor is not None:
673 results[old_minor] = old_line
674 old_minor = int(lresult.group(1))
677 if old_minor is not None:
678 old_line += " " + line.strip()
680 if old_minor is not None:
681 results[old_minor] = old_line
685 def _GetVersion(cls):
686 """Return the DRBD version.
688 This will return a dict with keys:
694 - proto2 (only on drbd > 8.2.X)
697 proc_data = cls._GetProcData()
698 first_line = proc_data[0].strip()
699 version = cls._VERSION_RE.match(first_line)
701 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
704 values = version.groups()
705 retval = {'k_major': int(values[0]),
706 'k_minor': int(values[1]),
707 'k_point': int(values[2]),
708 'api': int(values[3]),
709 'proto': int(values[4]),
711 if values[5] is not None:
712 retval['proto2'] = values[5]
718 """Return the path to a drbd device for a given minor.
721 return "/dev/drbd%d" % minor
724 def GetUsedDevs(cls):
725 """Compute the list of used DRBD devices.
728 data = cls._GetProcData()
731 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
733 match = valid_line.match(line)
736 minor = int(match.group(1))
737 state = match.group(2)
738 if state == cls._ST_UNCONFIGURED:
740 used_devs[minor] = state, line
744 def _SetFromMinor(self, minor):
745 """Set our parameters based on the given minor.
747 This sets our minor variable and our dev_path.
751 self.minor = self.dev_path = None
752 self.attached = False
755 self.dev_path = self._DevPath(minor)
759 def _CheckMetaSize(meta_device):
760 """Check if the given meta device looks like a valid one.
762 This currently only check the size, which must be around
766 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
768 _ThrowError("Failed to get device size: %s - %s",
769 result.fail_reason, result.output)
771 sectors = int(result.stdout)
773 _ThrowError("Invalid output from blockdev: '%s'", result.stdout)
774 bytes = sectors * 512
775 if bytes < 128 * 1024 * 1024: # less than 128MiB
776 _ThrowError("Meta device too small (%.2fMib)", (bytes / 1024 / 1024))
777 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
778 _ThrowError("Meta device too big (%.2fMiB)", (bytes / 1024 / 1024))
780 def Rename(self, new_id):
783 This is not supported for drbd devices.
786 raise errors.ProgrammerError("Can't rename a drbd device")
789 class DRBD8(BaseDRBD):
790 """DRBD v8.x block device.
792 This implements the local host part of the DRBD device, i.e. it
793 doesn't do anything to the supposed peer. If you need a fully
794 connected DRBD pair, you need to use this class on both hosts.
796 The unique_id for the drbd device is the (local_ip, local_port,
797 remote_ip, remote_port) tuple, and it must have two children: the
798 data device and the meta_device. The meta device is checked for
799 valid size and is zeroed on create.
806 _NET_RECONFIG_TIMEOUT = 60
808 def __init__(self, unique_id, children):
809 if children and children.count(None) > 0:
811 super(DRBD8, self).__init__(unique_id, children)
812 self.major = self._DRBD_MAJOR
813 version = self._GetVersion()
814 if version['k_major'] != 8 :
815 _ThrowError("Mismatch in DRBD kernel version and requested ganeti"
816 " usage: kernel is %s.%s, ganeti wants 8.x",
817 version['k_major'], version['k_minor'])
819 if len(children) not in (0, 2):
820 raise ValueError("Invalid configuration data %s" % str(children))
821 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
822 raise ValueError("Invalid configuration data %s" % str(unique_id))
823 (self._lhost, self._lport,
824 self._rhost, self._rport,
825 self._aminor, self._secret) = unique_id
826 if (self._lhost is not None and self._lhost == self._rhost and
827 self._lport == self._rport):
828 raise ValueError("Invalid configuration data, same local/remote %s" %
833 def _InitMeta(cls, minor, dev_path):
834 """Initialize a meta device.
836 This will not work if the given minor is in use.
839 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
840 "v08", dev_path, "0", "create-md"])
842 _ThrowError("Can't initialize meta device: %s", result.output)
845 def _FindUnusedMinor(cls):
846 """Find an unused DRBD device.
848 This is specific to 8.x as the minors are allocated dynamically,
849 so non-existing numbers up to a max minor count are actually free.
852 data = cls._GetProcData()
854 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
855 used_line = re.compile("^ *([0-9]+): cs:")
858 match = unused_line.match(line)
860 return int(match.group(1))
861 match = used_line.match(line)
863 minor = int(match.group(1))
864 highest = max(highest, minor)
865 if highest is None: # there are no minors in use at all
867 if highest >= cls._MAX_MINORS:
868 logging.error("Error: no free drbd minors!")
869 raise errors.BlockDeviceError("Can't find a free DRBD minor")
873 def _GetShowParser(cls):
874 """Return a parser for `drbd show` output.
876 This will either create or return an already-create parser for the
877 output of the command `drbd show`.
880 if cls._PARSE_SHOW is not None:
881 return cls._PARSE_SHOW
884 lbrace = pyp.Literal("{").suppress()
885 rbrace = pyp.Literal("}").suppress()
886 semi = pyp.Literal(";").suppress()
887 # this also converts the value to an int
888 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
890 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
891 defa = pyp.Literal("_is_default").suppress()
892 dbl_quote = pyp.Literal('"').suppress()
894 keyword = pyp.Word(pyp.alphanums + '-')
897 value = pyp.Word(pyp.alphanums + '_-/.:')
898 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
899 addr_type = (pyp.Optional(pyp.Literal("ipv4")).suppress() +
900 pyp.Optional(pyp.Literal("ipv6")).suppress())
901 addr_port = (addr_type + pyp.Word(pyp.nums + '.') +
902 pyp.Literal(':').suppress() + number)
903 # meta device, extended syntax
904 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
905 number + pyp.Word(']').suppress())
906 # device name, extended syntax
907 device_value = pyp.Literal("minor").suppress() + number
910 stmt = (~rbrace + keyword + ~lbrace +
911 pyp.Optional(addr_port ^ value ^ quoted ^ meta_value ^
913 pyp.Optional(defa) + semi +
914 pyp.Optional(pyp.restOfLine).suppress())
917 section_name = pyp.Word(pyp.alphas + '_')
918 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
920 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
923 cls._PARSE_SHOW = bnf
928 def _GetShowData(cls, minor):
929 """Return the `drbdsetup show` data for a minor.
932 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
934 logging.error("Can't display the drbd config: %s - %s",
935 result.fail_reason, result.output)
940 def _GetDevInfo(cls, out):
941 """Parse details about a given DRBD minor.
943 This return, if available, the local backing device (as a path)
944 and the local and remote (ip, port) information from a string
945 containing the output of the `drbdsetup show` command as returned
953 bnf = cls._GetShowParser()
957 results = bnf.parseString(out)
958 except pyp.ParseException, err:
959 _ThrowError("Can't parse drbdsetup show output: %s", str(err))
961 # and massage the results into our desired format
962 for section in results:
964 if sname == "_this_host":
965 for lst in section[1:]:
967 data["local_dev"] = lst[1]
968 elif lst[0] == "meta-disk":
969 data["meta_dev"] = lst[1]
970 data["meta_index"] = lst[2]
971 elif lst[0] == "address":
972 data["local_addr"] = tuple(lst[1:])
973 elif sname == "_remote_host":
974 for lst in section[1:]:
975 if lst[0] == "address":
976 data["remote_addr"] = tuple(lst[1:])
979 def _MatchesLocal(self, info):
980 """Test if our local config matches with an existing device.
982 The parameter should be as returned from `_GetDevInfo()`. This
983 method tests if our local backing device is the same as the one in
984 the info parameter, in effect testing if we look like the given
989 backend, meta = self._children
991 backend = meta = None
993 if backend is not None:
994 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
996 retval = ("local_dev" not in info)
999 retval = retval and ("meta_dev" in info and
1000 info["meta_dev"] == meta.dev_path)
1001 retval = retval and ("meta_index" in info and
1002 info["meta_index"] == 0)
1004 retval = retval and ("meta_dev" not in info and
1005 "meta_index" not in info)
1008 def _MatchesNet(self, info):
1009 """Test if our network config matches with an existing device.
1011 The parameter should be as returned from `_GetDevInfo()`. This
1012 method tests if our network configuration is the same as the one
1013 in the info parameter, in effect testing if we look like the given
1017 if (((self._lhost is None and not ("local_addr" in info)) and
1018 (self._rhost is None and not ("remote_addr" in info)))):
1021 if self._lhost is None:
1024 if not ("local_addr" in info and
1025 "remote_addr" in info):
1028 retval = (info["local_addr"] == (self._lhost, self._lport))
1029 retval = (retval and
1030 info["remote_addr"] == (self._rhost, self._rport))
1034 def _AssembleLocal(cls, minor, backend, meta):
1035 """Configure the local part of a DRBD device.
1038 args = ["drbdsetup", cls._DevPath(minor), "disk",
1039 backend, meta, "0", "-e", "detach", "--create-device"]
1040 result = utils.RunCmd(args)
1042 _ThrowError("drbd%d: can't attach local disk: %s", minor, result.output)
1045 def _AssembleNet(cls, minor, net_info, protocol,
1046 dual_pri=False, hmac=None, secret=None):
1047 """Configure the network part of the device.
1050 lhost, lport, rhost, rport = net_info
1051 if None in net_info:
1052 # we don't want network connection and actually want to make
1054 cls._ShutdownNet(minor)
1057 # Workaround for a race condition. When DRBD is doing its dance to
1058 # establish a connection with its peer, it also sends the
1059 # synchronization speed over the wire. In some cases setting the
1060 # sync speed only after setting up both sides can race with DRBD
1061 # connecting, hence we set it here before telling DRBD anything
1063 cls._SetMinorSyncSpeed(minor, constants.SYNC_SPEED)
1065 args = ["drbdsetup", cls._DevPath(minor), "net",
1066 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
1067 "-A", "discard-zero-changes",
1074 args.extend(["-a", hmac, "-x", secret])
1075 result = utils.RunCmd(args)
1077 _ThrowError("drbd%d: can't setup network: %s - %s",
1078 minor, result.fail_reason, result.output)
1080 timeout = time.time() + 10
1082 while time.time() < timeout:
1083 info = cls._GetDevInfo(cls._GetShowData(minor))
1084 if not "local_addr" in info or not "remote_addr" in info:
1087 if (info["local_addr"] != (lhost, lport) or
1088 info["remote_addr"] != (rhost, rport)):
1094 _ThrowError("drbd%d: timeout while configuring network", minor)
1096 def AddChildren(self, devices):
1097 """Add a disk to the DRBD device.
1100 if self.minor is None:
1101 _ThrowError("drbd%d: can't attach to dbrd8 during AddChildren",
1103 if len(devices) != 2:
1104 _ThrowError("drbd%d: need two devices for AddChildren", self.minor)
1105 info = self._GetDevInfo(self._GetShowData(self.minor))
1106 if "local_dev" in info:
1107 _ThrowError("drbd%d: already attached to a local disk", self.minor)
1108 backend, meta = devices
1109 if backend.dev_path is None or meta.dev_path is None:
1110 _ThrowError("drbd%d: children not ready during AddChildren", self.minor)
1113 self._CheckMetaSize(meta.dev_path)
1114 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1116 self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path)
1117 self._children = devices
1119 def RemoveChildren(self, devices):
1120 """Detach the drbd device from local storage.
1123 if self.minor is None:
1124 _ThrowError("drbd%d: can't attach to drbd8 during RemoveChildren",
1126 # early return if we don't actually have backing storage
1127 info = self._GetDevInfo(self._GetShowData(self.minor))
1128 if "local_dev" not in info:
1130 if len(self._children) != 2:
1131 _ThrowError("drbd%d: we don't have two children: %s", self.minor,
1133 if self._children.count(None) == 2: # we don't actually have children :)
1134 logging.warning("drbd%d: requested detach while detached", self.minor)
1136 if len(devices) != 2:
1137 _ThrowError("drbd%d: we need two children in RemoveChildren", self.minor)
1138 for child, dev in zip(self._children, devices):
1139 if dev != child.dev_path:
1140 _ThrowError("drbd%d: mismatch in local storage (%s != %s) in"
1141 " RemoveChildren", self.minor, dev, child.dev_path)
1143 self._ShutdownLocal(self.minor)
1147 def _SetMinorSyncSpeed(cls, minor, kbytes):
1148 """Set the speed of the DRBD syncer.
1150 This is the low-level implementation.
1153 @param minor: the drbd minor whose settings we change
1155 @param kbytes: the speed in kbytes/second
1157 @return: the success of the operation
1160 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "syncer",
1161 "-r", "%d" % kbytes, "--create-device"])
1163 logging.error("Can't change syncer rate: %s - %s",
1164 result.fail_reason, result.output)
1165 return not result.failed
1167 def SetSyncSpeed(self, kbytes):
1168 """Set the speed of the DRBD syncer.
1171 @param kbytes: the speed in kbytes/second
1173 @return: the success of the operation
1176 if self.minor is None:
1177 logging.info("Not attached during SetSyncSpeed")
1179 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1180 return self._SetMinorSyncSpeed(self.minor, kbytes) and children_result
1182 def GetProcStatus(self):
1183 """Return device data from /proc.
1186 if self.minor is None:
1187 _ThrowError("drbd%d: GetStats() called while not attached", self._aminor)
1188 proc_info = self._MassageProcData(self._GetProcData())
1189 if self.minor not in proc_info:
1190 _ThrowError("drbd%d: can't find myself in /proc", self.minor)
1191 return DRBD8Status(proc_info[self.minor])
1193 def GetSyncStatus(self):
1194 """Returns the sync status of the device.
1197 If sync_percent is None, it means all is ok
1198 If estimated_time is None, it means we can't esimate
1199 the time needed, otherwise it's the time left in seconds.
1202 We set the is_degraded parameter to True on two conditions:
1203 network not connected or local disk missing.
1205 We compute the ldisk parameter based on wheter we have a local
1209 @return: (sync_percent, estimated_time, is_degraded, ldisk)
1212 if self.minor is None and not self.Attach():
1213 _ThrowError("drbd%d: can't Attach() in GetSyncStatus", self._aminor)
1214 stats = self.GetProcStatus()
1215 ldisk = not stats.is_disk_uptodate
1216 is_degraded = not stats.is_connected
1217 return stats.sync_percent, stats.est_time, is_degraded or ldisk, ldisk
1219 def Open(self, force=False):
1220 """Make the local state primary.
1222 If the 'force' parameter is given, the '-o' option is passed to
1223 drbdsetup. Since this is a potentially dangerous operation, the
1224 force flag should be only given after creation, when it actually
1228 if self.minor is None and not self.Attach():
1229 logging.error("DRBD cannot attach to a device during open")
1231 cmd = ["drbdsetup", self.dev_path, "primary"]
1234 result = utils.RunCmd(cmd)
1236 _ThrowError("drbd%d: can't make drbd device primary: %s", self.minor,
1240 """Make the local state secondary.
1242 This will, of course, fail if the device is in use.
1245 if self.minor is None and not self.Attach():
1246 _ThrowError("drbd%d: can't Attach() in Close()", self._aminor)
1247 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1249 _ThrowError("drbd%d: can't switch drbd device to secondary: %s",
1250 self.minor, result.output)
1252 def DisconnectNet(self):
1253 """Removes network configuration.
1255 This method shutdowns the network side of the device.
1257 The method will wait up to a hardcoded timeout for the device to
1258 go into standalone after the 'disconnect' command before
1259 re-configuring it, as sometimes it takes a while for the
1260 disconnect to actually propagate and thus we might issue a 'net'
1261 command while the device is still connected. If the device will
1262 still be attached to the network and we time out, we raise an
1266 if self.minor is None:
1267 _ThrowError("drbd%d: disk not attached in re-attach net", self._aminor)
1269 if None in (self._lhost, self._lport, self._rhost, self._rport):
1270 _ThrowError("drbd%d: DRBD disk missing network info in"
1271 " DisconnectNet()", self.minor)
1273 ever_disconnected = _IgnoreError(self._ShutdownNet, self.minor)
1274 timeout_limit = time.time() + self._NET_RECONFIG_TIMEOUT
1275 sleep_time = 0.100 # we start the retry time at 100 miliseconds
1276 while time.time() < timeout_limit:
1277 status = self.GetProcStatus()
1278 if status.is_standalone:
1280 # retry the disconnect, it seems possible that due to a
1281 # well-time disconnect on the peer, my disconnect command might
1282 # be ingored and forgotten
1283 ever_disconnected = _IgnoreError(self._ShutdownNet, self.minor) or \
1285 time.sleep(sleep_time)
1286 sleep_time = min(2, sleep_time * 1.5)
1288 if not status.is_standalone:
1289 if ever_disconnected:
1290 msg = ("drbd%d: device did not react to the"
1291 " 'disconnect' command in a timely manner")
1293 msg = "drbd%d: can't shutdown network, even after multiple retries"
1294 _ThrowError(msg, self.minor)
1296 reconfig_time = time.time() - timeout_limit + self._NET_RECONFIG_TIMEOUT
1297 if reconfig_time > 15: # hardcoded alert limit
1298 logging.info("drbd%d: DisconnectNet: detach took %.3f seconds",
1299 self.minor, reconfig_time)
1301 def AttachNet(self, multimaster):
1302 """Reconnects the network.
1304 This method connects the network side of the device with a
1305 specified multi-master flag. The device needs to be 'Standalone'
1306 but have valid network configuration data.
1309 - multimaster: init the network in dual-primary mode
1312 if self.minor is None:
1313 _ThrowError("drbd%d: device not attached in AttachNet", self._aminor)
1315 if None in (self._lhost, self._lport, self._rhost, self._rport):
1316 _ThrowError("drbd%d: missing network info in AttachNet()", self.minor)
1318 status = self.GetProcStatus()
1320 if not status.is_standalone:
1321 _ThrowError("drbd%d: device is not standalone in AttachNet", self.minor)
1323 self._AssembleNet(self.minor,
1324 (self._lhost, self._lport, self._rhost, self._rport),
1325 constants.DRBD_NET_PROTOCOL, dual_pri=multimaster,
1326 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1329 """Check if our minor is configured.
1331 This doesn't do any device configurations - it only checks if the
1332 minor is in a state different from Unconfigured.
1334 Note that this function will not change the state of the system in
1335 any way (except in case of side-effects caused by reading from
1339 used_devs = self.GetUsedDevs()
1340 if self._aminor in used_devs:
1341 minor = self._aminor
1345 self._SetFromMinor(minor)
1346 return minor is not None
1349 """Assemble the drbd.
1352 - if we have a configured device, we try to ensure that it matches
1354 - if not, we create it from zero
1357 super(DRBD8, self).Assemble()
1360 if self.minor is None:
1361 # local device completely unconfigured
1362 self._FastAssemble()
1364 # we have to recheck the local and network status and try to fix
1366 self._SlowAssemble()
1368 def _SlowAssemble(self):
1369 """Assembles the DRBD device from a (partially) configured device.
1371 In case of partially attached (local device matches but no network
1372 setup), we perform the network attach. If successful, we re-test
1373 the attach if can return success.
1376 net_data = (self._lhost, self._lport, self._rhost, self._rport)
1377 for minor in (self._aminor,):
1378 info = self._GetDevInfo(self._GetShowData(minor))
1379 match_l = self._MatchesLocal(info)
1380 match_r = self._MatchesNet(info)
1382 if match_l and match_r:
1383 # everything matches
1386 if match_l and not match_r and "local_addr" not in info:
1387 # disk matches, but not attached to network, attach and recheck
1388 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1389 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1390 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1393 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1394 " show' disagrees", minor)
1396 if match_r and "local_dev" not in info:
1397 # no local disk, but network attached and it matches
1398 self._AssembleLocal(minor, self._children[0].dev_path,
1399 self._children[1].dev_path)
1400 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1403 _ThrowError("drbd%d: disk attach successful, but 'drbdsetup"
1404 " show' disagrees", minor)
1406 # this case must be considered only if we actually have local
1407 # storage, i.e. not in diskless mode, because all diskless
1408 # devices are equal from the point of view of local
1410 if (match_l and "local_dev" in info and
1411 not match_r and "local_addr" in info):
1412 # strange case - the device network part points to somewhere
1413 # else, even though its local storage is ours; as we own the
1414 # drbd space, we try to disconnect from the remote peer and
1415 # reconnect to our correct one
1417 self._ShutdownNet(minor)
1418 except errors.BlockDeviceError, err:
1419 _ThrowError("drbd%d: device has correct local storage, wrong"
1420 " remote peer and is unable to disconnect in order"
1421 " to attach to the correct peer: %s", minor, str(err))
1422 # note: _AssembleNet also handles the case when we don't want
1423 # local storage (i.e. one or more of the _[lr](host|port) is
1425 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1426 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1427 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1430 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1431 " show' disagrees", minor)
1436 self._SetFromMinor(minor)
1438 _ThrowError("drbd%d: cannot activate, unknown or unhandled reason",
1441 def _FastAssemble(self):
1442 """Assemble the drbd device from zero.
1444 This is run when in Assemble we detect our minor is unused.
1447 minor = self._aminor
1448 if self._children and self._children[0] and self._children[1]:
1449 self._AssembleLocal(minor, self._children[0].dev_path,
1450 self._children[1].dev_path)
1451 if self._lhost and self._lport and self._rhost and self._rport:
1452 self._AssembleNet(minor,
1453 (self._lhost, self._lport, self._rhost, self._rport),
1454 constants.DRBD_NET_PROTOCOL,
1455 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1456 self._SetFromMinor(minor)
1459 def _ShutdownLocal(cls, minor):
1460 """Detach from the local device.
1462 I/Os will continue to be served from the remote device. If we
1463 don't have a remote device, this operation will fail.
1466 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1468 _ThrowError("drbd%d: can't detach local disk: %s", minor, result.output)
1471 def _ShutdownNet(cls, minor):
1472 """Disconnect from the remote peer.
1474 This fails if we don't have a local device.
1477 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1479 _ThrowError("drbd%d: can't shutdown network: %s", minor, result.output)
1482 def _ShutdownAll(cls, minor):
1483 """Deactivate the device.
1485 This will, of course, fail if the device is in use.
1488 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1490 _ThrowError("drbd%d: can't shutdown drbd device: %s",
1491 minor, result.output)
1494 """Shutdown the DRBD device.
1497 if self.minor is None and not self.Attach():
1498 logging.info("drbd%d: not attached during Shutdown()", self._aminor)
1502 self.dev_path = None
1503 self._ShutdownAll(minor)
1506 """Stub remove for DRBD devices.
1512 def Create(cls, unique_id, children, size):
1513 """Create a new DRBD8 device.
1515 Since DRBD devices are not created per se, just assembled, this
1516 function only initializes the metadata.
1519 if len(children) != 2:
1520 raise errors.ProgrammerError("Invalid setup for the drbd device")
1521 # check that the minor is unused
1522 aminor = unique_id[4]
1523 proc_info = cls._MassageProcData(cls._GetProcData())
1524 if aminor in proc_info:
1525 status = DRBD8Status(proc_info[aminor])
1526 in_use = status.is_in_use
1530 _ThrowError("drbd%d: minor is already in use at Create() time", aminor)
1533 if not meta.Attach():
1534 _ThrowError("drbd%d: can't attach to meta device '%s'",
1536 cls._CheckMetaSize(meta.dev_path)
1537 cls._InitMeta(aminor, meta.dev_path)
1538 return cls(unique_id, children)
1540 def Grow(self, amount):
1541 """Resize the DRBD device and its backing storage.
1544 if self.minor is None:
1545 _ThrowError("drbd%d: Grow called while not attached", self._aminor)
1546 if len(self._children) != 2 or None in self._children:
1547 _ThrowError("drbd%d: cannot grow diskless device", self.minor)
1548 self._children[0].Grow(amount)
1549 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize"])
1551 _ThrowError("drbd%d: resize failed: %s", self.minor, result.output)
1554 class FileStorage(BlockDev):
1557 This class represents the a file storage backend device.
1559 The unique_id for the file device is a (file_driver, file_path) tuple.
1562 def __init__(self, unique_id, children):
1563 """Initalizes a file device backend.
1567 raise errors.BlockDeviceError("Invalid setup for file device")
1568 super(FileStorage, self).__init__(unique_id, children)
1569 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1570 raise ValueError("Invalid configuration data %s" % str(unique_id))
1571 self.driver = unique_id[0]
1572 self.dev_path = unique_id[1]
1576 """Assemble the device.
1578 Checks whether the file device exists, raises BlockDeviceError otherwise.
1581 if not os.path.exists(self.dev_path):
1582 _ThrowError("File device '%s' does not exist" % self.dev_path)
1585 """Shutdown the device.
1587 This is a no-op for the file type, as we don't deacivate
1588 the file on shutdown.
1593 def Open(self, force=False):
1594 """Make the device ready for I/O.
1596 This is a no-op for the file type.
1602 """Notifies that the device will no longer be used for I/O.
1604 This is a no-op for the file type.
1610 """Remove the file backing the block device.
1613 @return: True if the removal was successful
1617 os.remove(self.dev_path)
1618 except OSError, err:
1619 if err.errno != errno.ENOENT:
1620 _ThrowError("Can't remove file '%s': %s", self.dev_path, err)
1623 """Attach to an existing file.
1625 Check if this file already exists.
1628 @return: True if file exists
1631 self.attached = os.path.exists(self.dev_path)
1632 return self.attached
1635 def Create(cls, unique_id, children, size):
1636 """Create a new file.
1638 @param size: the size of file in MiB
1640 @rtype: L{bdev.FileStorage}
1641 @return: an instance of FileStorage
1644 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1645 raise ValueError("Invalid configuration data %s" % str(unique_id))
1646 dev_path = unique_id[1]
1647 if os.path.exists(dev_path):
1648 _ThrowError("File already existing: %s", dev_path)
1650 f = open(dev_path, 'w')
1651 f.truncate(size * 1024 * 1024)
1653 except IOError, err:
1654 _ThrowError("Error in file creation: %", str(err))
1656 return FileStorage(unique_id, children)
1660 constants.LD_LV: LogicalVolume,
1661 constants.LD_DRBD8: DRBD8,
1662 constants.LD_FILE: FileStorage,
1666 def FindDevice(dev_type, unique_id, children):
1667 """Search for an existing, assembled device.
1669 This will succeed only if the device exists and is assembled, but it
1670 does not do any actions in order to activate the device.
1673 if dev_type not in DEV_MAP:
1674 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1675 device = DEV_MAP[dev_type](unique_id, children)
1676 if not device.attached:
1681 def Assemble(dev_type, unique_id, children):
1682 """Try to attach or assemble an existing device.
1684 This will attach to assemble the device, as needed, to bring it
1685 fully up. It must be safe to run on already-assembled devices.
1688 if dev_type not in DEV_MAP:
1689 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1690 device = DEV_MAP[dev_type](unique_id, children)
1695 def Create(dev_type, unique_id, children, size):
1699 if dev_type not in DEV_MAP:
1700 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1701 device = DEV_MAP[dev_type].Create(unique_id, children, size)