4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
29 import pyparsing as pyp
33 from ganeti import utils
34 from ganeti import errors
35 from ganeti import constants
36 from ganeti import objects
37 from ganeti import compat
38 from ganeti import netutils
41 # Size of reads in _CanReadDevice
42 _DEVICE_READ_SIZE = 128 * 1024
45 def _IgnoreError(fn, *args, **kwargs):
46 """Executes the given function, ignoring BlockDeviceErrors.
48 This is used in order to simplify the execution of cleanup or
52 @return: True when fn didn't raise an exception, False otherwise
58 except errors.BlockDeviceError, err:
59 logging.warning("Caught BlockDeviceError but ignoring: %s", str(err))
63 def _ThrowError(msg, *args):
64 """Log an error to the node daemon and the raise an exception.
67 @param msg: the text of the exception
68 @raise errors.BlockDeviceError
74 raise errors.BlockDeviceError(msg)
77 def _CanReadDevice(path):
78 """Check if we can read from the given device.
80 This tries to read the first 128k of the device.
84 utils.ReadFile(path, size=_DEVICE_READ_SIZE)
86 except EnvironmentError:
87 logging.warning("Can't read from device %s", path, exc_info=True)
91 class BlockDev(object):
92 """Block device abstract class.
94 A block device can be in the following states:
95 - not existing on the system, and by `Create()` it goes into:
96 - existing but not setup/not active, and by `Assemble()` goes into:
97 - active read-write and by `Open()` it goes into
98 - online (=used, or ready for use)
100 A device can also be online but read-only, however we are not using
101 the readonly state (LV has it, if needed in the future) and we are
102 usually looking at this like at a stack, so it's easier to
103 conceptualise the transition from not-existing to online and back
106 The many different states of the device are due to the fact that we
107 need to cover many device types:
108 - logical volumes are created, lvchange -a y $lv, and used
109 - drbd devices are attached to a local disk/remote peer and made primary
111 A block device is identified by three items:
112 - the /dev path of the device (dynamic)
113 - a unique ID of the device (static)
114 - it's major/minor pair (dynamic)
116 Not all devices implement both the first two as distinct items. LVM
117 logical volumes have their unique ID (the pair volume group, logical
118 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
119 the /dev path is again dynamic and the unique id is the pair (host1,
120 dev1), (host2, dev2).
122 You can get to a device in two ways:
123 - creating the (real) device, which returns you
124 an attached instance (lvcreate)
125 - attaching of a python instance to an existing (real) device
127 The second point, the attachement to a device, is different
128 depending on whether the device is assembled or not. At init() time,
129 we search for a device with the same unique_id as us. If found,
130 good. It also means that the device is already assembled. If not,
131 after assembly we'll have our correct major/minor.
134 def __init__(self, unique_id, children, size, params):
135 self._children = children
137 self.unique_id = unique_id
140 self.attached = False
145 """Assemble the device from its components.
147 Implementations of this method by child classes must ensure that:
148 - after the device has been assembled, it knows its major/minor
149 numbers; this allows other devices (usually parents) to probe
150 correctly for their children
151 - calling this method on an existing, in-use device is safe
152 - if the device is already configured (and in an OK state),
153 this method is idempotent
159 """Find a device which matches our config and attach to it.
162 raise NotImplementedError
165 """Notifies that the device will no longer be used for I/O.
168 raise NotImplementedError
171 def Create(cls, unique_id, children, size, params):
172 """Create the device.
174 If the device cannot be created, it will return None
175 instead. Error messages go to the logging system.
177 Note that for some devices, the unique_id is used, and for other,
178 the children. The idea is that these two, taken together, are
179 enough for both creation and assembly (later).
182 raise NotImplementedError
185 """Remove this device.
187 This makes sense only for some of the device types: LV and file
188 storage. Also note that if the device can't attach, the removal
192 raise NotImplementedError
194 def Rename(self, new_id):
195 """Rename this device.
197 This may or may not make sense for a given device type.
200 raise NotImplementedError
202 def Open(self, force=False):
203 """Make the device ready for use.
205 This makes the device ready for I/O. For now, just the DRBD
208 The force parameter signifies that if the device has any kind of
209 --force thing, it should be used, we know what we are doing.
212 raise NotImplementedError
215 """Shut down the device, freeing its children.
217 This undoes the `Assemble()` work, except for the child
218 assembling; as such, the children on the device are still
219 assembled after this call.
222 raise NotImplementedError
224 def SetSyncParams(self, params):
225 """Adjust the synchronization parameters of the mirror.
227 In case this is not a mirroring device, this is no-op.
229 @param params: dictionary of LD level disk parameters related to the
235 for child in self._children:
236 result = result and child.SetSyncParams(params)
239 def PauseResumeSync(self, pause):
240 """Pause/Resume the sync of the mirror.
242 In case this is not a mirroring device, this is no-op.
244 @param pause: Whether to pause or resume
249 for child in self._children:
250 result = result and child.PauseResumeSync(pause)
253 def GetSyncStatus(self):
254 """Returns the sync status of the device.
256 If this device is a mirroring device, this function returns the
257 status of the mirror.
259 If sync_percent is None, it means the device is not syncing.
261 If estimated_time is None, it means we can't estimate
262 the time needed, otherwise it's the time left in seconds.
264 If is_degraded is True, it means the device is missing
265 redundancy. This is usually a sign that something went wrong in
266 the device setup, if sync_percent is None.
268 The ldisk parameter represents the degradation of the local
269 data. This is only valid for some devices, the rest will always
270 return False (not degraded).
272 @rtype: objects.BlockDevStatus
275 return objects.BlockDevStatus(dev_path=self.dev_path,
281 ldisk_status=constants.LDS_OKAY)
283 def CombinedSyncStatus(self):
284 """Calculate the mirror status recursively for our children.
286 The return value is the same as for `GetSyncStatus()` except the
287 minimum percent and maximum time are calculated across our
290 @rtype: objects.BlockDevStatus
293 status = self.GetSyncStatus()
295 min_percent = status.sync_percent
296 max_time = status.estimated_time
297 is_degraded = status.is_degraded
298 ldisk_status = status.ldisk_status
301 for child in self._children:
302 child_status = child.GetSyncStatus()
304 if min_percent is None:
305 min_percent = child_status.sync_percent
306 elif child_status.sync_percent is not None:
307 min_percent = min(min_percent, child_status.sync_percent)
310 max_time = child_status.estimated_time
311 elif child_status.estimated_time is not None:
312 max_time = max(max_time, child_status.estimated_time)
314 is_degraded = is_degraded or child_status.is_degraded
316 if ldisk_status is None:
317 ldisk_status = child_status.ldisk_status
318 elif child_status.ldisk_status is not None:
319 ldisk_status = max(ldisk_status, child_status.ldisk_status)
321 return objects.BlockDevStatus(dev_path=self.dev_path,
324 sync_percent=min_percent,
325 estimated_time=max_time,
326 is_degraded=is_degraded,
327 ldisk_status=ldisk_status)
329 def SetInfo(self, text):
330 """Update metadata with info text.
332 Only supported for some device types.
335 for child in self._children:
338 def Grow(self, amount, dryrun):
339 """Grow the block device.
341 @type amount: integer
342 @param amount: the amount (in mebibytes) to grow with
343 @type dryrun: boolean
344 @param dryrun: whether to execute the operation in simulation mode
345 only, without actually increasing the size
348 raise NotImplementedError
350 def GetActualSize(self):
351 """Return the actual disk size.
353 @note: the device needs to be active when this is called
356 assert self.attached, "BlockDevice not attached in GetActualSize()"
357 result = utils.RunCmd(["blockdev", "--getsize64", self.dev_path])
359 _ThrowError("blockdev failed (%s): %s",
360 result.fail_reason, result.output)
362 sz = int(result.output.strip())
363 except (ValueError, TypeError), err:
364 _ThrowError("Failed to parse blockdev output: %s", str(err))
368 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
369 (self.__class__, self.unique_id, self._children,
370 self.major, self.minor, self.dev_path))
373 class LogicalVolume(BlockDev):
374 """Logical Volume block device.
377 _VALID_NAME_RE = re.compile("^[a-zA-Z0-9+_.-]*$")
378 _INVALID_NAMES = frozenset([".", "..", "snapshot", "pvmove"])
379 _INVALID_SUBSTRINGS = frozenset(["_mlog", "_mimage"])
381 def __init__(self, unique_id, children, size, params):
382 """Attaches to a LV device.
384 The unique_id is a tuple (vg_name, lv_name)
387 super(LogicalVolume, self).__init__(unique_id, children, size, params)
388 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
389 raise ValueError("Invalid configuration data %s" % str(unique_id))
390 self._vg_name, self._lv_name = unique_id
391 self._ValidateName(self._vg_name)
392 self._ValidateName(self._lv_name)
393 self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name)
394 self._degraded = True
395 self.major = self.minor = self.pe_size = self.stripe_count = None
399 def Create(cls, unique_id, children, size, params):
400 """Create a new logical volume.
403 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
404 raise errors.ProgrammerError("Invalid configuration data %s" %
406 vg_name, lv_name = unique_id
407 cls._ValidateName(vg_name)
408 cls._ValidateName(lv_name)
409 pvs_info = cls.GetPVInfo([vg_name])
411 _ThrowError("Can't compute PV info for vg %s", vg_name)
415 pvlist = [pv[1] for pv in pvs_info]
416 if compat.any(":" in v for v in pvlist):
417 _ThrowError("Some of your PVs have the invalid character ':' in their"
418 " name, this is not supported - please filter them out"
419 " in lvm.conf using either 'filter' or 'preferred_names'")
420 free_size = sum([pv[0] for pv in pvs_info])
421 current_pvs = len(pvlist)
422 desired_stripes = params[constants.LDP_STRIPES]
423 stripes = min(current_pvs, desired_stripes)
424 if stripes < desired_stripes:
425 logging.warning("Could not use %d stripes for VG %s, as only %d PVs are"
426 " available.", desired_stripes, vg_name, current_pvs)
428 # The size constraint should have been checked from the master before
429 # calling the create function.
431 _ThrowError("Not enough free space: required %s,"
432 " available %s", size, free_size)
433 cmd = ["lvcreate", "-L%dm" % size, "-n%s" % lv_name]
434 # If the free space is not well distributed, we won't be able to
435 # create an optimally-striped volume; in that case, we want to try
436 # with N, N-1, ..., 2, and finally 1 (non-stripped) number of
438 for stripes_arg in range(stripes, 0, -1):
439 result = utils.RunCmd(cmd + ["-i%d" % stripes_arg] + [vg_name] + pvlist)
440 if not result.failed:
443 _ThrowError("LV create failed (%s): %s",
444 result.fail_reason, result.output)
445 return LogicalVolume(unique_id, children, size, params)
448 def _GetVolumeInfo(lvm_cmd, fields):
449 """Returns LVM Volumen infos using lvm_cmd
451 @param lvm_cmd: Should be one of "pvs", "vgs" or "lvs"
452 @param fields: Fields to return
453 @return: A list of dicts each with the parsed fields
457 raise errors.ProgrammerError("No fields specified")
460 cmd = [lvm_cmd, "--noheadings", "--nosuffix", "--units=m", "--unbuffered",
461 "--separator=%s" % sep, "-o%s" % ",".join(fields)]
463 result = utils.RunCmd(cmd)
465 raise errors.CommandError("Can't get the volume information: %s - %s" %
466 (result.fail_reason, result.output))
469 for line in result.stdout.splitlines():
470 splitted_fields = line.strip().split(sep)
472 if len(fields) != len(splitted_fields):
473 raise errors.CommandError("Can't parse %s output: line '%s'" %
476 data.append(splitted_fields)
481 def GetPVInfo(cls, vg_names, filter_allocatable=True):
482 """Get the free space info for PVs in a volume group.
484 @param vg_names: list of volume group names, if empty all will be returned
485 @param filter_allocatable: whether to skip over unallocatable PVs
488 @return: list of tuples (free_space, name) with free_space in mebibytes
492 info = cls._GetVolumeInfo("pvs", ["pv_name", "vg_name", "pv_free",
494 except errors.GenericError, err:
495 logging.error("Can't get PV information: %s", err)
499 for pv_name, vg_name, pv_free, pv_attr in info:
500 # (possibly) skip over pvs which are not allocatable
501 if filter_allocatable and pv_attr[0] != "a":
503 # (possibly) skip over pvs which are not in the right volume group(s)
504 if vg_names and vg_name not in vg_names:
506 data.append((float(pv_free), pv_name, vg_name))
511 def GetVGInfo(cls, vg_names, filter_readonly=True):
512 """Get the free space info for specific VGs.
514 @param vg_names: list of volume group names, if empty all will be returned
515 @param filter_readonly: whether to skip over readonly VGs
518 @return: list of tuples (free_space, total_size, name) with free_space in
523 info = cls._GetVolumeInfo("vgs", ["vg_name", "vg_free", "vg_attr",
525 except errors.GenericError, err:
526 logging.error("Can't get VG information: %s", err)
530 for vg_name, vg_free, vg_attr, vg_size in info:
531 # (possibly) skip over vgs which are not writable
532 if filter_readonly and vg_attr[0] == "r":
534 # (possibly) skip over vgs which are not in the right volume group(s)
535 if vg_names and vg_name not in vg_names:
537 data.append((float(vg_free), float(vg_size), vg_name))
542 def _ValidateName(cls, name):
543 """Validates that a given name is valid as VG or LV name.
545 The list of valid characters and restricted names is taken out of
546 the lvm(8) manpage, with the simplification that we enforce both
547 VG and LV restrictions on the names.
550 if (not cls._VALID_NAME_RE.match(name) or
551 name in cls._INVALID_NAMES or
552 compat.any(substring in name for substring in cls._INVALID_SUBSTRINGS)):
553 _ThrowError("Invalid LVM name '%s'", name)
556 """Remove this logical volume.
559 if not self.minor and not self.Attach():
560 # the LV does not exist
562 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
563 (self._vg_name, self._lv_name)])
565 _ThrowError("Can't lvremove: %s - %s", result.fail_reason, result.output)
567 def Rename(self, new_id):
568 """Rename this logical volume.
571 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
572 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
573 new_vg, new_name = new_id
574 if new_vg != self._vg_name:
575 raise errors.ProgrammerError("Can't move a logical volume across"
576 " volume groups (from %s to to %s)" %
577 (self._vg_name, new_vg))
578 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
580 _ThrowError("Failed to rename the logical volume: %s", result.output)
581 self._lv_name = new_name
582 self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name)
585 """Attach to an existing LV.
587 This method will try to see if an existing and active LV exists
588 which matches our name. If so, its major/minor will be
592 self.attached = False
593 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
594 "--units=m", "--nosuffix",
595 "-olv_attr,lv_kernel_major,lv_kernel_minor,"
596 "vg_extent_size,stripes", self.dev_path])
598 logging.error("Can't find LV %s: %s, %s",
599 self.dev_path, result.fail_reason, result.output)
601 # the output can (and will) have multiple lines for multi-segment
602 # LVs, as the 'stripes' parameter is a segment one, so we take
603 # only the last entry, which is the one we're interested in; note
604 # that with LVM2 anyway the 'stripes' value must be constant
605 # across segments, so this is a no-op actually
606 out = result.stdout.splitlines()
607 if not out: # totally empty result? splitlines() returns at least
608 # one line for any non-empty string
609 logging.error("Can't parse LVS output, no lines? Got '%s'", str(out))
611 out = out[-1].strip().rstrip(",")
614 logging.error("Can't parse LVS output, len(%s) != 5", str(out))
617 status, major, minor, pe_size, stripes = out
619 logging.error("lvs lv_attr is not 6 characters (%s)", status)
625 except (TypeError, ValueError), err:
626 logging.error("lvs major/minor cannot be parsed: %s", str(err))
629 pe_size = int(float(pe_size))
630 except (TypeError, ValueError), err:
631 logging.error("Can't parse vg extent size: %s", err)
635 stripes = int(stripes)
636 except (TypeError, ValueError), err:
637 logging.error("Can't parse the number of stripes: %s", err)
642 self.pe_size = pe_size
643 self.stripe_count = stripes
644 self._degraded = status[0] == "v" # virtual volume, i.e. doesn't backing
650 """Assemble the device.
652 We always run `lvchange -ay` on the LV to ensure it's active before
653 use, as there were cases when xenvg was not active after boot
654 (also possibly after disk issues).
657 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
659 _ThrowError("Can't activate lv %s: %s", self.dev_path, result.output)
662 """Shutdown the device.
664 This is a no-op for the LV device type, as we don't deactivate the
670 def GetSyncStatus(self):
671 """Returns the sync status of the device.
673 If this device is a mirroring device, this function returns the
674 status of the mirror.
676 For logical volumes, sync_percent and estimated_time are always
677 None (no recovery in progress, as we don't handle the mirrored LV
678 case). The is_degraded parameter is the inverse of the ldisk
681 For the ldisk parameter, we check if the logical volume has the
682 'virtual' type, which means it's not backed by existing storage
683 anymore (read from it return I/O error). This happens after a
684 physical disk failure and subsequent 'vgreduce --removemissing' on
687 The status was already read in Attach, so we just return it.
689 @rtype: objects.BlockDevStatus
693 ldisk_status = constants.LDS_FAULTY
695 ldisk_status = constants.LDS_OKAY
697 return objects.BlockDevStatus(dev_path=self.dev_path,
702 is_degraded=self._degraded,
703 ldisk_status=ldisk_status)
705 def Open(self, force=False):
706 """Make the device ready for I/O.
708 This is a no-op for the LV device type.
714 """Notifies that the device will no longer be used for I/O.
716 This is a no-op for the LV device type.
721 def Snapshot(self, size):
722 """Create a snapshot copy of an lvm block device.
724 @returns: tuple (vg, lv)
727 snap_name = self._lv_name + ".snap"
729 # remove existing snapshot if found
730 snap = LogicalVolume((self._vg_name, snap_name), None, size, self.params)
731 _IgnoreError(snap.Remove)
733 vg_info = self.GetVGInfo([self._vg_name])
735 _ThrowError("Can't compute VG info for vg %s", self._vg_name)
736 free_size, _, _ = vg_info[0]
738 _ThrowError("Not enough free space: required %s,"
739 " available %s", size, free_size)
741 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
742 "-n%s" % snap_name, self.dev_path])
744 _ThrowError("command: %s error: %s - %s",
745 result.cmd, result.fail_reason, result.output)
747 return (self._vg_name, snap_name)
749 def SetInfo(self, text):
750 """Update metadata with info text.
753 BlockDev.SetInfo(self, text)
755 # Replace invalid characters
756 text = re.sub("^[^A-Za-z0-9_+.]", "_", text)
757 text = re.sub("[^-A-Za-z0-9_+.]", "_", text)
759 # Only up to 128 characters are allowed
762 result = utils.RunCmd(["lvchange", "--addtag", text,
765 _ThrowError("Command: %s error: %s - %s", result.cmd, result.fail_reason,
768 def Grow(self, amount, dryrun):
769 """Grow the logical volume.
772 if self.pe_size is None or self.stripe_count is None:
773 if not self.Attach():
774 _ThrowError("Can't attach to LV during Grow()")
775 full_stripe_size = self.pe_size * self.stripe_count
776 rest = amount % full_stripe_size
778 amount += full_stripe_size - rest
779 cmd = ["lvextend", "-L", "+%dm" % amount]
782 # we try multiple algorithms since the 'best' ones might not have
783 # space available in the right place, but later ones might (since
784 # they have less constraints); also note that only recent LVM
786 for alloc_policy in "contiguous", "cling", "normal":
787 result = utils.RunCmd(cmd + ["--alloc", alloc_policy, self.dev_path])
788 if not result.failed:
790 _ThrowError("Can't grow LV %s: %s", self.dev_path, result.output)
793 class DRBD8Status(object):
794 """A DRBD status representation class.
796 Note that this doesn't support unconfigured devices (cs:Unconfigured).
799 UNCONF_RE = re.compile(r"\s*[0-9]+:\s*cs:Unconfigured$")
800 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+(?:st|ro):([^/]+)/(\S+)"
801 "\s+ds:([^/]+)/(\S+)\s+.*$")
802 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
803 # Due to a bug in drbd in the kernel, introduced in
804 # commit 4b0715f096 (still unfixed as of 2011-08-22)
806 "finish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
808 CS_UNCONFIGURED = "Unconfigured"
809 CS_STANDALONE = "StandAlone"
810 CS_WFCONNECTION = "WFConnection"
811 CS_WFREPORTPARAMS = "WFReportParams"
812 CS_CONNECTED = "Connected"
813 CS_STARTINGSYNCS = "StartingSyncS"
814 CS_STARTINGSYNCT = "StartingSyncT"
815 CS_WFBITMAPS = "WFBitMapS"
816 CS_WFBITMAPT = "WFBitMapT"
817 CS_WFSYNCUUID = "WFSyncUUID"
818 CS_SYNCSOURCE = "SyncSource"
819 CS_SYNCTARGET = "SyncTarget"
820 CS_PAUSEDSYNCS = "PausedSyncS"
821 CS_PAUSEDSYNCT = "PausedSyncT"
822 CSET_SYNC = frozenset([
835 DS_DISKLESS = "Diskless"
836 DS_ATTACHING = "Attaching" # transient state
837 DS_FAILED = "Failed" # transient state, next: diskless
838 DS_NEGOTIATING = "Negotiating" # transient state
839 DS_INCONSISTENT = "Inconsistent" # while syncing or after creation
840 DS_OUTDATED = "Outdated"
841 DS_DUNKNOWN = "DUnknown" # shown for peer disk when not connected
842 DS_CONSISTENT = "Consistent"
843 DS_UPTODATE = "UpToDate" # normal state
845 RO_PRIMARY = "Primary"
846 RO_SECONDARY = "Secondary"
847 RO_UNKNOWN = "Unknown"
849 def __init__(self, procline):
850 u = self.UNCONF_RE.match(procline)
852 self.cstatus = self.CS_UNCONFIGURED
853 self.lrole = self.rrole = self.ldisk = self.rdisk = None
855 m = self.LINE_RE.match(procline)
857 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
858 self.cstatus = m.group(1)
859 self.lrole = m.group(2)
860 self.rrole = m.group(3)
861 self.ldisk = m.group(4)
862 self.rdisk = m.group(5)
864 # end reading of data from the LINE_RE or UNCONF_RE
866 self.is_standalone = self.cstatus == self.CS_STANDALONE
867 self.is_wfconn = self.cstatus == self.CS_WFCONNECTION
868 self.is_connected = self.cstatus == self.CS_CONNECTED
869 self.is_primary = self.lrole == self.RO_PRIMARY
870 self.is_secondary = self.lrole == self.RO_SECONDARY
871 self.peer_primary = self.rrole == self.RO_PRIMARY
872 self.peer_secondary = self.rrole == self.RO_SECONDARY
873 self.both_primary = self.is_primary and self.peer_primary
874 self.both_secondary = self.is_secondary and self.peer_secondary
876 self.is_diskless = self.ldisk == self.DS_DISKLESS
877 self.is_disk_uptodate = self.ldisk == self.DS_UPTODATE
879 self.is_in_resync = self.cstatus in self.CSET_SYNC
880 self.is_in_use = self.cstatus != self.CS_UNCONFIGURED
882 m = self.SYNC_RE.match(procline)
884 self.sync_percent = float(m.group(1))
885 hours = int(m.group(2))
886 minutes = int(m.group(3))
887 seconds = int(m.group(4))
888 self.est_time = hours * 3600 + minutes * 60 + seconds
890 # we have (in this if branch) no percent information, but if
891 # we're resyncing we need to 'fake' a sync percent information,
892 # as this is how cmdlib determines if it makes sense to wait for
894 if self.is_in_resync:
895 self.sync_percent = 0
897 self.sync_percent = None
901 class BaseDRBD(BlockDev): # pylint: disable=W0223
904 This class contains a few bits of common functionality between the
905 0.7 and 8.x versions of DRBD.
908 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)(?:\.\d+)?"
909 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
910 _VALID_LINE_RE = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
911 _UNUSED_LINE_RE = re.compile("^ *([0-9]+): cs:Unconfigured$")
914 _ST_UNCONFIGURED = "Unconfigured"
915 _ST_WFCONNECTION = "WFConnection"
916 _ST_CONNECTED = "Connected"
918 _STATUS_FILE = "/proc/drbd"
919 _USERMODE_HELPER_FILE = "/sys/module/drbd/parameters/usermode_helper"
922 def _GetProcData(filename=_STATUS_FILE):
923 """Return data from /proc/drbd.
927 data = utils.ReadFile(filename).splitlines()
928 except EnvironmentError, err:
929 if err.errno == errno.ENOENT:
930 _ThrowError("The file %s cannot be opened, check if the module"
931 " is loaded (%s)", filename, str(err))
933 _ThrowError("Can't read the DRBD proc file %s: %s", filename, str(err))
935 _ThrowError("Can't read any data from %s", filename)
939 def _MassageProcData(cls, data):
940 """Transform the output of _GetProdData into a nicer form.
942 @return: a dictionary of minor: joined lines from /proc/drbd
947 old_minor = old_line = None
949 if not line: # completely empty lines, as can be returned by drbd8.0+
951 lresult = cls._VALID_LINE_RE.match(line)
952 if lresult is not None:
953 if old_minor is not None:
954 results[old_minor] = old_line
955 old_minor = int(lresult.group(1))
958 if old_minor is not None:
959 old_line += " " + line.strip()
961 if old_minor is not None:
962 results[old_minor] = old_line
966 def _GetVersion(cls, proc_data):
967 """Return the DRBD version.
969 This will return a dict with keys:
975 - proto2 (only on drbd > 8.2.X)
978 first_line = proc_data[0].strip()
979 version = cls._VERSION_RE.match(first_line)
981 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
984 values = version.groups()
985 retval = {"k_major": int(values[0]),
986 "k_minor": int(values[1]),
987 "k_point": int(values[2]),
988 "api": int(values[3]),
989 "proto": int(values[4]),
991 if values[5] is not None:
992 retval["proto2"] = values[5]
997 def GetUsermodeHelper(filename=_USERMODE_HELPER_FILE):
998 """Returns DRBD usermode_helper currently set.
1002 helper = utils.ReadFile(filename).splitlines()[0]
1003 except EnvironmentError, err:
1004 if err.errno == errno.ENOENT:
1005 _ThrowError("The file %s cannot be opened, check if the module"
1006 " is loaded (%s)", filename, str(err))
1008 _ThrowError("Can't read DRBD helper file %s: %s", filename, str(err))
1010 _ThrowError("Can't read any data from %s", filename)
1014 def _DevPath(minor):
1015 """Return the path to a drbd device for a given minor.
1018 return "/dev/drbd%d" % minor
1021 def GetUsedDevs(cls):
1022 """Compute the list of used DRBD devices.
1025 data = cls._GetProcData()
1029 match = cls._VALID_LINE_RE.match(line)
1032 minor = int(match.group(1))
1033 state = match.group(2)
1034 if state == cls._ST_UNCONFIGURED:
1036 used_devs[minor] = state, line
1040 def _SetFromMinor(self, minor):
1041 """Set our parameters based on the given minor.
1043 This sets our minor variable and our dev_path.
1047 self.minor = self.dev_path = None
1048 self.attached = False
1051 self.dev_path = self._DevPath(minor)
1052 self.attached = True
1055 def _CheckMetaSize(meta_device):
1056 """Check if the given meta device looks like a valid one.
1058 This currently only check the size, which must be around
1062 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1064 _ThrowError("Failed to get device size: %s - %s",
1065 result.fail_reason, result.output)
1067 sectors = int(result.stdout)
1068 except (TypeError, ValueError):
1069 _ThrowError("Invalid output from blockdev: '%s'", result.stdout)
1070 num_bytes = sectors * 512
1071 if num_bytes < 128 * 1024 * 1024: # less than 128MiB
1072 _ThrowError("Meta device too small (%.2fMib)", (num_bytes / 1024 / 1024))
1073 # the maximum *valid* size of the meta device when living on top
1074 # of LVM is hard to compute: it depends on the number of stripes
1075 # and the PE size; e.g. a 2-stripe, 64MB PE will result in a 128MB
1076 # (normal size), but an eight-stripe 128MB PE will result in a 1GB
1077 # size meta device; as such, we restrict it to 1GB (a little bit
1078 # too generous, but making assumptions about PE size is hard)
1079 if num_bytes > 1024 * 1024 * 1024:
1080 _ThrowError("Meta device too big (%.2fMiB)", (num_bytes / 1024 / 1024))
1082 def Rename(self, new_id):
1085 This is not supported for drbd devices.
1088 raise errors.ProgrammerError("Can't rename a drbd device")
1091 class DRBD8(BaseDRBD):
1092 """DRBD v8.x block device.
1094 This implements the local host part of the DRBD device, i.e. it
1095 doesn't do anything to the supposed peer. If you need a fully
1096 connected DRBD pair, you need to use this class on both hosts.
1098 The unique_id for the drbd device is the (local_ip, local_port,
1099 remote_ip, remote_port) tuple, and it must have two children: the
1100 data device and the meta_device. The meta device is checked for
1101 valid size and is zeroed on create.
1108 _NET_RECONFIG_TIMEOUT = 60
1110 # command line options for barriers
1111 _DISABLE_DISK_OPTION = "--no-disk-barrier" # -a
1112 _DISABLE_DRAIN_OPTION = "--no-disk-drain" # -D
1113 _DISABLE_FLUSH_OPTION = "--no-disk-flushes" # -i
1114 _DISABLE_META_FLUSH_OPTION = "--no-md-flushes" # -m
1116 def __init__(self, unique_id, children, size, params):
1117 if children and children.count(None) > 0:
1119 if len(children) not in (0, 2):
1120 raise ValueError("Invalid configuration data %s" % str(children))
1121 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
1122 raise ValueError("Invalid configuration data %s" % str(unique_id))
1123 (self._lhost, self._lport,
1124 self._rhost, self._rport,
1125 self._aminor, self._secret) = unique_id
1127 if not _CanReadDevice(children[1].dev_path):
1128 logging.info("drbd%s: Ignoring unreadable meta device", self._aminor)
1130 super(DRBD8, self).__init__(unique_id, children, size, params)
1131 self.major = self._DRBD_MAJOR
1132 version = self._GetVersion(self._GetProcData())
1133 if version["k_major"] != 8:
1134 _ThrowError("Mismatch in DRBD kernel version and requested ganeti"
1135 " usage: kernel is %s.%s, ganeti wants 8.x",
1136 version["k_major"], version["k_minor"])
1138 if (self._lhost is not None and self._lhost == self._rhost and
1139 self._lport == self._rport):
1140 raise ValueError("Invalid configuration data, same local/remote %s" %
1145 def _InitMeta(cls, minor, dev_path):
1146 """Initialize a meta device.
1148 This will not work if the given minor is in use.
1151 # Zero the metadata first, in order to make sure drbdmeta doesn't
1152 # try to auto-detect existing filesystems or similar (see
1153 # http://code.google.com/p/ganeti/issues/detail?id=182); we only
1154 # care about the first 128MB of data in the device, even though it
1156 result = utils.RunCmd([constants.DD_CMD,
1157 "if=/dev/zero", "of=%s" % dev_path,
1158 "bs=1048576", "count=128", "oflag=direct"])
1160 _ThrowError("Can't wipe the meta device: %s", result.output)
1162 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
1163 "v08", dev_path, "0", "create-md"])
1165 _ThrowError("Can't initialize meta device: %s", result.output)
1168 def _FindUnusedMinor(cls):
1169 """Find an unused DRBD device.
1171 This is specific to 8.x as the minors are allocated dynamically,
1172 so non-existing numbers up to a max minor count are actually free.
1175 data = cls._GetProcData()
1179 match = cls._UNUSED_LINE_RE.match(line)
1181 return int(match.group(1))
1182 match = cls._VALID_LINE_RE.match(line)
1184 minor = int(match.group(1))
1185 highest = max(highest, minor)
1186 if highest is None: # there are no minors in use at all
1188 if highest >= cls._MAX_MINORS:
1189 logging.error("Error: no free drbd minors!")
1190 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1194 def _GetShowParser(cls):
1195 """Return a parser for `drbd show` output.
1197 This will either create or return an already-create parser for the
1198 output of the command `drbd show`.
1201 if cls._PARSE_SHOW is not None:
1202 return cls._PARSE_SHOW
1205 lbrace = pyp.Literal("{").suppress()
1206 rbrace = pyp.Literal("}").suppress()
1207 lbracket = pyp.Literal("[").suppress()
1208 rbracket = pyp.Literal("]").suppress()
1209 semi = pyp.Literal(";").suppress()
1210 colon = pyp.Literal(":").suppress()
1211 # this also converts the value to an int
1212 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
1214 comment = pyp.Literal("#") + pyp.Optional(pyp.restOfLine)
1215 defa = pyp.Literal("_is_default").suppress()
1216 dbl_quote = pyp.Literal('"').suppress()
1218 keyword = pyp.Word(pyp.alphanums + '-')
1221 value = pyp.Word(pyp.alphanums + '_-/.:')
1222 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
1223 ipv4_addr = (pyp.Optional(pyp.Literal("ipv4")).suppress() +
1224 pyp.Word(pyp.nums + ".") + colon + number)
1225 ipv6_addr = (pyp.Optional(pyp.Literal("ipv6")).suppress() +
1226 pyp.Optional(lbracket) + pyp.Word(pyp.hexnums + ":") +
1227 pyp.Optional(rbracket) + colon + number)
1228 # meta device, extended syntax
1229 meta_value = ((value ^ quoted) + lbracket + number + rbracket)
1230 # device name, extended syntax
1231 device_value = pyp.Literal("minor").suppress() + number
1234 stmt = (~rbrace + keyword + ~lbrace +
1235 pyp.Optional(ipv4_addr ^ ipv6_addr ^ value ^ quoted ^ meta_value ^
1237 pyp.Optional(defa) + semi +
1238 pyp.Optional(pyp.restOfLine).suppress())
1241 section_name = pyp.Word(pyp.alphas + "_")
1242 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
1244 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
1247 cls._PARSE_SHOW = bnf
1252 def _GetShowData(cls, minor):
1253 """Return the `drbdsetup show` data for a minor.
1256 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1258 logging.error("Can't display the drbd config: %s - %s",
1259 result.fail_reason, result.output)
1261 return result.stdout
1264 def _GetDevInfo(cls, out):
1265 """Parse details about a given DRBD minor.
1267 This return, if available, the local backing device (as a path)
1268 and the local and remote (ip, port) information from a string
1269 containing the output of the `drbdsetup show` command as returned
1277 bnf = cls._GetShowParser()
1281 results = bnf.parseString(out)
1282 except pyp.ParseException, err:
1283 _ThrowError("Can't parse drbdsetup show output: %s", str(err))
1285 # and massage the results into our desired format
1286 for section in results:
1288 if sname == "_this_host":
1289 for lst in section[1:]:
1290 if lst[0] == "disk":
1291 data["local_dev"] = lst[1]
1292 elif lst[0] == "meta-disk":
1293 data["meta_dev"] = lst[1]
1294 data["meta_index"] = lst[2]
1295 elif lst[0] == "address":
1296 data["local_addr"] = tuple(lst[1:])
1297 elif sname == "_remote_host":
1298 for lst in section[1:]:
1299 if lst[0] == "address":
1300 data["remote_addr"] = tuple(lst[1:])
1303 def _MatchesLocal(self, info):
1304 """Test if our local config matches with an existing device.
1306 The parameter should be as returned from `_GetDevInfo()`. This
1307 method tests if our local backing device is the same as the one in
1308 the info parameter, in effect testing if we look like the given
1313 backend, meta = self._children
1315 backend = meta = None
1317 if backend is not None:
1318 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
1320 retval = ("local_dev" not in info)
1322 if meta is not None:
1323 retval = retval and ("meta_dev" in info and
1324 info["meta_dev"] == meta.dev_path)
1325 retval = retval and ("meta_index" in info and
1326 info["meta_index"] == 0)
1328 retval = retval and ("meta_dev" not in info and
1329 "meta_index" not in info)
1332 def _MatchesNet(self, info):
1333 """Test if our network config matches with an existing device.
1335 The parameter should be as returned from `_GetDevInfo()`. This
1336 method tests if our network configuration is the same as the one
1337 in the info parameter, in effect testing if we look like the given
1341 if (((self._lhost is None and not ("local_addr" in info)) and
1342 (self._rhost is None and not ("remote_addr" in info)))):
1345 if self._lhost is None:
1348 if not ("local_addr" in info and
1349 "remote_addr" in info):
1352 retval = (info["local_addr"] == (self._lhost, self._lport))
1353 retval = (retval and
1354 info["remote_addr"] == (self._rhost, self._rport))
1357 def _AssembleLocal(self, minor, backend, meta, size):
1358 """Configure the local part of a DRBD device.
1361 args = ["drbdsetup", self._DevPath(minor), "disk",
1366 args.extend(["-d", "%sm" % size])
1368 version = self._GetVersion(self._GetProcData())
1369 vmaj = version["k_major"]
1370 vmin = version["k_minor"]
1371 vrel = version["k_point"]
1374 self._ComputeDiskBarrierArgs(vmaj, vmin, vrel,
1375 self.params[constants.LDP_BARRIERS],
1376 self.params[constants.LDP_NO_META_FLUSH])
1377 args.extend(barrier_args)
1379 if self.params[constants.LDP_DISK_CUSTOM]:
1380 args.extend(shlex.split(self.params[constants.LDP_DISK_CUSTOM]))
1382 result = utils.RunCmd(args)
1384 _ThrowError("drbd%d: can't attach local disk: %s", minor, result.output)
1387 def _ComputeDiskBarrierArgs(cls, vmaj, vmin, vrel, disabled_barriers,
1388 disable_meta_flush):
1389 """Compute the DRBD command line parameters for disk barriers
1391 Returns a list of the disk barrier parameters as requested via the
1392 disabled_barriers and disable_meta_flush arguments, and according to the
1393 supported ones in the DRBD version vmaj.vmin.vrel
1395 If the desired option is unsupported, raises errors.BlockDeviceError.
1398 disabled_barriers_set = frozenset(disabled_barriers)
1399 if not disabled_barriers_set in constants.DRBD_VALID_BARRIER_OPT:
1400 raise errors.BlockDeviceError("%s is not a valid option set for DRBD"
1401 " barriers" % disabled_barriers)
1405 # The following code assumes DRBD 8.x, with x < 4 and x != 1 (DRBD 8.1.x
1407 if not vmaj == 8 and vmin in (0, 2, 3):
1408 raise errors.BlockDeviceError("Unsupported DRBD version: %d.%d.%d" %
1411 def _AppendOrRaise(option, min_version):
1412 """Helper for DRBD options"""
1413 if min_version is not None and vrel >= min_version:
1416 raise errors.BlockDeviceError("Could not use the option %s as the"
1417 " DRBD version %d.%d.%d does not support"
1418 " it." % (option, vmaj, vmin, vrel))
1420 # the minimum version for each feature is encoded via pairs of (minor
1421 # version -> x) where x is version in which support for the option was
1423 meta_flush_supported = disk_flush_supported = {
1429 disk_drain_supported = {
1434 disk_barriers_supported = {
1439 if disable_meta_flush:
1440 _AppendOrRaise(cls._DISABLE_META_FLUSH_OPTION,
1441 meta_flush_supported.get(vmin, None))
1444 if constants.DRBD_B_DISK_FLUSH in disabled_barriers_set:
1445 _AppendOrRaise(cls._DISABLE_FLUSH_OPTION,
1446 disk_flush_supported.get(vmin, None))
1449 if constants.DRBD_B_DISK_DRAIN in disabled_barriers_set:
1450 _AppendOrRaise(cls._DISABLE_DRAIN_OPTION,
1451 disk_drain_supported.get(vmin, None))
1454 if constants.DRBD_B_DISK_BARRIERS in disabled_barriers_set:
1455 _AppendOrRaise(cls._DISABLE_DISK_OPTION,
1456 disk_barriers_supported.get(vmin, None))
1460 def _AssembleNet(self, minor, net_info, protocol,
1461 dual_pri=False, hmac=None, secret=None):
1462 """Configure the network part of the device.
1465 lhost, lport, rhost, rport = net_info
1466 if None in net_info:
1467 # we don't want network connection and actually want to make
1469 self._ShutdownNet(minor)
1472 # Workaround for a race condition. When DRBD is doing its dance to
1473 # establish a connection with its peer, it also sends the
1474 # synchronization speed over the wire. In some cases setting the
1475 # sync speed only after setting up both sides can race with DRBD
1476 # connecting, hence we set it here before telling DRBD anything
1478 self._SetMinorSyncParams(minor, self.params)
1480 if netutils.IP6Address.IsValid(lhost):
1481 if not netutils.IP6Address.IsValid(rhost):
1482 _ThrowError("drbd%d: can't connect ip %s to ip %s" %
1483 (minor, lhost, rhost))
1485 elif netutils.IP4Address.IsValid(lhost):
1486 if not netutils.IP4Address.IsValid(rhost):
1487 _ThrowError("drbd%d: can't connect ip %s to ip %s" %
1488 (minor, lhost, rhost))
1491 _ThrowError("drbd%d: Invalid ip %s" % (minor, lhost))
1493 args = ["drbdsetup", self._DevPath(minor), "net",
1494 "%s:%s:%s" % (family, lhost, lport),
1495 "%s:%s:%s" % (family, rhost, rport), protocol,
1496 "-A", "discard-zero-changes",
1503 args.extend(["-a", hmac, "-x", secret])
1505 if self.params[constants.LDP_NET_CUSTOM]:
1506 args.extend(shlex.split(self.params[constants.LDP_NET_CUSTOM]))
1508 result = utils.RunCmd(args)
1510 _ThrowError("drbd%d: can't setup network: %s - %s",
1511 minor, result.fail_reason, result.output)
1513 def _CheckNetworkConfig():
1514 info = self._GetDevInfo(self._GetShowData(minor))
1515 if not "local_addr" in info or not "remote_addr" in info:
1516 raise utils.RetryAgain()
1518 if (info["local_addr"] != (lhost, lport) or
1519 info["remote_addr"] != (rhost, rport)):
1520 raise utils.RetryAgain()
1523 utils.Retry(_CheckNetworkConfig, 1.0, 10.0)
1524 except utils.RetryTimeout:
1525 _ThrowError("drbd%d: timeout while configuring network", minor)
1527 def AddChildren(self, devices):
1528 """Add a disk to the DRBD device.
1531 if self.minor is None:
1532 _ThrowError("drbd%d: can't attach to dbrd8 during AddChildren",
1534 if len(devices) != 2:
1535 _ThrowError("drbd%d: need two devices for AddChildren", self.minor)
1536 info = self._GetDevInfo(self._GetShowData(self.minor))
1537 if "local_dev" in info:
1538 _ThrowError("drbd%d: already attached to a local disk", self.minor)
1539 backend, meta = devices
1540 if backend.dev_path is None or meta.dev_path is None:
1541 _ThrowError("drbd%d: children not ready during AddChildren", self.minor)
1544 self._CheckMetaSize(meta.dev_path)
1545 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1547 self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path, self.size)
1548 self._children = devices
1550 def RemoveChildren(self, devices):
1551 """Detach the drbd device from local storage.
1554 if self.minor is None:
1555 _ThrowError("drbd%d: can't attach to drbd8 during RemoveChildren",
1557 # early return if we don't actually have backing storage
1558 info = self._GetDevInfo(self._GetShowData(self.minor))
1559 if "local_dev" not in info:
1561 if len(self._children) != 2:
1562 _ThrowError("drbd%d: we don't have two children: %s", self.minor,
1564 if self._children.count(None) == 2: # we don't actually have children :)
1565 logging.warning("drbd%d: requested detach while detached", self.minor)
1567 if len(devices) != 2:
1568 _ThrowError("drbd%d: we need two children in RemoveChildren", self.minor)
1569 for child, dev in zip(self._children, devices):
1570 if dev != child.dev_path:
1571 _ThrowError("drbd%d: mismatch in local storage (%s != %s) in"
1572 " RemoveChildren", self.minor, dev, child.dev_path)
1574 self._ShutdownLocal(self.minor)
1578 def _SetMinorSyncParams(cls, minor, params):
1579 """Set the parameters of the DRBD syncer.
1581 This is the low-level implementation.
1584 @param minor: the drbd minor whose settings we change
1586 @param params: LD level disk parameters related to the synchronization
1588 @return: the success of the operation
1592 args = ["drbdsetup", cls._DevPath(minor), "syncer"]
1593 if params[constants.LDP_DYNAMIC_RESYNC]:
1594 version = cls._GetVersion(cls._GetProcData())
1595 vmin = version["k_minor"]
1596 vrel = version["k_point"]
1598 # By definition we are using 8.x, so just check the rest of the version
1600 if vmin != 3 or vrel < 9:
1601 logging.error("The current DRBD version (8.%d.%d) does not support the"
1602 " dynamic resync speed controller", vmin, vrel)
1605 # add the c-* parameters to args
1606 # TODO(spadaccio) use the actual parameters
1607 args.extend(["--c-plan-ahead", "20"])
1610 args.extend(["-r", "%d" % params[constants.LDP_RESYNC_RATE]])
1612 args.append("--create-device")
1613 result = utils.RunCmd(args)
1615 logging.error("Can't change syncer rate: %s - %s",
1616 result.fail_reason, result.output)
1617 return not result.failed
1619 def SetSyncParams(self, params):
1620 """Set the synchronization parameters of the DRBD syncer.
1623 @param params: LD level disk parameters related to the synchronization
1625 @return: the success of the operation
1628 if self.minor is None:
1629 logging.info("Not attached during SetSyncParams")
1631 children_result = super(DRBD8, self).SetSyncParams(params)
1632 return self._SetMinorSyncParams(self.minor, params) and children_result
1634 def PauseResumeSync(self, pause):
1635 """Pauses or resumes the sync of a DRBD device.
1637 @param pause: Wether to pause or resume
1638 @return: the success of the operation
1641 if self.minor is None:
1642 logging.info("Not attached during PauseSync")
1645 children_result = super(DRBD8, self).PauseResumeSync(pause)
1652 result = utils.RunCmd(["drbdsetup", self.dev_path, cmd])
1654 logging.error("Can't %s: %s - %s", cmd,
1655 result.fail_reason, result.output)
1656 return not result.failed and children_result
1658 def GetProcStatus(self):
1659 """Return device data from /proc.
1662 if self.minor is None:
1663 _ThrowError("drbd%d: GetStats() called while not attached", self._aminor)
1664 proc_info = self._MassageProcData(self._GetProcData())
1665 if self.minor not in proc_info:
1666 _ThrowError("drbd%d: can't find myself in /proc", self.minor)
1667 return DRBD8Status(proc_info[self.minor])
1669 def GetSyncStatus(self):
1670 """Returns the sync status of the device.
1673 If sync_percent is None, it means all is ok
1674 If estimated_time is None, it means we can't estimate
1675 the time needed, otherwise it's the time left in seconds.
1678 We set the is_degraded parameter to True on two conditions:
1679 network not connected or local disk missing.
1681 We compute the ldisk parameter based on whether we have a local
1684 @rtype: objects.BlockDevStatus
1687 if self.minor is None and not self.Attach():
1688 _ThrowError("drbd%d: can't Attach() in GetSyncStatus", self._aminor)
1690 stats = self.GetProcStatus()
1691 is_degraded = not stats.is_connected or not stats.is_disk_uptodate
1693 if stats.is_disk_uptodate:
1694 ldisk_status = constants.LDS_OKAY
1695 elif stats.is_diskless:
1696 ldisk_status = constants.LDS_FAULTY
1698 ldisk_status = constants.LDS_UNKNOWN
1700 return objects.BlockDevStatus(dev_path=self.dev_path,
1703 sync_percent=stats.sync_percent,
1704 estimated_time=stats.est_time,
1705 is_degraded=is_degraded,
1706 ldisk_status=ldisk_status)
1708 def Open(self, force=False):
1709 """Make the local state primary.
1711 If the 'force' parameter is given, the '-o' option is passed to
1712 drbdsetup. Since this is a potentially dangerous operation, the
1713 force flag should be only given after creation, when it actually
1717 if self.minor is None and not self.Attach():
1718 logging.error("DRBD cannot attach to a device during open")
1720 cmd = ["drbdsetup", self.dev_path, "primary"]
1723 result = utils.RunCmd(cmd)
1725 _ThrowError("drbd%d: can't make drbd device primary: %s", self.minor,
1729 """Make the local state secondary.
1731 This will, of course, fail if the device is in use.
1734 if self.minor is None and not self.Attach():
1735 _ThrowError("drbd%d: can't Attach() in Close()", self._aminor)
1736 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1738 _ThrowError("drbd%d: can't switch drbd device to secondary: %s",
1739 self.minor, result.output)
1741 def DisconnectNet(self):
1742 """Removes network configuration.
1744 This method shutdowns the network side of the device.
1746 The method will wait up to a hardcoded timeout for the device to
1747 go into standalone after the 'disconnect' command before
1748 re-configuring it, as sometimes it takes a while for the
1749 disconnect to actually propagate and thus we might issue a 'net'
1750 command while the device is still connected. If the device will
1751 still be attached to the network and we time out, we raise an
1755 if self.minor is None:
1756 _ThrowError("drbd%d: disk not attached in re-attach net", self._aminor)
1758 if None in (self._lhost, self._lport, self._rhost, self._rport):
1759 _ThrowError("drbd%d: DRBD disk missing network info in"
1760 " DisconnectNet()", self.minor)
1762 class _DisconnectStatus:
1763 def __init__(self, ever_disconnected):
1764 self.ever_disconnected = ever_disconnected
1766 dstatus = _DisconnectStatus(_IgnoreError(self._ShutdownNet, self.minor))
1768 def _WaitForDisconnect():
1769 if self.GetProcStatus().is_standalone:
1772 # retry the disconnect, it seems possible that due to a well-time
1773 # disconnect on the peer, my disconnect command might be ignored and
1775 dstatus.ever_disconnected = \
1776 _IgnoreError(self._ShutdownNet, self.minor) or dstatus.ever_disconnected
1778 raise utils.RetryAgain()
1781 start_time = time.time()
1784 # Start delay at 100 milliseconds and grow up to 2 seconds
1785 utils.Retry(_WaitForDisconnect, (0.1, 1.5, 2.0),
1786 self._NET_RECONFIG_TIMEOUT)
1787 except utils.RetryTimeout:
1788 if dstatus.ever_disconnected:
1789 msg = ("drbd%d: device did not react to the"
1790 " 'disconnect' command in a timely manner")
1792 msg = "drbd%d: can't shutdown network, even after multiple retries"
1794 _ThrowError(msg, self.minor)
1796 reconfig_time = time.time() - start_time
1797 if reconfig_time > (self._NET_RECONFIG_TIMEOUT * 0.25):
1798 logging.info("drbd%d: DisconnectNet: detach took %.3f seconds",
1799 self.minor, reconfig_time)
1801 def AttachNet(self, multimaster):
1802 """Reconnects the network.
1804 This method connects the network side of the device with a
1805 specified multi-master flag. The device needs to be 'Standalone'
1806 but have valid network configuration data.
1809 - multimaster: init the network in dual-primary mode
1812 if self.minor is None:
1813 _ThrowError("drbd%d: device not attached in AttachNet", self._aminor)
1815 if None in (self._lhost, self._lport, self._rhost, self._rport):
1816 _ThrowError("drbd%d: missing network info in AttachNet()", self.minor)
1818 status = self.GetProcStatus()
1820 if not status.is_standalone:
1821 _ThrowError("drbd%d: device is not standalone in AttachNet", self.minor)
1823 self._AssembleNet(self.minor,
1824 (self._lhost, self._lport, self._rhost, self._rport),
1825 constants.DRBD_NET_PROTOCOL, dual_pri=multimaster,
1826 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1829 """Check if our minor is configured.
1831 This doesn't do any device configurations - it only checks if the
1832 minor is in a state different from Unconfigured.
1834 Note that this function will not change the state of the system in
1835 any way (except in case of side-effects caused by reading from
1839 used_devs = self.GetUsedDevs()
1840 if self._aminor in used_devs:
1841 minor = self._aminor
1845 self._SetFromMinor(minor)
1846 return minor is not None
1849 """Assemble the drbd.
1852 - if we have a configured device, we try to ensure that it matches
1854 - if not, we create it from zero
1855 - anyway, set the device parameters
1858 super(DRBD8, self).Assemble()
1861 if self.minor is None:
1862 # local device completely unconfigured
1863 self._FastAssemble()
1865 # we have to recheck the local and network status and try to fix
1867 self._SlowAssemble()
1869 self.SetSyncParams(self.params)
1871 def _SlowAssemble(self):
1872 """Assembles the DRBD device from a (partially) configured device.
1874 In case of partially attached (local device matches but no network
1875 setup), we perform the network attach. If successful, we re-test
1876 the attach if can return success.
1879 # TODO: Rewrite to not use a for loop just because there is 'break'
1880 # pylint: disable=W0631
1881 net_data = (self._lhost, self._lport, self._rhost, self._rport)
1882 for minor in (self._aminor,):
1883 info = self._GetDevInfo(self._GetShowData(minor))
1884 match_l = self._MatchesLocal(info)
1885 match_r = self._MatchesNet(info)
1887 if match_l and match_r:
1888 # everything matches
1891 if match_l and not match_r and "local_addr" not in info:
1892 # disk matches, but not attached to network, attach and recheck
1893 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1894 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1895 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1898 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1899 " show' disagrees", minor)
1901 if match_r and "local_dev" not in info:
1902 # no local disk, but network attached and it matches
1903 self._AssembleLocal(minor, self._children[0].dev_path,
1904 self._children[1].dev_path, self.size)
1905 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1908 _ThrowError("drbd%d: disk attach successful, but 'drbdsetup"
1909 " show' disagrees", minor)
1911 # this case must be considered only if we actually have local
1912 # storage, i.e. not in diskless mode, because all diskless
1913 # devices are equal from the point of view of local
1915 if (match_l and "local_dev" in info and
1916 not match_r and "local_addr" in info):
1917 # strange case - the device network part points to somewhere
1918 # else, even though its local storage is ours; as we own the
1919 # drbd space, we try to disconnect from the remote peer and
1920 # reconnect to our correct one
1922 self._ShutdownNet(minor)
1923 except errors.BlockDeviceError, err:
1924 _ThrowError("drbd%d: device has correct local storage, wrong"
1925 " remote peer and is unable to disconnect in order"
1926 " to attach to the correct peer: %s", minor, str(err))
1927 # note: _AssembleNet also handles the case when we don't want
1928 # local storage (i.e. one or more of the _[lr](host|port) is
1930 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1931 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1932 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1935 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1936 " show' disagrees", minor)
1941 self._SetFromMinor(minor)
1943 _ThrowError("drbd%d: cannot activate, unknown or unhandled reason",
1946 def _FastAssemble(self):
1947 """Assemble the drbd device from zero.
1949 This is run when in Assemble we detect our minor is unused.
1952 minor = self._aminor
1953 if self._children and self._children[0] and self._children[1]:
1954 self._AssembleLocal(minor, self._children[0].dev_path,
1955 self._children[1].dev_path, self.size)
1956 if self._lhost and self._lport and self._rhost and self._rport:
1957 self._AssembleNet(minor,
1958 (self._lhost, self._lport, self._rhost, self._rport),
1959 constants.DRBD_NET_PROTOCOL,
1960 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1961 self._SetFromMinor(minor)
1964 def _ShutdownLocal(cls, minor):
1965 """Detach from the local device.
1967 I/Os will continue to be served from the remote device. If we
1968 don't have a remote device, this operation will fail.
1971 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1973 _ThrowError("drbd%d: can't detach local disk: %s", minor, result.output)
1976 def _ShutdownNet(cls, minor):
1977 """Disconnect from the remote peer.
1979 This fails if we don't have a local device.
1982 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1984 _ThrowError("drbd%d: can't shutdown network: %s", minor, result.output)
1987 def _ShutdownAll(cls, minor):
1988 """Deactivate the device.
1990 This will, of course, fail if the device is in use.
1993 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1995 _ThrowError("drbd%d: can't shutdown drbd device: %s",
1996 minor, result.output)
1999 """Shutdown the DRBD device.
2002 if self.minor is None and not self.Attach():
2003 logging.info("drbd%d: not attached during Shutdown()", self._aminor)
2007 self.dev_path = None
2008 self._ShutdownAll(minor)
2011 """Stub remove for DRBD devices.
2017 def Create(cls, unique_id, children, size, params):
2018 """Create a new DRBD8 device.
2020 Since DRBD devices are not created per se, just assembled, this
2021 function only initializes the metadata.
2024 if len(children) != 2:
2025 raise errors.ProgrammerError("Invalid setup for the drbd device")
2026 # check that the minor is unused
2027 aminor = unique_id[4]
2028 proc_info = cls._MassageProcData(cls._GetProcData())
2029 if aminor in proc_info:
2030 status = DRBD8Status(proc_info[aminor])
2031 in_use = status.is_in_use
2035 _ThrowError("drbd%d: minor is already in use at Create() time", aminor)
2038 if not meta.Attach():
2039 _ThrowError("drbd%d: can't attach to meta device '%s'",
2041 cls._CheckMetaSize(meta.dev_path)
2042 cls._InitMeta(aminor, meta.dev_path)
2043 return cls(unique_id, children, size, params)
2045 def Grow(self, amount, dryrun):
2046 """Resize the DRBD device and its backing storage.
2049 if self.minor is None:
2050 _ThrowError("drbd%d: Grow called while not attached", self._aminor)
2051 if len(self._children) != 2 or None in self._children:
2052 _ThrowError("drbd%d: cannot grow diskless device", self.minor)
2053 self._children[0].Grow(amount, dryrun)
2055 # DRBD does not support dry-run mode, so we'll return here
2057 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize", "-s",
2058 "%dm" % (self.size + amount)])
2060 _ThrowError("drbd%d: resize failed: %s", self.minor, result.output)
2063 class FileStorage(BlockDev):
2066 This class represents the a file storage backend device.
2068 The unique_id for the file device is a (file_driver, file_path) tuple.
2071 def __init__(self, unique_id, children, size, params):
2072 """Initalizes a file device backend.
2076 raise errors.BlockDeviceError("Invalid setup for file device")
2077 super(FileStorage, self).__init__(unique_id, children, size, params)
2078 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
2079 raise ValueError("Invalid configuration data %s" % str(unique_id))
2080 self.driver = unique_id[0]
2081 self.dev_path = unique_id[1]
2085 """Assemble the device.
2087 Checks whether the file device exists, raises BlockDeviceError otherwise.
2090 if not os.path.exists(self.dev_path):
2091 _ThrowError("File device '%s' does not exist" % self.dev_path)
2094 """Shutdown the device.
2096 This is a no-op for the file type, as we don't deactivate
2097 the file on shutdown.
2102 def Open(self, force=False):
2103 """Make the device ready for I/O.
2105 This is a no-op for the file type.
2111 """Notifies that the device will no longer be used for I/O.
2113 This is a no-op for the file type.
2119 """Remove the file backing the block device.
2122 @return: True if the removal was successful
2126 os.remove(self.dev_path)
2127 except OSError, err:
2128 if err.errno != errno.ENOENT:
2129 _ThrowError("Can't remove file '%s': %s", self.dev_path, err)
2131 def Rename(self, new_id):
2132 """Renames the file.
2135 # TODO: implement rename for file-based storage
2136 _ThrowError("Rename is not supported for file-based storage")
2138 def Grow(self, amount, dryrun):
2141 @param amount: the amount (in mebibytes) to grow with
2144 # Check that the file exists
2146 current_size = self.GetActualSize()
2147 new_size = current_size + amount * 1024 * 1024
2148 assert new_size > current_size, "Cannot Grow with a negative amount"
2149 # We can't really simulate the growth
2153 f = open(self.dev_path, "a+")
2154 f.truncate(new_size)
2156 except EnvironmentError, err:
2157 _ThrowError("Error in file growth: %", str(err))
2160 """Attach to an existing file.
2162 Check if this file already exists.
2165 @return: True if file exists
2168 self.attached = os.path.exists(self.dev_path)
2169 return self.attached
2171 def GetActualSize(self):
2172 """Return the actual disk size.
2174 @note: the device needs to be active when this is called
2177 assert self.attached, "BlockDevice not attached in GetActualSize()"
2179 st = os.stat(self.dev_path)
2181 except OSError, err:
2182 _ThrowError("Can't stat %s: %s", self.dev_path, err)
2185 def Create(cls, unique_id, children, size, params):
2186 """Create a new file.
2188 @param size: the size of file in MiB
2190 @rtype: L{bdev.FileStorage}
2191 @return: an instance of FileStorage
2194 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
2195 raise ValueError("Invalid configuration data %s" % str(unique_id))
2196 dev_path = unique_id[1]
2198 fd = os.open(dev_path, os.O_RDWR | os.O_CREAT | os.O_EXCL)
2199 f = os.fdopen(fd, "w")
2200 f.truncate(size * 1024 * 1024)
2202 except EnvironmentError, err:
2203 if err.errno == errno.EEXIST:
2204 _ThrowError("File already existing: %s", dev_path)
2205 _ThrowError("Error in file creation: %", str(err))
2207 return FileStorage(unique_id, children, size, params)
2210 class PersistentBlockDevice(BlockDev):
2211 """A block device with persistent node
2213 May be either directly attached, or exposed through DM (e.g. dm-multipath).
2214 udev helpers are probably required to give persistent, human-friendly
2217 For the time being, pathnames are required to lie under /dev.
2220 def __init__(self, unique_id, children, size, params):
2221 """Attaches to a static block device.
2223 The unique_id is a path under /dev.
2226 super(PersistentBlockDevice, self).__init__(unique_id, children, size,
2228 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
2229 raise ValueError("Invalid configuration data %s" % str(unique_id))
2230 self.dev_path = unique_id[1]
2231 if not os.path.realpath(self.dev_path).startswith("/dev/"):
2232 raise ValueError("Full path '%s' lies outside /dev" %
2233 os.path.realpath(self.dev_path))
2234 # TODO: this is just a safety guard checking that we only deal with devices
2235 # we know how to handle. In the future this will be integrated with
2236 # external storage backends and possible values will probably be collected
2237 # from the cluster configuration.
2238 if unique_id[0] != constants.BLOCKDEV_DRIVER_MANUAL:
2239 raise ValueError("Got persistent block device of invalid type: %s" %
2242 self.major = self.minor = None
2246 def Create(cls, unique_id, children, size, params):
2247 """Create a new device
2249 This is a noop, we only return a PersistentBlockDevice instance
2252 return PersistentBlockDevice(unique_id, children, 0, params)
2262 def Rename(self, new_id):
2263 """Rename this device.
2266 _ThrowError("Rename is not supported for PersistentBlockDev storage")
2269 """Attach to an existing block device.
2273 self.attached = False
2275 st = os.stat(self.dev_path)
2276 except OSError, err:
2277 logging.error("Error stat()'ing %s: %s", self.dev_path, str(err))
2280 if not stat.S_ISBLK(st.st_mode):
2281 logging.error("%s is not a block device", self.dev_path)
2284 self.major = os.major(st.st_rdev)
2285 self.minor = os.minor(st.st_rdev)
2286 self.attached = True
2291 """Assemble the device.
2297 """Shutdown the device.
2302 def Open(self, force=False):
2303 """Make the device ready for I/O.
2309 """Notifies that the device will no longer be used for I/O.
2314 def Grow(self, amount, dryrun):
2315 """Grow the logical volume.
2318 _ThrowError("Grow is not supported for PersistentBlockDev storage")
2322 constants.LD_LV: LogicalVolume,
2323 constants.LD_DRBD8: DRBD8,
2324 constants.LD_BLOCKDEV: PersistentBlockDevice,
2327 if constants.ENABLE_FILE_STORAGE or constants.ENABLE_SHARED_FILE_STORAGE:
2328 DEV_MAP[constants.LD_FILE] = FileStorage
2331 def _VerifyDiskType(dev_type):
2332 if dev_type not in DEV_MAP:
2333 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2336 def FindDevice(disk, children):
2337 """Search for an existing, assembled device.
2339 This will succeed only if the device exists and is assembled, but it
2340 does not do any actions in order to activate the device.
2342 @type disk: L{objects.Disk}
2343 @param disk: the disk object to find
2344 @type children: list of L{bdev.BlockDev}
2345 @param children: the list of block devices that are children of the device
2346 represented by the disk parameter
2349 _VerifyDiskType(disk.dev_type)
2350 dev_params = objects.FillDict(constants.DISK_LD_DEFAULTS[disk.dev_type],
2352 device = DEV_MAP[disk.dev_type](disk.physical_id, children, disk.size,
2354 if not device.attached:
2359 def Assemble(disk, children):
2360 """Try to attach or assemble an existing device.
2362 This will attach to assemble the device, as needed, to bring it
2363 fully up. It must be safe to run on already-assembled devices.
2365 @type disk: L{objects.Disk}
2366 @param disk: the disk object to assemble
2367 @type children: list of L{bdev.BlockDev}
2368 @param children: the list of block devices that are children of the device
2369 represented by the disk parameter
2372 _VerifyDiskType(disk.dev_type)
2373 dev_params = objects.FillDict(constants.DISK_LD_DEFAULTS[disk.dev_type],
2375 device = DEV_MAP[disk.dev_type](disk.physical_id, children, disk.size,
2381 def Create(disk, children):
2384 @type disk: L{objects.Disk}
2385 @param disk: the disk object to create
2386 @type children: list of L{bdev.BlockDev}
2387 @param children: the list of block devices that are children of the device
2388 represented by the disk parameter
2391 _VerifyDiskType(disk.dev_type)
2392 dev_params = objects.FillDict(constants.DISK_LD_DEFAULTS[disk.dev_type],
2394 device = DEV_MAP[disk.dev_type].Create(disk.physical_id, children, disk.size,