4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
28 import pyparsing as pyp
32 from ganeti import utils
33 from ganeti import errors
34 from ganeti import constants
35 from ganeti import objects
36 from ganeti import compat
37 from ganeti import netutils
40 # Size of reads in _CanReadDevice
41 _DEVICE_READ_SIZE = 128 * 1024
44 def _IgnoreError(fn, *args, **kwargs):
45 """Executes the given function, ignoring BlockDeviceErrors.
47 This is used in order to simplify the execution of cleanup or
51 @return: True when fn didn't raise an exception, False otherwise
57 except errors.BlockDeviceError, err:
58 logging.warning("Caught BlockDeviceError but ignoring: %s", str(err))
62 def _ThrowError(msg, *args):
63 """Log an error to the node daemon and the raise an exception.
66 @param msg: the text of the exception
67 @raise errors.BlockDeviceError
73 raise errors.BlockDeviceError(msg)
76 def _CanReadDevice(path):
77 """Check if we can read from the given device.
79 This tries to read the first 128k of the device.
83 utils.ReadFile(path, size=_DEVICE_READ_SIZE)
85 except EnvironmentError:
86 logging.warning("Can't read from device %s", path, exc_info=True)
90 class BlockDev(object):
91 """Block device abstract class.
93 A block device can be in the following states:
94 - not existing on the system, and by `Create()` it goes into:
95 - existing but not setup/not active, and by `Assemble()` goes into:
96 - active read-write and by `Open()` it goes into
97 - online (=used, or ready for use)
99 A device can also be online but read-only, however we are not using
100 the readonly state (LV has it, if needed in the future) and we are
101 usually looking at this like at a stack, so it's easier to
102 conceptualise the transition from not-existing to online and back
105 The many different states of the device are due to the fact that we
106 need to cover many device types:
107 - logical volumes are created, lvchange -a y $lv, and used
108 - drbd devices are attached to a local disk/remote peer and made primary
110 A block device is identified by three items:
111 - the /dev path of the device (dynamic)
112 - a unique ID of the device (static)
113 - it's major/minor pair (dynamic)
115 Not all devices implement both the first two as distinct items. LVM
116 logical volumes have their unique ID (the pair volume group, logical
117 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
118 the /dev path is again dynamic and the unique id is the pair (host1,
119 dev1), (host2, dev2).
121 You can get to a device in two ways:
122 - creating the (real) device, which returns you
123 an attached instance (lvcreate)
124 - attaching of a python instance to an existing (real) device
126 The second point, the attachement to a device, is different
127 depending on whether the device is assembled or not. At init() time,
128 we search for a device with the same unique_id as us. If found,
129 good. It also means that the device is already assembled. If not,
130 after assembly we'll have our correct major/minor.
133 def __init__(self, unique_id, children, size):
134 self._children = children
136 self.unique_id = unique_id
139 self.attached = False
143 """Assemble the device from its components.
145 Implementations of this method by child classes must ensure that:
146 - after the device has been assembled, it knows its major/minor
147 numbers; this allows other devices (usually parents) to probe
148 correctly for their children
149 - calling this method on an existing, in-use device is safe
150 - if the device is already configured (and in an OK state),
151 this method is idempotent
157 """Find a device which matches our config and attach to it.
160 raise NotImplementedError
163 """Notifies that the device will no longer be used for I/O.
166 raise NotImplementedError
169 def Create(cls, unique_id, children, size):
170 """Create the device.
172 If the device cannot be created, it will return None
173 instead. Error messages go to the logging system.
175 Note that for some devices, the unique_id is used, and for other,
176 the children. The idea is that these two, taken together, are
177 enough for both creation and assembly (later).
180 raise NotImplementedError
183 """Remove this device.
185 This makes sense only for some of the device types: LV and file
186 storage. Also note that if the device can't attach, the removal
190 raise NotImplementedError
192 def Rename(self, new_id):
193 """Rename this device.
195 This may or may not make sense for a given device type.
198 raise NotImplementedError
200 def Open(self, force=False):
201 """Make the device ready for use.
203 This makes the device ready for I/O. For now, just the DRBD
206 The force parameter signifies that if the device has any kind of
207 --force thing, it should be used, we know what we are doing.
210 raise NotImplementedError
213 """Shut down the device, freeing its children.
215 This undoes the `Assemble()` work, except for the child
216 assembling; as such, the children on the device are still
217 assembled after this call.
220 raise NotImplementedError
222 def SetSyncSpeed(self, speed):
223 """Adjust the sync speed of the mirror.
225 In case this is not a mirroring device, this is no-op.
230 for child in self._children:
231 result = result and child.SetSyncSpeed(speed)
234 def PauseResumeSync(self, pause):
235 """Pause/Resume the sync of the mirror.
237 In case this is not a mirroring device, this is no-op.
239 @param pause: Wheater to pause or resume
244 for child in self._children:
245 result = result and child.PauseResumeSync(pause)
248 def GetSyncStatus(self):
249 """Returns the sync status of the device.
251 If this device is a mirroring device, this function returns the
252 status of the mirror.
254 If sync_percent is None, it means the device is not syncing.
256 If estimated_time is None, it means we can't estimate
257 the time needed, otherwise it's the time left in seconds.
259 If is_degraded is True, it means the device is missing
260 redundancy. This is usually a sign that something went wrong in
261 the device setup, if sync_percent is None.
263 The ldisk parameter represents the degradation of the local
264 data. This is only valid for some devices, the rest will always
265 return False (not degraded).
267 @rtype: objects.BlockDevStatus
270 return objects.BlockDevStatus(dev_path=self.dev_path,
276 ldisk_status=constants.LDS_OKAY)
278 def CombinedSyncStatus(self):
279 """Calculate the mirror status recursively for our children.
281 The return value is the same as for `GetSyncStatus()` except the
282 minimum percent and maximum time are calculated across our
285 @rtype: objects.BlockDevStatus
288 status = self.GetSyncStatus()
290 min_percent = status.sync_percent
291 max_time = status.estimated_time
292 is_degraded = status.is_degraded
293 ldisk_status = status.ldisk_status
296 for child in self._children:
297 child_status = child.GetSyncStatus()
299 if min_percent is None:
300 min_percent = child_status.sync_percent
301 elif child_status.sync_percent is not None:
302 min_percent = min(min_percent, child_status.sync_percent)
305 max_time = child_status.estimated_time
306 elif child_status.estimated_time is not None:
307 max_time = max(max_time, child_status.estimated_time)
309 is_degraded = is_degraded or child_status.is_degraded
311 if ldisk_status is None:
312 ldisk_status = child_status.ldisk_status
313 elif child_status.ldisk_status is not None:
314 ldisk_status = max(ldisk_status, child_status.ldisk_status)
316 return objects.BlockDevStatus(dev_path=self.dev_path,
319 sync_percent=min_percent,
320 estimated_time=max_time,
321 is_degraded=is_degraded,
322 ldisk_status=ldisk_status)
325 def SetInfo(self, text):
326 """Update metadata with info text.
328 Only supported for some device types.
331 for child in self._children:
334 def Grow(self, amount):
335 """Grow the block device.
337 @param amount: the amount (in mebibytes) to grow with
340 raise NotImplementedError
342 def GetActualSize(self):
343 """Return the actual disk size.
345 @note: the device needs to be active when this is called
348 assert self.attached, "BlockDevice not attached in GetActualSize()"
349 result = utils.RunCmd(["blockdev", "--getsize64", self.dev_path])
351 _ThrowError("blockdev failed (%s): %s",
352 result.fail_reason, result.output)
354 sz = int(result.output.strip())
355 except (ValueError, TypeError), err:
356 _ThrowError("Failed to parse blockdev output: %s", str(err))
360 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
361 (self.__class__, self.unique_id, self._children,
362 self.major, self.minor, self.dev_path))
365 class LogicalVolume(BlockDev):
366 """Logical Volume block device.
369 _VALID_NAME_RE = re.compile("^[a-zA-Z0-9+_.-]*$")
370 _INVALID_NAMES = frozenset([".", "..", "snapshot", "pvmove"])
371 _INVALID_SUBSTRINGS = frozenset(["_mlog", "_mimage"])
373 def __init__(self, unique_id, children, size):
374 """Attaches to a LV device.
376 The unique_id is a tuple (vg_name, lv_name)
379 super(LogicalVolume, self).__init__(unique_id, children, size)
380 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
381 raise ValueError("Invalid configuration data %s" % str(unique_id))
382 self._vg_name, self._lv_name = unique_id
383 self._ValidateName(self._vg_name)
384 self._ValidateName(self._lv_name)
385 self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name)
386 self._degraded = True
387 self.major = self.minor = self.pe_size = self.stripe_count = None
391 def Create(cls, unique_id, children, size):
392 """Create a new logical volume.
395 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
396 raise errors.ProgrammerError("Invalid configuration data %s" %
398 vg_name, lv_name = unique_id
399 cls._ValidateName(vg_name)
400 cls._ValidateName(lv_name)
401 pvs_info = cls.GetPVInfo([vg_name])
403 _ThrowError("Can't compute PV info for vg %s", vg_name)
407 pvlist = [ pv[1] for pv in pvs_info ]
408 if compat.any(":" in v for v in pvlist):
409 _ThrowError("Some of your PVs have the invalid character ':' in their"
410 " name, this is not supported - please filter them out"
411 " in lvm.conf using either 'filter' or 'preferred_names'")
412 free_size = sum([ pv[0] for pv in pvs_info ])
413 current_pvs = len(pvlist)
414 stripes = min(current_pvs, constants.LVM_STRIPECOUNT)
416 # The size constraint should have been checked from the master before
417 # calling the create function.
419 _ThrowError("Not enough free space: required %s,"
420 " available %s", size, free_size)
421 cmd = ["lvcreate", "-L%dm" % size, "-n%s" % lv_name]
422 # If the free space is not well distributed, we won't be able to
423 # create an optimally-striped volume; in that case, we want to try
424 # with N, N-1, ..., 2, and finally 1 (non-stripped) number of
426 for stripes_arg in range(stripes, 0, -1):
427 result = utils.RunCmd(cmd + ["-i%d" % stripes_arg] + [vg_name] + pvlist)
428 if not result.failed:
431 _ThrowError("LV create failed (%s): %s",
432 result.fail_reason, result.output)
433 return LogicalVolume(unique_id, children, size)
436 def _GetVolumeInfo(lvm_cmd, fields):
437 """Returns LVM Volumen infos using lvm_cmd
439 @param lvm_cmd: Should be one of "pvs", "vgs" or "lvs"
440 @param fields: Fields to return
441 @return: A list of dicts each with the parsed fields
445 raise errors.ProgrammerError("No fields specified")
448 cmd = [lvm_cmd, "--noheadings", "--nosuffix", "--units=m", "--unbuffered",
449 "--separator=%s" % sep, "-o%s" % ",".join(fields)]
451 result = utils.RunCmd(cmd)
453 raise errors.CommandError("Can't get the volume information: %s - %s" %
454 (result.fail_reason, result.output))
457 for line in result.stdout.splitlines():
458 splitted_fields = line.strip().split(sep)
460 if len(fields) != len(splitted_fields):
461 raise errors.CommandError("Can't parse %s output: line '%s'" %
464 data.append(splitted_fields)
469 def GetPVInfo(cls, vg_names, filter_allocatable=True):
470 """Get the free space info for PVs in a volume group.
472 @param vg_names: list of volume group names, if empty all will be returned
473 @param filter_allocatable: whether to skip over unallocatable PVs
476 @return: list of tuples (free_space, name) with free_space in mebibytes
480 info = cls._GetVolumeInfo("pvs", ["pv_name", "vg_name", "pv_free",
482 except errors.GenericError, err:
483 logging.error("Can't get PV information: %s", err)
487 for pv_name, vg_name, pv_free, pv_attr in info:
488 # (possibly) skip over pvs which are not allocatable
489 if filter_allocatable and pv_attr[0] != "a":
491 # (possibly) skip over pvs which are not in the right volume group(s)
492 if vg_names and vg_name not in vg_names:
494 data.append((float(pv_free), pv_name, vg_name))
499 def GetVGInfo(cls, vg_names, filter_readonly=True):
500 """Get the free space info for specific VGs.
502 @param vg_names: list of volume group names, if empty all will be returned
503 @param filter_readonly: whether to skip over readonly VGs
506 @return: list of tuples (free_space, total_size, name) with free_space in
511 info = cls._GetVolumeInfo("vgs", ["vg_name", "vg_free", "vg_attr",
513 except errors.GenericError, err:
514 logging.error("Can't get VG information: %s", err)
518 for vg_name, vg_free, vg_attr, vg_size in info:
519 # (possibly) skip over vgs which are not writable
520 if filter_readonly and vg_attr[0] == "r":
522 # (possibly) skip over vgs which are not in the right volume group(s)
523 if vg_names and vg_name not in vg_names:
525 data.append((float(vg_free), float(vg_size), vg_name))
530 def _ValidateName(cls, name):
531 """Validates that a given name is valid as VG or LV name.
533 The list of valid characters and restricted names is taken out of
534 the lvm(8) manpage, with the simplification that we enforce both
535 VG and LV restrictions on the names.
538 if (not cls._VALID_NAME_RE.match(name) or
539 name in cls._INVALID_NAMES or
540 compat.any(substring in name for substring in cls._INVALID_SUBSTRINGS)):
541 _ThrowError("Invalid LVM name '%s'", name)
544 """Remove this logical volume.
547 if not self.minor and not self.Attach():
548 # the LV does not exist
550 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
551 (self._vg_name, self._lv_name)])
553 _ThrowError("Can't lvremove: %s - %s", result.fail_reason, result.output)
555 def Rename(self, new_id):
556 """Rename this logical volume.
559 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
560 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
561 new_vg, new_name = new_id
562 if new_vg != self._vg_name:
563 raise errors.ProgrammerError("Can't move a logical volume across"
564 " volume groups (from %s to to %s)" %
565 (self._vg_name, new_vg))
566 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
568 _ThrowError("Failed to rename the logical volume: %s", result.output)
569 self._lv_name = new_name
570 self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name)
573 """Attach to an existing LV.
575 This method will try to see if an existing and active LV exists
576 which matches our name. If so, its major/minor will be
580 self.attached = False
581 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
582 "--units=m", "--nosuffix",
583 "-olv_attr,lv_kernel_major,lv_kernel_minor,"
584 "vg_extent_size,stripes", self.dev_path])
586 logging.error("Can't find LV %s: %s, %s",
587 self.dev_path, result.fail_reason, result.output)
589 # the output can (and will) have multiple lines for multi-segment
590 # LVs, as the 'stripes' parameter is a segment one, so we take
591 # only the last entry, which is the one we're interested in; note
592 # that with LVM2 anyway the 'stripes' value must be constant
593 # across segments, so this is a no-op actually
594 out = result.stdout.splitlines()
595 if not out: # totally empty result? splitlines() returns at least
596 # one line for any non-empty string
597 logging.error("Can't parse LVS output, no lines? Got '%s'", str(out))
599 out = out[-1].strip().rstrip(',')
602 logging.error("Can't parse LVS output, len(%s) != 5", str(out))
605 status, major, minor, pe_size, stripes = out
607 logging.error("lvs lv_attr is not 6 characters (%s)", status)
613 except (TypeError, ValueError), err:
614 logging.error("lvs major/minor cannot be parsed: %s", str(err))
617 pe_size = int(float(pe_size))
618 except (TypeError, ValueError), err:
619 logging.error("Can't parse vg extent size: %s", err)
623 stripes = int(stripes)
624 except (TypeError, ValueError), err:
625 logging.error("Can't parse the number of stripes: %s", err)
630 self.pe_size = pe_size
631 self.stripe_count = stripes
632 self._degraded = status[0] == 'v' # virtual volume, i.e. doesn't backing
638 """Assemble the device.
640 We always run `lvchange -ay` on the LV to ensure it's active before
641 use, as there were cases when xenvg was not active after boot
642 (also possibly after disk issues).
645 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
647 _ThrowError("Can't activate lv %s: %s", self.dev_path, result.output)
650 """Shutdown the device.
652 This is a no-op for the LV device type, as we don't deactivate the
658 def GetSyncStatus(self):
659 """Returns the sync status of the device.
661 If this device is a mirroring device, this function returns the
662 status of the mirror.
664 For logical volumes, sync_percent and estimated_time are always
665 None (no recovery in progress, as we don't handle the mirrored LV
666 case). The is_degraded parameter is the inverse of the ldisk
669 For the ldisk parameter, we check if the logical volume has the
670 'virtual' type, which means it's not backed by existing storage
671 anymore (read from it return I/O error). This happens after a
672 physical disk failure and subsequent 'vgreduce --removemissing' on
675 The status was already read in Attach, so we just return it.
677 @rtype: objects.BlockDevStatus
681 ldisk_status = constants.LDS_FAULTY
683 ldisk_status = constants.LDS_OKAY
685 return objects.BlockDevStatus(dev_path=self.dev_path,
690 is_degraded=self._degraded,
691 ldisk_status=ldisk_status)
693 def Open(self, force=False):
694 """Make the device ready for I/O.
696 This is a no-op for the LV device type.
702 """Notifies that the device will no longer be used for I/O.
704 This is a no-op for the LV device type.
709 def Snapshot(self, size):
710 """Create a snapshot copy of an lvm block device.
712 @returns: tuple (vg, lv)
715 snap_name = self._lv_name + ".snap"
717 # remove existing snapshot if found
718 snap = LogicalVolume((self._vg_name, snap_name), None, size)
719 _IgnoreError(snap.Remove)
721 vg_info = self.GetVGInfo([self._vg_name])
723 _ThrowError("Can't compute VG info for vg %s", self._vg_name)
724 free_size, _, _ = vg_info[0]
726 _ThrowError("Not enough free space: required %s,"
727 " available %s", size, free_size)
729 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
730 "-n%s" % snap_name, self.dev_path])
732 _ThrowError("command: %s error: %s - %s",
733 result.cmd, result.fail_reason, result.output)
735 return (self._vg_name, snap_name)
737 def SetInfo(self, text):
738 """Update metadata with info text.
741 BlockDev.SetInfo(self, text)
743 # Replace invalid characters
744 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
745 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
747 # Only up to 128 characters are allowed
750 result = utils.RunCmd(["lvchange", "--addtag", text,
753 _ThrowError("Command: %s error: %s - %s", result.cmd, result.fail_reason,
756 def Grow(self, amount):
757 """Grow the logical volume.
760 if self.pe_size is None or self.stripe_count is None:
761 if not self.Attach():
762 _ThrowError("Can't attach to LV during Grow()")
763 full_stripe_size = self.pe_size * self.stripe_count
764 rest = amount % full_stripe_size
766 amount += full_stripe_size - rest
767 # we try multiple algorithms since the 'best' ones might not have
768 # space available in the right place, but later ones might (since
769 # they have less constraints); also note that only recent LVM
771 for alloc_policy in "contiguous", "cling", "normal":
772 result = utils.RunCmd(["lvextend", "--alloc", alloc_policy,
773 "-L", "+%dm" % amount, self.dev_path])
774 if not result.failed:
776 _ThrowError("Can't grow LV %s: %s", self.dev_path, result.output)
779 class DRBD8Status(object):
780 """A DRBD status representation class.
782 Note that this doesn't support unconfigured devices (cs:Unconfigured).
785 UNCONF_RE = re.compile(r"\s*[0-9]+:\s*cs:Unconfigured$")
786 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+(?:st|ro):([^/]+)/(\S+)"
787 "\s+ds:([^/]+)/(\S+)\s+.*$")
788 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
789 "\sfinish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
791 CS_UNCONFIGURED = "Unconfigured"
792 CS_STANDALONE = "StandAlone"
793 CS_WFCONNECTION = "WFConnection"
794 CS_WFREPORTPARAMS = "WFReportParams"
795 CS_CONNECTED = "Connected"
796 CS_STARTINGSYNCS = "StartingSyncS"
797 CS_STARTINGSYNCT = "StartingSyncT"
798 CS_WFBITMAPS = "WFBitMapS"
799 CS_WFBITMAPT = "WFBitMapT"
800 CS_WFSYNCUUID = "WFSyncUUID"
801 CS_SYNCSOURCE = "SyncSource"
802 CS_SYNCTARGET = "SyncTarget"
803 CS_PAUSEDSYNCS = "PausedSyncS"
804 CS_PAUSEDSYNCT = "PausedSyncT"
805 CSET_SYNC = frozenset([
818 DS_DISKLESS = "Diskless"
819 DS_ATTACHING = "Attaching" # transient state
820 DS_FAILED = "Failed" # transient state, next: diskless
821 DS_NEGOTIATING = "Negotiating" # transient state
822 DS_INCONSISTENT = "Inconsistent" # while syncing or after creation
823 DS_OUTDATED = "Outdated"
824 DS_DUNKNOWN = "DUnknown" # shown for peer disk when not connected
825 DS_CONSISTENT = "Consistent"
826 DS_UPTODATE = "UpToDate" # normal state
828 RO_PRIMARY = "Primary"
829 RO_SECONDARY = "Secondary"
830 RO_UNKNOWN = "Unknown"
832 def __init__(self, procline):
833 u = self.UNCONF_RE.match(procline)
835 self.cstatus = self.CS_UNCONFIGURED
836 self.lrole = self.rrole = self.ldisk = self.rdisk = None
838 m = self.LINE_RE.match(procline)
840 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
841 self.cstatus = m.group(1)
842 self.lrole = m.group(2)
843 self.rrole = m.group(3)
844 self.ldisk = m.group(4)
845 self.rdisk = m.group(5)
847 # end reading of data from the LINE_RE or UNCONF_RE
849 self.is_standalone = self.cstatus == self.CS_STANDALONE
850 self.is_wfconn = self.cstatus == self.CS_WFCONNECTION
851 self.is_connected = self.cstatus == self.CS_CONNECTED
852 self.is_primary = self.lrole == self.RO_PRIMARY
853 self.is_secondary = self.lrole == self.RO_SECONDARY
854 self.peer_primary = self.rrole == self.RO_PRIMARY
855 self.peer_secondary = self.rrole == self.RO_SECONDARY
856 self.both_primary = self.is_primary and self.peer_primary
857 self.both_secondary = self.is_secondary and self.peer_secondary
859 self.is_diskless = self.ldisk == self.DS_DISKLESS
860 self.is_disk_uptodate = self.ldisk == self.DS_UPTODATE
862 self.is_in_resync = self.cstatus in self.CSET_SYNC
863 self.is_in_use = self.cstatus != self.CS_UNCONFIGURED
865 m = self.SYNC_RE.match(procline)
867 self.sync_percent = float(m.group(1))
868 hours = int(m.group(2))
869 minutes = int(m.group(3))
870 seconds = int(m.group(4))
871 self.est_time = hours * 3600 + minutes * 60 + seconds
873 # we have (in this if branch) no percent information, but if
874 # we're resyncing we need to 'fake' a sync percent information,
875 # as this is how cmdlib determines if it makes sense to wait for
877 if self.is_in_resync:
878 self.sync_percent = 0
880 self.sync_percent = None
884 class BaseDRBD(BlockDev): # pylint: disable-msg=W0223
887 This class contains a few bits of common functionality between the
888 0.7 and 8.x versions of DRBD.
891 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)(?:\.\d+)?"
892 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
893 _VALID_LINE_RE = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
894 _UNUSED_LINE_RE = re.compile("^ *([0-9]+): cs:Unconfigured$")
897 _ST_UNCONFIGURED = "Unconfigured"
898 _ST_WFCONNECTION = "WFConnection"
899 _ST_CONNECTED = "Connected"
901 _STATUS_FILE = "/proc/drbd"
902 _USERMODE_HELPER_FILE = "/sys/module/drbd/parameters/usermode_helper"
905 def _GetProcData(filename=_STATUS_FILE):
906 """Return data from /proc/drbd.
910 data = utils.ReadFile(filename).splitlines()
911 except EnvironmentError, err:
912 if err.errno == errno.ENOENT:
913 _ThrowError("The file %s cannot be opened, check if the module"
914 " is loaded (%s)", filename, str(err))
916 _ThrowError("Can't read the DRBD proc file %s: %s", filename, str(err))
918 _ThrowError("Can't read any data from %s", filename)
922 def _MassageProcData(cls, data):
923 """Transform the output of _GetProdData into a nicer form.
925 @return: a dictionary of minor: joined lines from /proc/drbd
930 old_minor = old_line = None
932 if not line: # completely empty lines, as can be returned by drbd8.0+
934 lresult = cls._VALID_LINE_RE.match(line)
935 if lresult is not None:
936 if old_minor is not None:
937 results[old_minor] = old_line
938 old_minor = int(lresult.group(1))
941 if old_minor is not None:
942 old_line += " " + line.strip()
944 if old_minor is not None:
945 results[old_minor] = old_line
949 def _GetVersion(cls, proc_data):
950 """Return the DRBD version.
952 This will return a dict with keys:
958 - proto2 (only on drbd > 8.2.X)
961 first_line = proc_data[0].strip()
962 version = cls._VERSION_RE.match(first_line)
964 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
967 values = version.groups()
968 retval = {'k_major': int(values[0]),
969 'k_minor': int(values[1]),
970 'k_point': int(values[2]),
971 'api': int(values[3]),
972 'proto': int(values[4]),
974 if values[5] is not None:
975 retval['proto2'] = values[5]
980 def GetUsermodeHelper(filename=_USERMODE_HELPER_FILE):
981 """Returns DRBD usermode_helper currently set.
985 helper = utils.ReadFile(filename).splitlines()[0]
986 except EnvironmentError, err:
987 if err.errno == errno.ENOENT:
988 _ThrowError("The file %s cannot be opened, check if the module"
989 " is loaded (%s)", filename, str(err))
991 _ThrowError("Can't read DRBD helper file %s: %s", filename, str(err))
993 _ThrowError("Can't read any data from %s", filename)
998 """Return the path to a drbd device for a given minor.
1001 return "/dev/drbd%d" % minor
1004 def GetUsedDevs(cls):
1005 """Compute the list of used DRBD devices.
1008 data = cls._GetProcData()
1012 match = cls._VALID_LINE_RE.match(line)
1015 minor = int(match.group(1))
1016 state = match.group(2)
1017 if state == cls._ST_UNCONFIGURED:
1019 used_devs[minor] = state, line
1023 def _SetFromMinor(self, minor):
1024 """Set our parameters based on the given minor.
1026 This sets our minor variable and our dev_path.
1030 self.minor = self.dev_path = None
1031 self.attached = False
1034 self.dev_path = self._DevPath(minor)
1035 self.attached = True
1038 def _CheckMetaSize(meta_device):
1039 """Check if the given meta device looks like a valid one.
1041 This currently only check the size, which must be around
1045 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1047 _ThrowError("Failed to get device size: %s - %s",
1048 result.fail_reason, result.output)
1050 sectors = int(result.stdout)
1051 except (TypeError, ValueError):
1052 _ThrowError("Invalid output from blockdev: '%s'", result.stdout)
1053 num_bytes = sectors * 512
1054 if num_bytes < 128 * 1024 * 1024: # less than 128MiB
1055 _ThrowError("Meta device too small (%.2fMib)", (num_bytes / 1024 / 1024))
1056 # the maximum *valid* size of the meta device when living on top
1057 # of LVM is hard to compute: it depends on the number of stripes
1058 # and the PE size; e.g. a 2-stripe, 64MB PE will result in a 128MB
1059 # (normal size), but an eight-stripe 128MB PE will result in a 1GB
1060 # size meta device; as such, we restrict it to 1GB (a little bit
1061 # too generous, but making assumptions about PE size is hard)
1062 if num_bytes > 1024 * 1024 * 1024:
1063 _ThrowError("Meta device too big (%.2fMiB)", (num_bytes / 1024 / 1024))
1065 def Rename(self, new_id):
1068 This is not supported for drbd devices.
1071 raise errors.ProgrammerError("Can't rename a drbd device")
1074 class DRBD8(BaseDRBD):
1075 """DRBD v8.x block device.
1077 This implements the local host part of the DRBD device, i.e. it
1078 doesn't do anything to the supposed peer. If you need a fully
1079 connected DRBD pair, you need to use this class on both hosts.
1081 The unique_id for the drbd device is the (local_ip, local_port,
1082 remote_ip, remote_port) tuple, and it must have two children: the
1083 data device and the meta_device. The meta device is checked for
1084 valid size and is zeroed on create.
1091 _NET_RECONFIG_TIMEOUT = 60
1093 def __init__(self, unique_id, children, size):
1094 if children and children.count(None) > 0:
1096 if len(children) not in (0, 2):
1097 raise ValueError("Invalid configuration data %s" % str(children))
1098 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
1099 raise ValueError("Invalid configuration data %s" % str(unique_id))
1100 (self._lhost, self._lport,
1101 self._rhost, self._rport,
1102 self._aminor, self._secret) = unique_id
1104 if not _CanReadDevice(children[1].dev_path):
1105 logging.info("drbd%s: Ignoring unreadable meta device", self._aminor)
1107 super(DRBD8, self).__init__(unique_id, children, size)
1108 self.major = self._DRBD_MAJOR
1109 version = self._GetVersion(self._GetProcData())
1110 if version['k_major'] != 8 :
1111 _ThrowError("Mismatch in DRBD kernel version and requested ganeti"
1112 " usage: kernel is %s.%s, ganeti wants 8.x",
1113 version['k_major'], version['k_minor'])
1115 if (self._lhost is not None and self._lhost == self._rhost and
1116 self._lport == self._rport):
1117 raise ValueError("Invalid configuration data, same local/remote %s" %
1122 def _InitMeta(cls, minor, dev_path):
1123 """Initialize a meta device.
1125 This will not work if the given minor is in use.
1128 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
1129 "v08", dev_path, "0", "create-md"])
1131 _ThrowError("Can't initialize meta device: %s", result.output)
1134 def _FindUnusedMinor(cls):
1135 """Find an unused DRBD device.
1137 This is specific to 8.x as the minors are allocated dynamically,
1138 so non-existing numbers up to a max minor count are actually free.
1141 data = cls._GetProcData()
1145 match = cls._UNUSED_LINE_RE.match(line)
1147 return int(match.group(1))
1148 match = cls._VALID_LINE_RE.match(line)
1150 minor = int(match.group(1))
1151 highest = max(highest, minor)
1152 if highest is None: # there are no minors in use at all
1154 if highest >= cls._MAX_MINORS:
1155 logging.error("Error: no free drbd minors!")
1156 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1160 def _GetShowParser(cls):
1161 """Return a parser for `drbd show` output.
1163 This will either create or return an already-create parser for the
1164 output of the command `drbd show`.
1167 if cls._PARSE_SHOW is not None:
1168 return cls._PARSE_SHOW
1171 lbrace = pyp.Literal("{").suppress()
1172 rbrace = pyp.Literal("}").suppress()
1173 lbracket = pyp.Literal("[").suppress()
1174 rbracket = pyp.Literal("]").suppress()
1175 semi = pyp.Literal(";").suppress()
1176 colon = pyp.Literal(":").suppress()
1177 # this also converts the value to an int
1178 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
1180 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
1181 defa = pyp.Literal("_is_default").suppress()
1182 dbl_quote = pyp.Literal('"').suppress()
1184 keyword = pyp.Word(pyp.alphanums + '-')
1187 value = pyp.Word(pyp.alphanums + '_-/.:')
1188 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
1189 ipv4_addr = (pyp.Optional(pyp.Literal("ipv4")).suppress() +
1190 pyp.Word(pyp.nums + ".") + colon + number)
1191 ipv6_addr = (pyp.Optional(pyp.Literal("ipv6")).suppress() +
1192 pyp.Optional(lbracket) + pyp.Word(pyp.hexnums + ":") +
1193 pyp.Optional(rbracket) + colon + number)
1194 # meta device, extended syntax
1195 meta_value = ((value ^ quoted) + lbracket + number + rbracket)
1196 # device name, extended syntax
1197 device_value = pyp.Literal("minor").suppress() + number
1200 stmt = (~rbrace + keyword + ~lbrace +
1201 pyp.Optional(ipv4_addr ^ ipv6_addr ^ value ^ quoted ^ meta_value ^
1203 pyp.Optional(defa) + semi +
1204 pyp.Optional(pyp.restOfLine).suppress())
1207 section_name = pyp.Word(pyp.alphas + '_')
1208 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
1210 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
1213 cls._PARSE_SHOW = bnf
1218 def _GetShowData(cls, minor):
1219 """Return the `drbdsetup show` data for a minor.
1222 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1224 logging.error("Can't display the drbd config: %s - %s",
1225 result.fail_reason, result.output)
1227 return result.stdout
1230 def _GetDevInfo(cls, out):
1231 """Parse details about a given DRBD minor.
1233 This return, if available, the local backing device (as a path)
1234 and the local and remote (ip, port) information from a string
1235 containing the output of the `drbdsetup show` command as returned
1243 bnf = cls._GetShowParser()
1247 results = bnf.parseString(out)
1248 except pyp.ParseException, err:
1249 _ThrowError("Can't parse drbdsetup show output: %s", str(err))
1251 # and massage the results into our desired format
1252 for section in results:
1254 if sname == "_this_host":
1255 for lst in section[1:]:
1256 if lst[0] == "disk":
1257 data["local_dev"] = lst[1]
1258 elif lst[0] == "meta-disk":
1259 data["meta_dev"] = lst[1]
1260 data["meta_index"] = lst[2]
1261 elif lst[0] == "address":
1262 data["local_addr"] = tuple(lst[1:])
1263 elif sname == "_remote_host":
1264 for lst in section[1:]:
1265 if lst[0] == "address":
1266 data["remote_addr"] = tuple(lst[1:])
1269 def _MatchesLocal(self, info):
1270 """Test if our local config matches with an existing device.
1272 The parameter should be as returned from `_GetDevInfo()`. This
1273 method tests if our local backing device is the same as the one in
1274 the info parameter, in effect testing if we look like the given
1279 backend, meta = self._children
1281 backend = meta = None
1283 if backend is not None:
1284 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
1286 retval = ("local_dev" not in info)
1288 if meta is not None:
1289 retval = retval and ("meta_dev" in info and
1290 info["meta_dev"] == meta.dev_path)
1291 retval = retval and ("meta_index" in info and
1292 info["meta_index"] == 0)
1294 retval = retval and ("meta_dev" not in info and
1295 "meta_index" not in info)
1298 def _MatchesNet(self, info):
1299 """Test if our network config matches with an existing device.
1301 The parameter should be as returned from `_GetDevInfo()`. This
1302 method tests if our network configuration is the same as the one
1303 in the info parameter, in effect testing if we look like the given
1307 if (((self._lhost is None and not ("local_addr" in info)) and
1308 (self._rhost is None and not ("remote_addr" in info)))):
1311 if self._lhost is None:
1314 if not ("local_addr" in info and
1315 "remote_addr" in info):
1318 retval = (info["local_addr"] == (self._lhost, self._lport))
1319 retval = (retval and
1320 info["remote_addr"] == (self._rhost, self._rport))
1324 def _AssembleLocal(cls, minor, backend, meta, size):
1325 """Configure the local part of a DRBD device.
1328 args = ["drbdsetup", cls._DevPath(minor), "disk",
1333 args.extend(["-d", "%sm" % size])
1334 if not constants.DRBD_BARRIERS: # disable barriers, if configured so
1335 version = cls._GetVersion(cls._GetProcData())
1336 # various DRBD versions support different disk barrier options;
1337 # what we aim here is to revert back to the 'drain' method of
1338 # disk flushes and to disable metadata barriers, in effect going
1339 # back to pre-8.0.7 behaviour
1340 vmaj = version['k_major']
1341 vmin = version['k_minor']
1342 vrel = version['k_point']
1344 if vmin == 0: # 8.0.x
1346 args.extend(['-i', '-m'])
1347 elif vmin == 2: # 8.2.x
1349 args.extend(['-i', '-m'])
1350 elif vmaj >= 3: # 8.3.x or newer
1351 args.extend(['-i', '-a', 'm'])
1352 result = utils.RunCmd(args)
1354 _ThrowError("drbd%d: can't attach local disk: %s", minor, result.output)
1357 def _AssembleNet(cls, minor, net_info, protocol,
1358 dual_pri=False, hmac=None, secret=None):
1359 """Configure the network part of the device.
1362 lhost, lport, rhost, rport = net_info
1363 if None in net_info:
1364 # we don't want network connection and actually want to make
1366 cls._ShutdownNet(minor)
1369 # Workaround for a race condition. When DRBD is doing its dance to
1370 # establish a connection with its peer, it also sends the
1371 # synchronization speed over the wire. In some cases setting the
1372 # sync speed only after setting up both sides can race with DRBD
1373 # connecting, hence we set it here before telling DRBD anything
1375 cls._SetMinorSyncSpeed(minor, constants.SYNC_SPEED)
1377 if netutils.IP6Address.IsValid(lhost):
1378 if not netutils.IP6Address.IsValid(rhost):
1379 _ThrowError("drbd%d: can't connect ip %s to ip %s" %
1380 (minor, lhost, rhost))
1382 elif netutils.IP4Address.IsValid(lhost):
1383 if not netutils.IP4Address.IsValid(rhost):
1384 _ThrowError("drbd%d: can't connect ip %s to ip %s" %
1385 (minor, lhost, rhost))
1388 _ThrowError("drbd%d: Invalid ip %s" % (minor, lhost))
1390 args = ["drbdsetup", cls._DevPath(minor), "net",
1391 "%s:%s:%s" % (family, lhost, lport),
1392 "%s:%s:%s" % (family, rhost, rport), protocol,
1393 "-A", "discard-zero-changes",
1400 args.extend(["-a", hmac, "-x", secret])
1401 result = utils.RunCmd(args)
1403 _ThrowError("drbd%d: can't setup network: %s - %s",
1404 minor, result.fail_reason, result.output)
1406 def _CheckNetworkConfig():
1407 info = cls._GetDevInfo(cls._GetShowData(minor))
1408 if not "local_addr" in info or not "remote_addr" in info:
1409 raise utils.RetryAgain()
1411 if (info["local_addr"] != (lhost, lport) or
1412 info["remote_addr"] != (rhost, rport)):
1413 raise utils.RetryAgain()
1416 utils.Retry(_CheckNetworkConfig, 1.0, 10.0)
1417 except utils.RetryTimeout:
1418 _ThrowError("drbd%d: timeout while configuring network", minor)
1420 def AddChildren(self, devices):
1421 """Add a disk to the DRBD device.
1424 if self.minor is None:
1425 _ThrowError("drbd%d: can't attach to dbrd8 during AddChildren",
1427 if len(devices) != 2:
1428 _ThrowError("drbd%d: need two devices for AddChildren", self.minor)
1429 info = self._GetDevInfo(self._GetShowData(self.minor))
1430 if "local_dev" in info:
1431 _ThrowError("drbd%d: already attached to a local disk", self.minor)
1432 backend, meta = devices
1433 if backend.dev_path is None or meta.dev_path is None:
1434 _ThrowError("drbd%d: children not ready during AddChildren", self.minor)
1437 self._CheckMetaSize(meta.dev_path)
1438 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1440 self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path, self.size)
1441 self._children = devices
1443 def RemoveChildren(self, devices):
1444 """Detach the drbd device from local storage.
1447 if self.minor is None:
1448 _ThrowError("drbd%d: can't attach to drbd8 during RemoveChildren",
1450 # early return if we don't actually have backing storage
1451 info = self._GetDevInfo(self._GetShowData(self.minor))
1452 if "local_dev" not in info:
1454 if len(self._children) != 2:
1455 _ThrowError("drbd%d: we don't have two children: %s", self.minor,
1457 if self._children.count(None) == 2: # we don't actually have children :)
1458 logging.warning("drbd%d: requested detach while detached", self.minor)
1460 if len(devices) != 2:
1461 _ThrowError("drbd%d: we need two children in RemoveChildren", self.minor)
1462 for child, dev in zip(self._children, devices):
1463 if dev != child.dev_path:
1464 _ThrowError("drbd%d: mismatch in local storage (%s != %s) in"
1465 " RemoveChildren", self.minor, dev, child.dev_path)
1467 self._ShutdownLocal(self.minor)
1471 def _SetMinorSyncSpeed(cls, minor, kbytes):
1472 """Set the speed of the DRBD syncer.
1474 This is the low-level implementation.
1477 @param minor: the drbd minor whose settings we change
1479 @param kbytes: the speed in kbytes/second
1481 @return: the success of the operation
1484 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "syncer",
1485 "-r", "%d" % kbytes, "--create-device"])
1487 logging.error("Can't change syncer rate: %s - %s",
1488 result.fail_reason, result.output)
1489 return not result.failed
1491 def SetSyncSpeed(self, kbytes):
1492 """Set the speed of the DRBD syncer.
1495 @param kbytes: the speed in kbytes/second
1497 @return: the success of the operation
1500 if self.minor is None:
1501 logging.info("Not attached during SetSyncSpeed")
1503 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1504 return self._SetMinorSyncSpeed(self.minor, kbytes) and children_result
1506 def PauseResumeSync(self, pause):
1507 """Pauses or resumes the sync of a DRBD device.
1509 @param pause: Wether to pause or resume
1510 @return: the success of the operation
1513 if self.minor is None:
1514 logging.info("Not attached during PauseSync")
1517 children_result = super(DRBD8, self).PauseResumeSync(pause)
1524 result = utils.RunCmd(["drbdsetup", self.dev_path, cmd])
1526 logging.error("Can't %s: %s - %s", cmd,
1527 result.fail_reason, result.output)
1528 return not result.failed and children_result
1530 def GetProcStatus(self):
1531 """Return device data from /proc.
1534 if self.minor is None:
1535 _ThrowError("drbd%d: GetStats() called while not attached", self._aminor)
1536 proc_info = self._MassageProcData(self._GetProcData())
1537 if self.minor not in proc_info:
1538 _ThrowError("drbd%d: can't find myself in /proc", self.minor)
1539 return DRBD8Status(proc_info[self.minor])
1541 def GetSyncStatus(self):
1542 """Returns the sync status of the device.
1545 If sync_percent is None, it means all is ok
1546 If estimated_time is None, it means we can't estimate
1547 the time needed, otherwise it's the time left in seconds.
1550 We set the is_degraded parameter to True on two conditions:
1551 network not connected or local disk missing.
1553 We compute the ldisk parameter based on whether we have a local
1556 @rtype: objects.BlockDevStatus
1559 if self.minor is None and not self.Attach():
1560 _ThrowError("drbd%d: can't Attach() in GetSyncStatus", self._aminor)
1562 stats = self.GetProcStatus()
1563 is_degraded = not stats.is_connected or not stats.is_disk_uptodate
1565 if stats.is_disk_uptodate:
1566 ldisk_status = constants.LDS_OKAY
1567 elif stats.is_diskless:
1568 ldisk_status = constants.LDS_FAULTY
1570 ldisk_status = constants.LDS_UNKNOWN
1572 return objects.BlockDevStatus(dev_path=self.dev_path,
1575 sync_percent=stats.sync_percent,
1576 estimated_time=stats.est_time,
1577 is_degraded=is_degraded,
1578 ldisk_status=ldisk_status)
1580 def Open(self, force=False):
1581 """Make the local state primary.
1583 If the 'force' parameter is given, the '-o' option is passed to
1584 drbdsetup. Since this is a potentially dangerous operation, the
1585 force flag should be only given after creation, when it actually
1589 if self.minor is None and not self.Attach():
1590 logging.error("DRBD cannot attach to a device during open")
1592 cmd = ["drbdsetup", self.dev_path, "primary"]
1595 result = utils.RunCmd(cmd)
1597 _ThrowError("drbd%d: can't make drbd device primary: %s", self.minor,
1601 """Make the local state secondary.
1603 This will, of course, fail if the device is in use.
1606 if self.minor is None and not self.Attach():
1607 _ThrowError("drbd%d: can't Attach() in Close()", self._aminor)
1608 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1610 _ThrowError("drbd%d: can't switch drbd device to secondary: %s",
1611 self.minor, result.output)
1613 def DisconnectNet(self):
1614 """Removes network configuration.
1616 This method shutdowns the network side of the device.
1618 The method will wait up to a hardcoded timeout for the device to
1619 go into standalone after the 'disconnect' command before
1620 re-configuring it, as sometimes it takes a while for the
1621 disconnect to actually propagate and thus we might issue a 'net'
1622 command while the device is still connected. If the device will
1623 still be attached to the network and we time out, we raise an
1627 if self.minor is None:
1628 _ThrowError("drbd%d: disk not attached in re-attach net", self._aminor)
1630 if None in (self._lhost, self._lport, self._rhost, self._rport):
1631 _ThrowError("drbd%d: DRBD disk missing network info in"
1632 " DisconnectNet()", self.minor)
1634 class _DisconnectStatus:
1635 def __init__(self, ever_disconnected):
1636 self.ever_disconnected = ever_disconnected
1638 dstatus = _DisconnectStatus(_IgnoreError(self._ShutdownNet, self.minor))
1640 def _WaitForDisconnect():
1641 if self.GetProcStatus().is_standalone:
1644 # retry the disconnect, it seems possible that due to a well-time
1645 # disconnect on the peer, my disconnect command might be ignored and
1647 dstatus.ever_disconnected = \
1648 _IgnoreError(self._ShutdownNet, self.minor) or dstatus.ever_disconnected
1650 raise utils.RetryAgain()
1653 start_time = time.time()
1656 # Start delay at 100 milliseconds and grow up to 2 seconds
1657 utils.Retry(_WaitForDisconnect, (0.1, 1.5, 2.0),
1658 self._NET_RECONFIG_TIMEOUT)
1659 except utils.RetryTimeout:
1660 if dstatus.ever_disconnected:
1661 msg = ("drbd%d: device did not react to the"
1662 " 'disconnect' command in a timely manner")
1664 msg = "drbd%d: can't shutdown network, even after multiple retries"
1666 _ThrowError(msg, self.minor)
1668 reconfig_time = time.time() - start_time
1669 if reconfig_time > (self._NET_RECONFIG_TIMEOUT * 0.25):
1670 logging.info("drbd%d: DisconnectNet: detach took %.3f seconds",
1671 self.minor, reconfig_time)
1673 def AttachNet(self, multimaster):
1674 """Reconnects the network.
1676 This method connects the network side of the device with a
1677 specified multi-master flag. The device needs to be 'Standalone'
1678 but have valid network configuration data.
1681 - multimaster: init the network in dual-primary mode
1684 if self.minor is None:
1685 _ThrowError("drbd%d: device not attached in AttachNet", self._aminor)
1687 if None in (self._lhost, self._lport, self._rhost, self._rport):
1688 _ThrowError("drbd%d: missing network info in AttachNet()", self.minor)
1690 status = self.GetProcStatus()
1692 if not status.is_standalone:
1693 _ThrowError("drbd%d: device is not standalone in AttachNet", self.minor)
1695 self._AssembleNet(self.minor,
1696 (self._lhost, self._lport, self._rhost, self._rport),
1697 constants.DRBD_NET_PROTOCOL, dual_pri=multimaster,
1698 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1701 """Check if our minor is configured.
1703 This doesn't do any device configurations - it only checks if the
1704 minor is in a state different from Unconfigured.
1706 Note that this function will not change the state of the system in
1707 any way (except in case of side-effects caused by reading from
1711 used_devs = self.GetUsedDevs()
1712 if self._aminor in used_devs:
1713 minor = self._aminor
1717 self._SetFromMinor(minor)
1718 return minor is not None
1721 """Assemble the drbd.
1724 - if we have a configured device, we try to ensure that it matches
1726 - if not, we create it from zero
1729 super(DRBD8, self).Assemble()
1732 if self.minor is None:
1733 # local device completely unconfigured
1734 self._FastAssemble()
1736 # we have to recheck the local and network status and try to fix
1738 self._SlowAssemble()
1740 def _SlowAssemble(self):
1741 """Assembles the DRBD device from a (partially) configured device.
1743 In case of partially attached (local device matches but no network
1744 setup), we perform the network attach. If successful, we re-test
1745 the attach if can return success.
1748 # TODO: Rewrite to not use a for loop just because there is 'break'
1749 # pylint: disable-msg=W0631
1750 net_data = (self._lhost, self._lport, self._rhost, self._rport)
1751 for minor in (self._aminor,):
1752 info = self._GetDevInfo(self._GetShowData(minor))
1753 match_l = self._MatchesLocal(info)
1754 match_r = self._MatchesNet(info)
1756 if match_l and match_r:
1757 # everything matches
1760 if match_l and not match_r and "local_addr" not in info:
1761 # disk matches, but not attached to network, attach and recheck
1762 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1763 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1764 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1767 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1768 " show' disagrees", minor)
1770 if match_r and "local_dev" not in info:
1771 # no local disk, but network attached and it matches
1772 self._AssembleLocal(minor, self._children[0].dev_path,
1773 self._children[1].dev_path, self.size)
1774 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1777 _ThrowError("drbd%d: disk attach successful, but 'drbdsetup"
1778 " show' disagrees", minor)
1780 # this case must be considered only if we actually have local
1781 # storage, i.e. not in diskless mode, because all diskless
1782 # devices are equal from the point of view of local
1784 if (match_l and "local_dev" in info and
1785 not match_r and "local_addr" in info):
1786 # strange case - the device network part points to somewhere
1787 # else, even though its local storage is ours; as we own the
1788 # drbd space, we try to disconnect from the remote peer and
1789 # reconnect to our correct one
1791 self._ShutdownNet(minor)
1792 except errors.BlockDeviceError, err:
1793 _ThrowError("drbd%d: device has correct local storage, wrong"
1794 " remote peer and is unable to disconnect in order"
1795 " to attach to the correct peer: %s", minor, str(err))
1796 # note: _AssembleNet also handles the case when we don't want
1797 # local storage (i.e. one or more of the _[lr](host|port) is
1799 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1800 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1801 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1804 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1805 " show' disagrees", minor)
1810 self._SetFromMinor(minor)
1812 _ThrowError("drbd%d: cannot activate, unknown or unhandled reason",
1815 def _FastAssemble(self):
1816 """Assemble the drbd device from zero.
1818 This is run when in Assemble we detect our minor is unused.
1821 minor = self._aminor
1822 if self._children and self._children[0] and self._children[1]:
1823 self._AssembleLocal(minor, self._children[0].dev_path,
1824 self._children[1].dev_path, self.size)
1825 if self._lhost and self._lport and self._rhost and self._rport:
1826 self._AssembleNet(minor,
1827 (self._lhost, self._lport, self._rhost, self._rport),
1828 constants.DRBD_NET_PROTOCOL,
1829 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1830 self._SetFromMinor(minor)
1833 def _ShutdownLocal(cls, minor):
1834 """Detach from the local device.
1836 I/Os will continue to be served from the remote device. If we
1837 don't have a remote device, this operation will fail.
1840 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1842 _ThrowError("drbd%d: can't detach local disk: %s", minor, result.output)
1845 def _ShutdownNet(cls, minor):
1846 """Disconnect from the remote peer.
1848 This fails if we don't have a local device.
1851 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1853 _ThrowError("drbd%d: can't shutdown network: %s", minor, result.output)
1856 def _ShutdownAll(cls, minor):
1857 """Deactivate the device.
1859 This will, of course, fail if the device is in use.
1862 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1864 _ThrowError("drbd%d: can't shutdown drbd device: %s",
1865 minor, result.output)
1868 """Shutdown the DRBD device.
1871 if self.minor is None and not self.Attach():
1872 logging.info("drbd%d: not attached during Shutdown()", self._aminor)
1876 self.dev_path = None
1877 self._ShutdownAll(minor)
1880 """Stub remove for DRBD devices.
1886 def Create(cls, unique_id, children, size):
1887 """Create a new DRBD8 device.
1889 Since DRBD devices are not created per se, just assembled, this
1890 function only initializes the metadata.
1893 if len(children) != 2:
1894 raise errors.ProgrammerError("Invalid setup for the drbd device")
1895 # check that the minor is unused
1896 aminor = unique_id[4]
1897 proc_info = cls._MassageProcData(cls._GetProcData())
1898 if aminor in proc_info:
1899 status = DRBD8Status(proc_info[aminor])
1900 in_use = status.is_in_use
1904 _ThrowError("drbd%d: minor is already in use at Create() time", aminor)
1907 if not meta.Attach():
1908 _ThrowError("drbd%d: can't attach to meta device '%s'",
1910 cls._CheckMetaSize(meta.dev_path)
1911 cls._InitMeta(aminor, meta.dev_path)
1912 return cls(unique_id, children, size)
1914 def Grow(self, amount):
1915 """Resize the DRBD device and its backing storage.
1918 if self.minor is None:
1919 _ThrowError("drbd%d: Grow called while not attached", self._aminor)
1920 if len(self._children) != 2 or None in self._children:
1921 _ThrowError("drbd%d: cannot grow diskless device", self.minor)
1922 self._children[0].Grow(amount)
1923 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize", "-s",
1924 "%dm" % (self.size + amount)])
1926 _ThrowError("drbd%d: resize failed: %s", self.minor, result.output)
1929 class FileStorage(BlockDev):
1932 This class represents the a file storage backend device.
1934 The unique_id for the file device is a (file_driver, file_path) tuple.
1937 def __init__(self, unique_id, children, size):
1938 """Initalizes a file device backend.
1942 raise errors.BlockDeviceError("Invalid setup for file device")
1943 super(FileStorage, self).__init__(unique_id, children, size)
1944 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1945 raise ValueError("Invalid configuration data %s" % str(unique_id))
1946 self.driver = unique_id[0]
1947 self.dev_path = unique_id[1]
1951 """Assemble the device.
1953 Checks whether the file device exists, raises BlockDeviceError otherwise.
1956 if not os.path.exists(self.dev_path):
1957 _ThrowError("File device '%s' does not exist" % self.dev_path)
1960 """Shutdown the device.
1962 This is a no-op for the file type, as we don't deactivate
1963 the file on shutdown.
1968 def Open(self, force=False):
1969 """Make the device ready for I/O.
1971 This is a no-op for the file type.
1977 """Notifies that the device will no longer be used for I/O.
1979 This is a no-op for the file type.
1985 """Remove the file backing the block device.
1988 @return: True if the removal was successful
1992 os.remove(self.dev_path)
1993 except OSError, err:
1994 if err.errno != errno.ENOENT:
1995 _ThrowError("Can't remove file '%s': %s", self.dev_path, err)
1997 def Rename(self, new_id):
1998 """Renames the file.
2001 # TODO: implement rename for file-based storage
2002 _ThrowError("Rename is not supported for file-based storage")
2004 def Grow(self, amount):
2007 @param amount: the amount (in mebibytes) to grow with
2010 # Check that the file exists
2012 current_size = self.GetActualSize()
2013 new_size = current_size + amount * 1024 * 1024
2014 assert new_size > current_size, "Cannot Grow with a negative amount"
2016 f = open(self.dev_path, "a+")
2017 f.truncate(new_size)
2019 except EnvironmentError, err:
2020 _ThrowError("Error in file growth: %", str(err))
2023 """Attach to an existing file.
2025 Check if this file already exists.
2028 @return: True if file exists
2031 self.attached = os.path.exists(self.dev_path)
2032 return self.attached
2034 def GetActualSize(self):
2035 """Return the actual disk size.
2037 @note: the device needs to be active when this is called
2040 assert self.attached, "BlockDevice not attached in GetActualSize()"
2042 st = os.stat(self.dev_path)
2044 except OSError, err:
2045 _ThrowError("Can't stat %s: %s", self.dev_path, err)
2048 def Create(cls, unique_id, children, size):
2049 """Create a new file.
2051 @param size: the size of file in MiB
2053 @rtype: L{bdev.FileStorage}
2054 @return: an instance of FileStorage
2057 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
2058 raise ValueError("Invalid configuration data %s" % str(unique_id))
2059 dev_path = unique_id[1]
2061 fd = os.open(dev_path, os.O_RDWR | os.O_CREAT | os.O_EXCL)
2062 f = os.fdopen(fd, "w")
2063 f.truncate(size * 1024 * 1024)
2065 except EnvironmentError, err:
2066 if err.errno == errno.EEXIST:
2067 _ThrowError("File already existing: %s", dev_path)
2068 _ThrowError("Error in file creation: %", str(err))
2070 return FileStorage(unique_id, children, size)
2073 class PersistentBlockDevice(BlockDev):
2074 """A block device with persistent node
2076 May be either directly attached, or exposed through DM (e.g. dm-multipath).
2077 udev helpers are probably required to give persistent, human-friendly
2080 For the time being, pathnames are required to lie under /dev.
2083 def __init__(self, unique_id, children, size):
2084 """Attaches to a static block device.
2086 The unique_id is a path under /dev.
2089 super(PersistentBlockDevice, self).__init__(unique_id, children, size)
2090 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
2091 raise ValueError("Invalid configuration data %s" % str(unique_id))
2092 self.dev_path = unique_id[1]
2093 if not os.path.realpath(self.dev_path).startswith('/dev/'):
2094 raise ValueError("Full path '%s' lies outside /dev" %
2095 os.path.realpath(self.dev_path))
2096 # TODO: this is just a safety guard checking that we only deal with devices
2097 # we know how to handle. In the future this will be integrated with
2098 # external storage backends and possible values will probably be collected
2099 # from the cluster configuration.
2100 if unique_id[0] != constants.BLOCKDEV_DRIVER_MANUAL:
2101 raise ValueError("Got persistent block device of invalid type: %s" %
2104 self.major = self.minor = None
2108 def Create(cls, unique_id, children, size):
2109 """Create a new device
2111 This is a noop, we only return a PersistentBlockDevice instance
2114 return PersistentBlockDevice(unique_id, children, 0)
2124 def Rename(self, new_id):
2125 """Rename this device.
2128 _ThrowError("Rename is not supported for PersistentBlockDev storage")
2131 """Attach to an existing block device.
2135 self.attached = False
2137 st = os.stat(self.dev_path)
2138 except OSError, err:
2139 logging.error("Error stat()'ing %s: %s", self.dev_path, str(err))
2142 if not stat.S_ISBLK(st.st_mode):
2143 logging.error("%s is not a block device", self.dev_path)
2146 self.major = os.major(st.st_rdev)
2147 self.minor = os.minor(st.st_rdev)
2148 self.attached = True
2153 """Assemble the device.
2159 """Shutdown the device.
2164 def Open(self, force=False):
2165 """Make the device ready for I/O.
2171 """Notifies that the device will no longer be used for I/O.
2176 def Grow(self, amount):
2177 """Grow the logical volume.
2180 _ThrowError("Grow is not supported for PersistentBlockDev storage")
2184 constants.LD_LV: LogicalVolume,
2185 constants.LD_DRBD8: DRBD8,
2186 constants.LD_BLOCKDEV: PersistentBlockDevice,
2189 if constants.ENABLE_FILE_STORAGE or constants.ENABLE_SHARED_FILE_STORAGE:
2190 DEV_MAP[constants.LD_FILE] = FileStorage
2193 def FindDevice(dev_type, unique_id, children, size):
2194 """Search for an existing, assembled device.
2196 This will succeed only if the device exists and is assembled, but it
2197 does not do any actions in order to activate the device.
2200 if dev_type not in DEV_MAP:
2201 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2202 device = DEV_MAP[dev_type](unique_id, children, size)
2203 if not device.attached:
2208 def Assemble(dev_type, unique_id, children, size):
2209 """Try to attach or assemble an existing device.
2211 This will attach to assemble the device, as needed, to bring it
2212 fully up. It must be safe to run on already-assembled devices.
2215 if dev_type not in DEV_MAP:
2216 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2217 device = DEV_MAP[dev_type](unique_id, children, size)
2222 def Create(dev_type, unique_id, children, size):
2226 if dev_type not in DEV_MAP:
2227 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2228 device = DEV_MAP[dev_type].Create(unique_id, children, size)