4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
28 import pyparsing as pyp
32 from ganeti import utils
33 from ganeti import errors
34 from ganeti import constants
35 from ganeti import objects
36 from ganeti import compat
37 from ganeti import netutils
40 # Size of reads in _CanReadDevice
41 _DEVICE_READ_SIZE = 128 * 1024
44 def _IgnoreError(fn, *args, **kwargs):
45 """Executes the given function, ignoring BlockDeviceErrors.
47 This is used in order to simplify the execution of cleanup or
51 @return: True when fn didn't raise an exception, False otherwise
57 except errors.BlockDeviceError, err:
58 logging.warning("Caught BlockDeviceError but ignoring: %s", str(err))
62 def _ThrowError(msg, *args):
63 """Log an error to the node daemon and the raise an exception.
66 @param msg: the text of the exception
67 @raise errors.BlockDeviceError
73 raise errors.BlockDeviceError(msg)
76 def _CanReadDevice(path):
77 """Check if we can read from the given device.
79 This tries to read the first 128k of the device.
83 utils.ReadFile(path, size=_DEVICE_READ_SIZE)
85 except EnvironmentError:
86 logging.warning("Can't read from device %s", path, exc_info=True)
90 class BlockDev(object):
91 """Block device abstract class.
93 A block device can be in the following states:
94 - not existing on the system, and by `Create()` it goes into:
95 - existing but not setup/not active, and by `Assemble()` goes into:
96 - active read-write and by `Open()` it goes into
97 - online (=used, or ready for use)
99 A device can also be online but read-only, however we are not using
100 the readonly state (LV has it, if needed in the future) and we are
101 usually looking at this like at a stack, so it's easier to
102 conceptualise the transition from not-existing to online and back
105 The many different states of the device are due to the fact that we
106 need to cover many device types:
107 - logical volumes are created, lvchange -a y $lv, and used
108 - drbd devices are attached to a local disk/remote peer and made primary
110 A block device is identified by three items:
111 - the /dev path of the device (dynamic)
112 - a unique ID of the device (static)
113 - it's major/minor pair (dynamic)
115 Not all devices implement both the first two as distinct items. LVM
116 logical volumes have their unique ID (the pair volume group, logical
117 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
118 the /dev path is again dynamic and the unique id is the pair (host1,
119 dev1), (host2, dev2).
121 You can get to a device in two ways:
122 - creating the (real) device, which returns you
123 an attached instance (lvcreate)
124 - attaching of a python instance to an existing (real) device
126 The second point, the attachement to a device, is different
127 depending on whether the device is assembled or not. At init() time,
128 we search for a device with the same unique_id as us. If found,
129 good. It also means that the device is already assembled. If not,
130 after assembly we'll have our correct major/minor.
133 def __init__(self, unique_id, children, size):
134 self._children = children
136 self.unique_id = unique_id
139 self.attached = False
143 """Assemble the device from its components.
145 Implementations of this method by child classes must ensure that:
146 - after the device has been assembled, it knows its major/minor
147 numbers; this allows other devices (usually parents) to probe
148 correctly for their children
149 - calling this method on an existing, in-use device is safe
150 - if the device is already configured (and in an OK state),
151 this method is idempotent
157 """Find a device which matches our config and attach to it.
160 raise NotImplementedError
163 """Notifies that the device will no longer be used for I/O.
166 raise NotImplementedError
169 def Create(cls, unique_id, children, size):
170 """Create the device.
172 If the device cannot be created, it will return None
173 instead. Error messages go to the logging system.
175 Note that for some devices, the unique_id is used, and for other,
176 the children. The idea is that these two, taken together, are
177 enough for both creation and assembly (later).
180 raise NotImplementedError
183 """Remove this device.
185 This makes sense only for some of the device types: LV and file
186 storage. Also note that if the device can't attach, the removal
190 raise NotImplementedError
192 def Rename(self, new_id):
193 """Rename this device.
195 This may or may not make sense for a given device type.
198 raise NotImplementedError
200 def Open(self, force=False):
201 """Make the device ready for use.
203 This makes the device ready for I/O. For now, just the DRBD
206 The force parameter signifies that if the device has any kind of
207 --force thing, it should be used, we know what we are doing.
210 raise NotImplementedError
213 """Shut down the device, freeing its children.
215 This undoes the `Assemble()` work, except for the child
216 assembling; as such, the children on the device are still
217 assembled after this call.
220 raise NotImplementedError
222 def SetSyncSpeed(self, speed):
223 """Adjust the sync speed of the mirror.
225 In case this is not a mirroring device, this is no-op.
230 for child in self._children:
231 result = result and child.SetSyncSpeed(speed)
234 def PauseResumeSync(self, pause):
235 """Pause/Resume the sync of the mirror.
237 In case this is not a mirroring device, this is no-op.
239 @param pause: Wheater to pause or resume
244 for child in self._children:
245 result = result and child.PauseResumeSync(pause)
248 def GetSyncStatus(self):
249 """Returns the sync status of the device.
251 If this device is a mirroring device, this function returns the
252 status of the mirror.
254 If sync_percent is None, it means the device is not syncing.
256 If estimated_time is None, it means we can't estimate
257 the time needed, otherwise it's the time left in seconds.
259 If is_degraded is True, it means the device is missing
260 redundancy. This is usually a sign that something went wrong in
261 the device setup, if sync_percent is None.
263 The ldisk parameter represents the degradation of the local
264 data. This is only valid for some devices, the rest will always
265 return False (not degraded).
267 @rtype: objects.BlockDevStatus
270 return objects.BlockDevStatus(dev_path=self.dev_path,
276 ldisk_status=constants.LDS_OKAY)
278 def CombinedSyncStatus(self):
279 """Calculate the mirror status recursively for our children.
281 The return value is the same as for `GetSyncStatus()` except the
282 minimum percent and maximum time are calculated across our
285 @rtype: objects.BlockDevStatus
288 status = self.GetSyncStatus()
290 min_percent = status.sync_percent
291 max_time = status.estimated_time
292 is_degraded = status.is_degraded
293 ldisk_status = status.ldisk_status
296 for child in self._children:
297 child_status = child.GetSyncStatus()
299 if min_percent is None:
300 min_percent = child_status.sync_percent
301 elif child_status.sync_percent is not None:
302 min_percent = min(min_percent, child_status.sync_percent)
305 max_time = child_status.estimated_time
306 elif child_status.estimated_time is not None:
307 max_time = max(max_time, child_status.estimated_time)
309 is_degraded = is_degraded or child_status.is_degraded
311 if ldisk_status is None:
312 ldisk_status = child_status.ldisk_status
313 elif child_status.ldisk_status is not None:
314 ldisk_status = max(ldisk_status, child_status.ldisk_status)
316 return objects.BlockDevStatus(dev_path=self.dev_path,
319 sync_percent=min_percent,
320 estimated_time=max_time,
321 is_degraded=is_degraded,
322 ldisk_status=ldisk_status)
324 def SetInfo(self, text):
325 """Update metadata with info text.
327 Only supported for some device types.
330 for child in self._children:
333 def Grow(self, amount, dryrun):
334 """Grow the block device.
336 @type amount: integer
337 @param amount: the amount (in mebibytes) to grow with
338 @type dryrun: boolean
339 @param dryrun: whether to execute the operation in simulation mode
340 only, without actually increasing the size
343 raise NotImplementedError
345 def GetActualSize(self):
346 """Return the actual disk size.
348 @note: the device needs to be active when this is called
351 assert self.attached, "BlockDevice not attached in GetActualSize()"
352 result = utils.RunCmd(["blockdev", "--getsize64", self.dev_path])
354 _ThrowError("blockdev failed (%s): %s",
355 result.fail_reason, result.output)
357 sz = int(result.output.strip())
358 except (ValueError, TypeError), err:
359 _ThrowError("Failed to parse blockdev output: %s", str(err))
363 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
364 (self.__class__, self.unique_id, self._children,
365 self.major, self.minor, self.dev_path))
368 class LogicalVolume(BlockDev):
369 """Logical Volume block device.
372 _VALID_NAME_RE = re.compile("^[a-zA-Z0-9+_.-]*$")
373 _INVALID_NAMES = frozenset([".", "..", "snapshot", "pvmove"])
374 _INVALID_SUBSTRINGS = frozenset(["_mlog", "_mimage"])
376 def __init__(self, unique_id, children, size):
377 """Attaches to a LV device.
379 The unique_id is a tuple (vg_name, lv_name)
382 super(LogicalVolume, self).__init__(unique_id, children, size)
383 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
384 raise ValueError("Invalid configuration data %s" % str(unique_id))
385 self._vg_name, self._lv_name = unique_id
386 self._ValidateName(self._vg_name)
387 self._ValidateName(self._lv_name)
388 self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name)
389 self._degraded = True
390 self.major = self.minor = self.pe_size = self.stripe_count = None
394 def Create(cls, unique_id, children, size):
395 """Create a new logical volume.
398 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
399 raise errors.ProgrammerError("Invalid configuration data %s" %
401 vg_name, lv_name = unique_id
402 cls._ValidateName(vg_name)
403 cls._ValidateName(lv_name)
404 pvs_info = cls.GetPVInfo([vg_name])
406 _ThrowError("Can't compute PV info for vg %s", vg_name)
410 pvlist = [pv[1] for pv in pvs_info]
411 if compat.any(":" in v for v in pvlist):
412 _ThrowError("Some of your PVs have the invalid character ':' in their"
413 " name, this is not supported - please filter them out"
414 " in lvm.conf using either 'filter' or 'preferred_names'")
415 free_size = sum([pv[0] for pv in pvs_info])
416 current_pvs = len(pvlist)
417 stripes = min(current_pvs, constants.LVM_STRIPECOUNT)
419 # The size constraint should have been checked from the master before
420 # calling the create function.
422 _ThrowError("Not enough free space: required %s,"
423 " available %s", size, free_size)
424 cmd = ["lvcreate", "-L%dm" % size, "-n%s" % lv_name]
425 # If the free space is not well distributed, we won't be able to
426 # create an optimally-striped volume; in that case, we want to try
427 # with N, N-1, ..., 2, and finally 1 (non-stripped) number of
429 for stripes_arg in range(stripes, 0, -1):
430 result = utils.RunCmd(cmd + ["-i%d" % stripes_arg] + [vg_name] + pvlist)
431 if not result.failed:
434 _ThrowError("LV create failed (%s): %s",
435 result.fail_reason, result.output)
436 return LogicalVolume(unique_id, children, size)
439 def _GetVolumeInfo(lvm_cmd, fields):
440 """Returns LVM Volumen infos using lvm_cmd
442 @param lvm_cmd: Should be one of "pvs", "vgs" or "lvs"
443 @param fields: Fields to return
444 @return: A list of dicts each with the parsed fields
448 raise errors.ProgrammerError("No fields specified")
451 cmd = [lvm_cmd, "--noheadings", "--nosuffix", "--units=m", "--unbuffered",
452 "--separator=%s" % sep, "-o%s" % ",".join(fields)]
454 result = utils.RunCmd(cmd)
456 raise errors.CommandError("Can't get the volume information: %s - %s" %
457 (result.fail_reason, result.output))
460 for line in result.stdout.splitlines():
461 splitted_fields = line.strip().split(sep)
463 if len(fields) != len(splitted_fields):
464 raise errors.CommandError("Can't parse %s output: line '%s'" %
467 data.append(splitted_fields)
472 def GetPVInfo(cls, vg_names, filter_allocatable=True):
473 """Get the free space info for PVs in a volume group.
475 @param vg_names: list of volume group names, if empty all will be returned
476 @param filter_allocatable: whether to skip over unallocatable PVs
479 @return: list of tuples (free_space, name) with free_space in mebibytes
483 info = cls._GetVolumeInfo("pvs", ["pv_name", "vg_name", "pv_free",
485 except errors.GenericError, err:
486 logging.error("Can't get PV information: %s", err)
490 for pv_name, vg_name, pv_free, pv_attr in info:
491 # (possibly) skip over pvs which are not allocatable
492 if filter_allocatable and pv_attr[0] != "a":
494 # (possibly) skip over pvs which are not in the right volume group(s)
495 if vg_names and vg_name not in vg_names:
497 data.append((float(pv_free), pv_name, vg_name))
502 def GetVGInfo(cls, vg_names, filter_readonly=True):
503 """Get the free space info for specific VGs.
505 @param vg_names: list of volume group names, if empty all will be returned
506 @param filter_readonly: whether to skip over readonly VGs
509 @return: list of tuples (free_space, total_size, name) with free_space in
514 info = cls._GetVolumeInfo("vgs", ["vg_name", "vg_free", "vg_attr",
516 except errors.GenericError, err:
517 logging.error("Can't get VG information: %s", err)
521 for vg_name, vg_free, vg_attr, vg_size in info:
522 # (possibly) skip over vgs which are not writable
523 if filter_readonly and vg_attr[0] == "r":
525 # (possibly) skip over vgs which are not in the right volume group(s)
526 if vg_names and vg_name not in vg_names:
528 data.append((float(vg_free), float(vg_size), vg_name))
533 def _ValidateName(cls, name):
534 """Validates that a given name is valid as VG or LV name.
536 The list of valid characters and restricted names is taken out of
537 the lvm(8) manpage, with the simplification that we enforce both
538 VG and LV restrictions on the names.
541 if (not cls._VALID_NAME_RE.match(name) or
542 name in cls._INVALID_NAMES or
543 compat.any(substring in name for substring in cls._INVALID_SUBSTRINGS)):
544 _ThrowError("Invalid LVM name '%s'", name)
547 """Remove this logical volume.
550 if not self.minor and not self.Attach():
551 # the LV does not exist
553 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
554 (self._vg_name, self._lv_name)])
556 _ThrowError("Can't lvremove: %s - %s", result.fail_reason, result.output)
558 def Rename(self, new_id):
559 """Rename this logical volume.
562 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
563 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
564 new_vg, new_name = new_id
565 if new_vg != self._vg_name:
566 raise errors.ProgrammerError("Can't move a logical volume across"
567 " volume groups (from %s to to %s)" %
568 (self._vg_name, new_vg))
569 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
571 _ThrowError("Failed to rename the logical volume: %s", result.output)
572 self._lv_name = new_name
573 self.dev_path = utils.PathJoin("/dev", self._vg_name, self._lv_name)
576 """Attach to an existing LV.
578 This method will try to see if an existing and active LV exists
579 which matches our name. If so, its major/minor will be
583 self.attached = False
584 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
585 "--units=m", "--nosuffix",
586 "-olv_attr,lv_kernel_major,lv_kernel_minor,"
587 "vg_extent_size,stripes", self.dev_path])
589 logging.error("Can't find LV %s: %s, %s",
590 self.dev_path, result.fail_reason, result.output)
592 # the output can (and will) have multiple lines for multi-segment
593 # LVs, as the 'stripes' parameter is a segment one, so we take
594 # only the last entry, which is the one we're interested in; note
595 # that with LVM2 anyway the 'stripes' value must be constant
596 # across segments, so this is a no-op actually
597 out = result.stdout.splitlines()
598 if not out: # totally empty result? splitlines() returns at least
599 # one line for any non-empty string
600 logging.error("Can't parse LVS output, no lines? Got '%s'", str(out))
602 out = out[-1].strip().rstrip(",")
605 logging.error("Can't parse LVS output, len(%s) != 5", str(out))
608 status, major, minor, pe_size, stripes = out
610 logging.error("lvs lv_attr is not 6 characters (%s)", status)
616 except (TypeError, ValueError), err:
617 logging.error("lvs major/minor cannot be parsed: %s", str(err))
620 pe_size = int(float(pe_size))
621 except (TypeError, ValueError), err:
622 logging.error("Can't parse vg extent size: %s", err)
626 stripes = int(stripes)
627 except (TypeError, ValueError), err:
628 logging.error("Can't parse the number of stripes: %s", err)
633 self.pe_size = pe_size
634 self.stripe_count = stripes
635 self._degraded = status[0] == "v" # virtual volume, i.e. doesn't backing
641 """Assemble the device.
643 We always run `lvchange -ay` on the LV to ensure it's active before
644 use, as there were cases when xenvg was not active after boot
645 (also possibly after disk issues).
648 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
650 _ThrowError("Can't activate lv %s: %s", self.dev_path, result.output)
653 """Shutdown the device.
655 This is a no-op for the LV device type, as we don't deactivate the
661 def GetSyncStatus(self):
662 """Returns the sync status of the device.
664 If this device is a mirroring device, this function returns the
665 status of the mirror.
667 For logical volumes, sync_percent and estimated_time are always
668 None (no recovery in progress, as we don't handle the mirrored LV
669 case). The is_degraded parameter is the inverse of the ldisk
672 For the ldisk parameter, we check if the logical volume has the
673 'virtual' type, which means it's not backed by existing storage
674 anymore (read from it return I/O error). This happens after a
675 physical disk failure and subsequent 'vgreduce --removemissing' on
678 The status was already read in Attach, so we just return it.
680 @rtype: objects.BlockDevStatus
684 ldisk_status = constants.LDS_FAULTY
686 ldisk_status = constants.LDS_OKAY
688 return objects.BlockDevStatus(dev_path=self.dev_path,
693 is_degraded=self._degraded,
694 ldisk_status=ldisk_status)
696 def Open(self, force=False):
697 """Make the device ready for I/O.
699 This is a no-op for the LV device type.
705 """Notifies that the device will no longer be used for I/O.
707 This is a no-op for the LV device type.
712 def Snapshot(self, size):
713 """Create a snapshot copy of an lvm block device.
715 @returns: tuple (vg, lv)
718 snap_name = self._lv_name + ".snap"
720 # remove existing snapshot if found
721 snap = LogicalVolume((self._vg_name, snap_name), None, size)
722 _IgnoreError(snap.Remove)
724 vg_info = self.GetVGInfo([self._vg_name])
726 _ThrowError("Can't compute VG info for vg %s", self._vg_name)
727 free_size, _, _ = vg_info[0]
729 _ThrowError("Not enough free space: required %s,"
730 " available %s", size, free_size)
732 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
733 "-n%s" % snap_name, self.dev_path])
735 _ThrowError("command: %s error: %s - %s",
736 result.cmd, result.fail_reason, result.output)
738 return (self._vg_name, snap_name)
740 def SetInfo(self, text):
741 """Update metadata with info text.
744 BlockDev.SetInfo(self, text)
746 # Replace invalid characters
747 text = re.sub("^[^A-Za-z0-9_+.]", "_", text)
748 text = re.sub("[^-A-Za-z0-9_+.]", "_", text)
750 # Only up to 128 characters are allowed
753 result = utils.RunCmd(["lvchange", "--addtag", text,
756 _ThrowError("Command: %s error: %s - %s", result.cmd, result.fail_reason,
759 def Grow(self, amount, dryrun):
760 """Grow the logical volume.
763 if self.pe_size is None or self.stripe_count is None:
764 if not self.Attach():
765 _ThrowError("Can't attach to LV during Grow()")
766 full_stripe_size = self.pe_size * self.stripe_count
767 rest = amount % full_stripe_size
769 amount += full_stripe_size - rest
770 cmd = ["lvextend", "-L", "+%dm" % amount]
773 # we try multiple algorithms since the 'best' ones might not have
774 # space available in the right place, but later ones might (since
775 # they have less constraints); also note that only recent LVM
777 for alloc_policy in "contiguous", "cling", "normal":
778 result = utils.RunCmd(cmd + ["--alloc", alloc_policy, self.dev_path])
779 if not result.failed:
781 _ThrowError("Can't grow LV %s: %s", self.dev_path, result.output)
784 class DRBD8Status(object):
785 """A DRBD status representation class.
787 Note that this doesn't support unconfigured devices (cs:Unconfigured).
790 UNCONF_RE = re.compile(r"\s*[0-9]+:\s*cs:Unconfigured$")
791 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+(?:st|ro):([^/]+)/(\S+)"
792 "\s+ds:([^/]+)/(\S+)\s+.*$")
793 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
794 # Due to a bug in drbd in the kernel, introduced in
795 # commit 4b0715f096 (still unfixed as of 2011-08-22)
797 "finish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
799 CS_UNCONFIGURED = "Unconfigured"
800 CS_STANDALONE = "StandAlone"
801 CS_WFCONNECTION = "WFConnection"
802 CS_WFREPORTPARAMS = "WFReportParams"
803 CS_CONNECTED = "Connected"
804 CS_STARTINGSYNCS = "StartingSyncS"
805 CS_STARTINGSYNCT = "StartingSyncT"
806 CS_WFBITMAPS = "WFBitMapS"
807 CS_WFBITMAPT = "WFBitMapT"
808 CS_WFSYNCUUID = "WFSyncUUID"
809 CS_SYNCSOURCE = "SyncSource"
810 CS_SYNCTARGET = "SyncTarget"
811 CS_PAUSEDSYNCS = "PausedSyncS"
812 CS_PAUSEDSYNCT = "PausedSyncT"
813 CSET_SYNC = frozenset([
826 DS_DISKLESS = "Diskless"
827 DS_ATTACHING = "Attaching" # transient state
828 DS_FAILED = "Failed" # transient state, next: diskless
829 DS_NEGOTIATING = "Negotiating" # transient state
830 DS_INCONSISTENT = "Inconsistent" # while syncing or after creation
831 DS_OUTDATED = "Outdated"
832 DS_DUNKNOWN = "DUnknown" # shown for peer disk when not connected
833 DS_CONSISTENT = "Consistent"
834 DS_UPTODATE = "UpToDate" # normal state
836 RO_PRIMARY = "Primary"
837 RO_SECONDARY = "Secondary"
838 RO_UNKNOWN = "Unknown"
840 def __init__(self, procline):
841 u = self.UNCONF_RE.match(procline)
843 self.cstatus = self.CS_UNCONFIGURED
844 self.lrole = self.rrole = self.ldisk = self.rdisk = None
846 m = self.LINE_RE.match(procline)
848 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
849 self.cstatus = m.group(1)
850 self.lrole = m.group(2)
851 self.rrole = m.group(3)
852 self.ldisk = m.group(4)
853 self.rdisk = m.group(5)
855 # end reading of data from the LINE_RE or UNCONF_RE
857 self.is_standalone = self.cstatus == self.CS_STANDALONE
858 self.is_wfconn = self.cstatus == self.CS_WFCONNECTION
859 self.is_connected = self.cstatus == self.CS_CONNECTED
860 self.is_primary = self.lrole == self.RO_PRIMARY
861 self.is_secondary = self.lrole == self.RO_SECONDARY
862 self.peer_primary = self.rrole == self.RO_PRIMARY
863 self.peer_secondary = self.rrole == self.RO_SECONDARY
864 self.both_primary = self.is_primary and self.peer_primary
865 self.both_secondary = self.is_secondary and self.peer_secondary
867 self.is_diskless = self.ldisk == self.DS_DISKLESS
868 self.is_disk_uptodate = self.ldisk == self.DS_UPTODATE
870 self.is_in_resync = self.cstatus in self.CSET_SYNC
871 self.is_in_use = self.cstatus != self.CS_UNCONFIGURED
873 m = self.SYNC_RE.match(procline)
875 self.sync_percent = float(m.group(1))
876 hours = int(m.group(2))
877 minutes = int(m.group(3))
878 seconds = int(m.group(4))
879 self.est_time = hours * 3600 + minutes * 60 + seconds
881 # we have (in this if branch) no percent information, but if
882 # we're resyncing we need to 'fake' a sync percent information,
883 # as this is how cmdlib determines if it makes sense to wait for
885 if self.is_in_resync:
886 self.sync_percent = 0
888 self.sync_percent = None
892 class BaseDRBD(BlockDev): # pylint: disable=W0223
895 This class contains a few bits of common functionality between the
896 0.7 and 8.x versions of DRBD.
899 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)(?:\.\d+)?"
900 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
901 _VALID_LINE_RE = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
902 _UNUSED_LINE_RE = re.compile("^ *([0-9]+): cs:Unconfigured$")
905 _ST_UNCONFIGURED = "Unconfigured"
906 _ST_WFCONNECTION = "WFConnection"
907 _ST_CONNECTED = "Connected"
909 _STATUS_FILE = "/proc/drbd"
910 _USERMODE_HELPER_FILE = "/sys/module/drbd/parameters/usermode_helper"
913 def _GetProcData(filename=_STATUS_FILE):
914 """Return data from /proc/drbd.
918 data = utils.ReadFile(filename).splitlines()
919 except EnvironmentError, err:
920 if err.errno == errno.ENOENT:
921 _ThrowError("The file %s cannot be opened, check if the module"
922 " is loaded (%s)", filename, str(err))
924 _ThrowError("Can't read the DRBD proc file %s: %s", filename, str(err))
926 _ThrowError("Can't read any data from %s", filename)
930 def _MassageProcData(cls, data):
931 """Transform the output of _GetProdData into a nicer form.
933 @return: a dictionary of minor: joined lines from /proc/drbd
938 old_minor = old_line = None
940 if not line: # completely empty lines, as can be returned by drbd8.0+
942 lresult = cls._VALID_LINE_RE.match(line)
943 if lresult is not None:
944 if old_minor is not None:
945 results[old_minor] = old_line
946 old_minor = int(lresult.group(1))
949 if old_minor is not None:
950 old_line += " " + line.strip()
952 if old_minor is not None:
953 results[old_minor] = old_line
957 def _GetVersion(cls, proc_data):
958 """Return the DRBD version.
960 This will return a dict with keys:
966 - proto2 (only on drbd > 8.2.X)
969 first_line = proc_data[0].strip()
970 version = cls._VERSION_RE.match(first_line)
972 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
975 values = version.groups()
976 retval = {"k_major": int(values[0]),
977 "k_minor": int(values[1]),
978 "k_point": int(values[2]),
979 "api": int(values[3]),
980 "proto": int(values[4]),
982 if values[5] is not None:
983 retval["proto2"] = values[5]
988 def GetUsermodeHelper(filename=_USERMODE_HELPER_FILE):
989 """Returns DRBD usermode_helper currently set.
993 helper = utils.ReadFile(filename).splitlines()[0]
994 except EnvironmentError, err:
995 if err.errno == errno.ENOENT:
996 _ThrowError("The file %s cannot be opened, check if the module"
997 " is loaded (%s)", filename, str(err))
999 _ThrowError("Can't read DRBD helper file %s: %s", filename, str(err))
1001 _ThrowError("Can't read any data from %s", filename)
1005 def _DevPath(minor):
1006 """Return the path to a drbd device for a given minor.
1009 return "/dev/drbd%d" % minor
1012 def GetUsedDevs(cls):
1013 """Compute the list of used DRBD devices.
1016 data = cls._GetProcData()
1020 match = cls._VALID_LINE_RE.match(line)
1023 minor = int(match.group(1))
1024 state = match.group(2)
1025 if state == cls._ST_UNCONFIGURED:
1027 used_devs[minor] = state, line
1031 def _SetFromMinor(self, minor):
1032 """Set our parameters based on the given minor.
1034 This sets our minor variable and our dev_path.
1038 self.minor = self.dev_path = None
1039 self.attached = False
1042 self.dev_path = self._DevPath(minor)
1043 self.attached = True
1046 def _CheckMetaSize(meta_device):
1047 """Check if the given meta device looks like a valid one.
1049 This currently only check the size, which must be around
1053 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
1055 _ThrowError("Failed to get device size: %s - %s",
1056 result.fail_reason, result.output)
1058 sectors = int(result.stdout)
1059 except (TypeError, ValueError):
1060 _ThrowError("Invalid output from blockdev: '%s'", result.stdout)
1061 num_bytes = sectors * 512
1062 if num_bytes < 128 * 1024 * 1024: # less than 128MiB
1063 _ThrowError("Meta device too small (%.2fMib)", (num_bytes / 1024 / 1024))
1064 # the maximum *valid* size of the meta device when living on top
1065 # of LVM is hard to compute: it depends on the number of stripes
1066 # and the PE size; e.g. a 2-stripe, 64MB PE will result in a 128MB
1067 # (normal size), but an eight-stripe 128MB PE will result in a 1GB
1068 # size meta device; as such, we restrict it to 1GB (a little bit
1069 # too generous, but making assumptions about PE size is hard)
1070 if num_bytes > 1024 * 1024 * 1024:
1071 _ThrowError("Meta device too big (%.2fMiB)", (num_bytes / 1024 / 1024))
1073 def Rename(self, new_id):
1076 This is not supported for drbd devices.
1079 raise errors.ProgrammerError("Can't rename a drbd device")
1082 class DRBD8(BaseDRBD):
1083 """DRBD v8.x block device.
1085 This implements the local host part of the DRBD device, i.e. it
1086 doesn't do anything to the supposed peer. If you need a fully
1087 connected DRBD pair, you need to use this class on both hosts.
1089 The unique_id for the drbd device is the (local_ip, local_port,
1090 remote_ip, remote_port) tuple, and it must have two children: the
1091 data device and the meta_device. The meta device is checked for
1092 valid size and is zeroed on create.
1099 _NET_RECONFIG_TIMEOUT = 60
1101 def __init__(self, unique_id, children, size):
1102 if children and children.count(None) > 0:
1104 if len(children) not in (0, 2):
1105 raise ValueError("Invalid configuration data %s" % str(children))
1106 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
1107 raise ValueError("Invalid configuration data %s" % str(unique_id))
1108 (self._lhost, self._lport,
1109 self._rhost, self._rport,
1110 self._aminor, self._secret) = unique_id
1112 if not _CanReadDevice(children[1].dev_path):
1113 logging.info("drbd%s: Ignoring unreadable meta device", self._aminor)
1115 super(DRBD8, self).__init__(unique_id, children, size)
1116 self.major = self._DRBD_MAJOR
1117 version = self._GetVersion(self._GetProcData())
1118 if version["k_major"] != 8:
1119 _ThrowError("Mismatch in DRBD kernel version and requested ganeti"
1120 " usage: kernel is %s.%s, ganeti wants 8.x",
1121 version["k_major"], version["k_minor"])
1123 if (self._lhost is not None and self._lhost == self._rhost and
1124 self._lport == self._rport):
1125 raise ValueError("Invalid configuration data, same local/remote %s" %
1130 def _InitMeta(cls, minor, dev_path):
1131 """Initialize a meta device.
1133 This will not work if the given minor is in use.
1136 # Zero the metadata first, in order to make sure drbdmeta doesn't
1137 # try to auto-detect existing filesystems or similar (see
1138 # http://code.google.com/p/ganeti/issues/detail?id=182); we only
1139 # care about the first 128MB of data in the device, even though it
1141 result = utils.RunCmd([constants.DD_CMD,
1142 "if=/dev/zero", "of=%s" % dev_path,
1143 "bs=1048576", "count=128", "oflag=direct"])
1145 _ThrowError("Can't wipe the meta device: %s", result.output)
1147 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
1148 "v08", dev_path, "0", "create-md"])
1150 _ThrowError("Can't initialize meta device: %s", result.output)
1153 def _FindUnusedMinor(cls):
1154 """Find an unused DRBD device.
1156 This is specific to 8.x as the minors are allocated dynamically,
1157 so non-existing numbers up to a max minor count are actually free.
1160 data = cls._GetProcData()
1164 match = cls._UNUSED_LINE_RE.match(line)
1166 return int(match.group(1))
1167 match = cls._VALID_LINE_RE.match(line)
1169 minor = int(match.group(1))
1170 highest = max(highest, minor)
1171 if highest is None: # there are no minors in use at all
1173 if highest >= cls._MAX_MINORS:
1174 logging.error("Error: no free drbd minors!")
1175 raise errors.BlockDeviceError("Can't find a free DRBD minor")
1179 def _GetShowParser(cls):
1180 """Return a parser for `drbd show` output.
1182 This will either create or return an already-create parser for the
1183 output of the command `drbd show`.
1186 if cls._PARSE_SHOW is not None:
1187 return cls._PARSE_SHOW
1190 lbrace = pyp.Literal("{").suppress()
1191 rbrace = pyp.Literal("}").suppress()
1192 lbracket = pyp.Literal("[").suppress()
1193 rbracket = pyp.Literal("]").suppress()
1194 semi = pyp.Literal(";").suppress()
1195 colon = pyp.Literal(":").suppress()
1196 # this also converts the value to an int
1197 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
1199 comment = pyp.Literal("#") + pyp.Optional(pyp.restOfLine)
1200 defa = pyp.Literal("_is_default").suppress()
1201 dbl_quote = pyp.Literal('"').suppress()
1203 keyword = pyp.Word(pyp.alphanums + '-')
1206 value = pyp.Word(pyp.alphanums + '_-/.:')
1207 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
1208 ipv4_addr = (pyp.Optional(pyp.Literal("ipv4")).suppress() +
1209 pyp.Word(pyp.nums + ".") + colon + number)
1210 ipv6_addr = (pyp.Optional(pyp.Literal("ipv6")).suppress() +
1211 pyp.Optional(lbracket) + pyp.Word(pyp.hexnums + ":") +
1212 pyp.Optional(rbracket) + colon + number)
1213 # meta device, extended syntax
1214 meta_value = ((value ^ quoted) + lbracket + number + rbracket)
1215 # device name, extended syntax
1216 device_value = pyp.Literal("minor").suppress() + number
1219 stmt = (~rbrace + keyword + ~lbrace +
1220 pyp.Optional(ipv4_addr ^ ipv6_addr ^ value ^ quoted ^ meta_value ^
1222 pyp.Optional(defa) + semi +
1223 pyp.Optional(pyp.restOfLine).suppress())
1226 section_name = pyp.Word(pyp.alphas + "_")
1227 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
1229 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
1232 cls._PARSE_SHOW = bnf
1237 def _GetShowData(cls, minor):
1238 """Return the `drbdsetup show` data for a minor.
1241 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
1243 logging.error("Can't display the drbd config: %s - %s",
1244 result.fail_reason, result.output)
1246 return result.stdout
1249 def _GetDevInfo(cls, out):
1250 """Parse details about a given DRBD minor.
1252 This return, if available, the local backing device (as a path)
1253 and the local and remote (ip, port) information from a string
1254 containing the output of the `drbdsetup show` command as returned
1262 bnf = cls._GetShowParser()
1266 results = bnf.parseString(out)
1267 except pyp.ParseException, err:
1268 _ThrowError("Can't parse drbdsetup show output: %s", str(err))
1270 # and massage the results into our desired format
1271 for section in results:
1273 if sname == "_this_host":
1274 for lst in section[1:]:
1275 if lst[0] == "disk":
1276 data["local_dev"] = lst[1]
1277 elif lst[0] == "meta-disk":
1278 data["meta_dev"] = lst[1]
1279 data["meta_index"] = lst[2]
1280 elif lst[0] == "address":
1281 data["local_addr"] = tuple(lst[1:])
1282 elif sname == "_remote_host":
1283 for lst in section[1:]:
1284 if lst[0] == "address":
1285 data["remote_addr"] = tuple(lst[1:])
1288 def _MatchesLocal(self, info):
1289 """Test if our local config matches with an existing device.
1291 The parameter should be as returned from `_GetDevInfo()`. This
1292 method tests if our local backing device is the same as the one in
1293 the info parameter, in effect testing if we look like the given
1298 backend, meta = self._children
1300 backend = meta = None
1302 if backend is not None:
1303 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
1305 retval = ("local_dev" not in info)
1307 if meta is not None:
1308 retval = retval and ("meta_dev" in info and
1309 info["meta_dev"] == meta.dev_path)
1310 retval = retval and ("meta_index" in info and
1311 info["meta_index"] == 0)
1313 retval = retval and ("meta_dev" not in info and
1314 "meta_index" not in info)
1317 def _MatchesNet(self, info):
1318 """Test if our network config matches with an existing device.
1320 The parameter should be as returned from `_GetDevInfo()`. This
1321 method tests if our network configuration is the same as the one
1322 in the info parameter, in effect testing if we look like the given
1326 if (((self._lhost is None and not ("local_addr" in info)) and
1327 (self._rhost is None and not ("remote_addr" in info)))):
1330 if self._lhost is None:
1333 if not ("local_addr" in info and
1334 "remote_addr" in info):
1337 retval = (info["local_addr"] == (self._lhost, self._lport))
1338 retval = (retval and
1339 info["remote_addr"] == (self._rhost, self._rport))
1343 def _AssembleLocal(cls, minor, backend, meta, size):
1344 """Configure the local part of a DRBD device.
1347 args = ["drbdsetup", cls._DevPath(minor), "disk",
1352 args.extend(["-d", "%sm" % size])
1353 if not constants.DRBD_BARRIERS: # disable barriers, if configured so
1354 version = cls._GetVersion(cls._GetProcData())
1355 # various DRBD versions support different disk barrier options;
1356 # what we aim here is to revert back to the 'drain' method of
1357 # disk flushes and to disable metadata barriers, in effect going
1358 # back to pre-8.0.7 behaviour
1359 vmaj = version["k_major"]
1360 vmin = version["k_minor"]
1361 vrel = version["k_point"]
1363 if vmin == 0: # 8.0.x
1365 args.extend(["-i", "-m"])
1366 elif vmin == 2: # 8.2.x
1368 args.extend(["-i", "-m"])
1369 elif vmaj >= 3: # 8.3.x or newer
1370 args.extend(["-i", "-a", "m"])
1371 result = utils.RunCmd(args)
1373 _ThrowError("drbd%d: can't attach local disk: %s", minor, result.output)
1376 def _AssembleNet(cls, minor, net_info, protocol,
1377 dual_pri=False, hmac=None, secret=None):
1378 """Configure the network part of the device.
1381 lhost, lport, rhost, rport = net_info
1382 if None in net_info:
1383 # we don't want network connection and actually want to make
1385 cls._ShutdownNet(minor)
1388 # Workaround for a race condition. When DRBD is doing its dance to
1389 # establish a connection with its peer, it also sends the
1390 # synchronization speed over the wire. In some cases setting the
1391 # sync speed only after setting up both sides can race with DRBD
1392 # connecting, hence we set it here before telling DRBD anything
1394 cls._SetMinorSyncSpeed(minor, constants.SYNC_SPEED)
1396 if netutils.IP6Address.IsValid(lhost):
1397 if not netutils.IP6Address.IsValid(rhost):
1398 _ThrowError("drbd%d: can't connect ip %s to ip %s" %
1399 (minor, lhost, rhost))
1401 elif netutils.IP4Address.IsValid(lhost):
1402 if not netutils.IP4Address.IsValid(rhost):
1403 _ThrowError("drbd%d: can't connect ip %s to ip %s" %
1404 (minor, lhost, rhost))
1407 _ThrowError("drbd%d: Invalid ip %s" % (minor, lhost))
1409 args = ["drbdsetup", cls._DevPath(minor), "net",
1410 "%s:%s:%s" % (family, lhost, lport),
1411 "%s:%s:%s" % (family, rhost, rport), protocol,
1412 "-A", "discard-zero-changes",
1419 args.extend(["-a", hmac, "-x", secret])
1420 result = utils.RunCmd(args)
1422 _ThrowError("drbd%d: can't setup network: %s - %s",
1423 minor, result.fail_reason, result.output)
1425 def _CheckNetworkConfig():
1426 info = cls._GetDevInfo(cls._GetShowData(minor))
1427 if not "local_addr" in info or not "remote_addr" in info:
1428 raise utils.RetryAgain()
1430 if (info["local_addr"] != (lhost, lport) or
1431 info["remote_addr"] != (rhost, rport)):
1432 raise utils.RetryAgain()
1435 utils.Retry(_CheckNetworkConfig, 1.0, 10.0)
1436 except utils.RetryTimeout:
1437 _ThrowError("drbd%d: timeout while configuring network", minor)
1439 def AddChildren(self, devices):
1440 """Add a disk to the DRBD device.
1443 if self.minor is None:
1444 _ThrowError("drbd%d: can't attach to dbrd8 during AddChildren",
1446 if len(devices) != 2:
1447 _ThrowError("drbd%d: need two devices for AddChildren", self.minor)
1448 info = self._GetDevInfo(self._GetShowData(self.minor))
1449 if "local_dev" in info:
1450 _ThrowError("drbd%d: already attached to a local disk", self.minor)
1451 backend, meta = devices
1452 if backend.dev_path is None or meta.dev_path is None:
1453 _ThrowError("drbd%d: children not ready during AddChildren", self.minor)
1456 self._CheckMetaSize(meta.dev_path)
1457 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1459 self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path, self.size)
1460 self._children = devices
1462 def RemoveChildren(self, devices):
1463 """Detach the drbd device from local storage.
1466 if self.minor is None:
1467 _ThrowError("drbd%d: can't attach to drbd8 during RemoveChildren",
1469 # early return if we don't actually have backing storage
1470 info = self._GetDevInfo(self._GetShowData(self.minor))
1471 if "local_dev" not in info:
1473 if len(self._children) != 2:
1474 _ThrowError("drbd%d: we don't have two children: %s", self.minor,
1476 if self._children.count(None) == 2: # we don't actually have children :)
1477 logging.warning("drbd%d: requested detach while detached", self.minor)
1479 if len(devices) != 2:
1480 _ThrowError("drbd%d: we need two children in RemoveChildren", self.minor)
1481 for child, dev in zip(self._children, devices):
1482 if dev != child.dev_path:
1483 _ThrowError("drbd%d: mismatch in local storage (%s != %s) in"
1484 " RemoveChildren", self.minor, dev, child.dev_path)
1486 self._ShutdownLocal(self.minor)
1490 def _SetMinorSyncSpeed(cls, minor, kbytes):
1491 """Set the speed of the DRBD syncer.
1493 This is the low-level implementation.
1496 @param minor: the drbd minor whose settings we change
1498 @param kbytes: the speed in kbytes/second
1500 @return: the success of the operation
1503 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "syncer",
1504 "-r", "%d" % kbytes, "--create-device"])
1506 logging.error("Can't change syncer rate: %s - %s",
1507 result.fail_reason, result.output)
1508 return not result.failed
1510 def SetSyncSpeed(self, kbytes):
1511 """Set the speed of the DRBD syncer.
1514 @param kbytes: the speed in kbytes/second
1516 @return: the success of the operation
1519 if self.minor is None:
1520 logging.info("Not attached during SetSyncSpeed")
1522 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1523 return self._SetMinorSyncSpeed(self.minor, kbytes) and children_result
1525 def PauseResumeSync(self, pause):
1526 """Pauses or resumes the sync of a DRBD device.
1528 @param pause: Wether to pause or resume
1529 @return: the success of the operation
1532 if self.minor is None:
1533 logging.info("Not attached during PauseSync")
1536 children_result = super(DRBD8, self).PauseResumeSync(pause)
1543 result = utils.RunCmd(["drbdsetup", self.dev_path, cmd])
1545 logging.error("Can't %s: %s - %s", cmd,
1546 result.fail_reason, result.output)
1547 return not result.failed and children_result
1549 def GetProcStatus(self):
1550 """Return device data from /proc.
1553 if self.minor is None:
1554 _ThrowError("drbd%d: GetStats() called while not attached", self._aminor)
1555 proc_info = self._MassageProcData(self._GetProcData())
1556 if self.minor not in proc_info:
1557 _ThrowError("drbd%d: can't find myself in /proc", self.minor)
1558 return DRBD8Status(proc_info[self.minor])
1560 def GetSyncStatus(self):
1561 """Returns the sync status of the device.
1564 If sync_percent is None, it means all is ok
1565 If estimated_time is None, it means we can't estimate
1566 the time needed, otherwise it's the time left in seconds.
1569 We set the is_degraded parameter to True on two conditions:
1570 network not connected or local disk missing.
1572 We compute the ldisk parameter based on whether we have a local
1575 @rtype: objects.BlockDevStatus
1578 if self.minor is None and not self.Attach():
1579 _ThrowError("drbd%d: can't Attach() in GetSyncStatus", self._aminor)
1581 stats = self.GetProcStatus()
1582 is_degraded = not stats.is_connected or not stats.is_disk_uptodate
1584 if stats.is_disk_uptodate:
1585 ldisk_status = constants.LDS_OKAY
1586 elif stats.is_diskless:
1587 ldisk_status = constants.LDS_FAULTY
1589 ldisk_status = constants.LDS_UNKNOWN
1591 return objects.BlockDevStatus(dev_path=self.dev_path,
1594 sync_percent=stats.sync_percent,
1595 estimated_time=stats.est_time,
1596 is_degraded=is_degraded,
1597 ldisk_status=ldisk_status)
1599 def Open(self, force=False):
1600 """Make the local state primary.
1602 If the 'force' parameter is given, the '-o' option is passed to
1603 drbdsetup. Since this is a potentially dangerous operation, the
1604 force flag should be only given after creation, when it actually
1608 if self.minor is None and not self.Attach():
1609 logging.error("DRBD cannot attach to a device during open")
1611 cmd = ["drbdsetup", self.dev_path, "primary"]
1614 result = utils.RunCmd(cmd)
1616 _ThrowError("drbd%d: can't make drbd device primary: %s", self.minor,
1620 """Make the local state secondary.
1622 This will, of course, fail if the device is in use.
1625 if self.minor is None and not self.Attach():
1626 _ThrowError("drbd%d: can't Attach() in Close()", self._aminor)
1627 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1629 _ThrowError("drbd%d: can't switch drbd device to secondary: %s",
1630 self.minor, result.output)
1632 def DisconnectNet(self):
1633 """Removes network configuration.
1635 This method shutdowns the network side of the device.
1637 The method will wait up to a hardcoded timeout for the device to
1638 go into standalone after the 'disconnect' command before
1639 re-configuring it, as sometimes it takes a while for the
1640 disconnect to actually propagate and thus we might issue a 'net'
1641 command while the device is still connected. If the device will
1642 still be attached to the network and we time out, we raise an
1646 if self.minor is None:
1647 _ThrowError("drbd%d: disk not attached in re-attach net", self._aminor)
1649 if None in (self._lhost, self._lport, self._rhost, self._rport):
1650 _ThrowError("drbd%d: DRBD disk missing network info in"
1651 " DisconnectNet()", self.minor)
1653 class _DisconnectStatus:
1654 def __init__(self, ever_disconnected):
1655 self.ever_disconnected = ever_disconnected
1657 dstatus = _DisconnectStatus(_IgnoreError(self._ShutdownNet, self.minor))
1659 def _WaitForDisconnect():
1660 if self.GetProcStatus().is_standalone:
1663 # retry the disconnect, it seems possible that due to a well-time
1664 # disconnect on the peer, my disconnect command might be ignored and
1666 dstatus.ever_disconnected = \
1667 _IgnoreError(self._ShutdownNet, self.minor) or dstatus.ever_disconnected
1669 raise utils.RetryAgain()
1672 start_time = time.time()
1675 # Start delay at 100 milliseconds and grow up to 2 seconds
1676 utils.Retry(_WaitForDisconnect, (0.1, 1.5, 2.0),
1677 self._NET_RECONFIG_TIMEOUT)
1678 except utils.RetryTimeout:
1679 if dstatus.ever_disconnected:
1680 msg = ("drbd%d: device did not react to the"
1681 " 'disconnect' command in a timely manner")
1683 msg = "drbd%d: can't shutdown network, even after multiple retries"
1685 _ThrowError(msg, self.minor)
1687 reconfig_time = time.time() - start_time
1688 if reconfig_time > (self._NET_RECONFIG_TIMEOUT * 0.25):
1689 logging.info("drbd%d: DisconnectNet: detach took %.3f seconds",
1690 self.minor, reconfig_time)
1692 def AttachNet(self, multimaster):
1693 """Reconnects the network.
1695 This method connects the network side of the device with a
1696 specified multi-master flag. The device needs to be 'Standalone'
1697 but have valid network configuration data.
1700 - multimaster: init the network in dual-primary mode
1703 if self.minor is None:
1704 _ThrowError("drbd%d: device not attached in AttachNet", self._aminor)
1706 if None in (self._lhost, self._lport, self._rhost, self._rport):
1707 _ThrowError("drbd%d: missing network info in AttachNet()", self.minor)
1709 status = self.GetProcStatus()
1711 if not status.is_standalone:
1712 _ThrowError("drbd%d: device is not standalone in AttachNet", self.minor)
1714 self._AssembleNet(self.minor,
1715 (self._lhost, self._lport, self._rhost, self._rport),
1716 constants.DRBD_NET_PROTOCOL, dual_pri=multimaster,
1717 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1720 """Check if our minor is configured.
1722 This doesn't do any device configurations - it only checks if the
1723 minor is in a state different from Unconfigured.
1725 Note that this function will not change the state of the system in
1726 any way (except in case of side-effects caused by reading from
1730 used_devs = self.GetUsedDevs()
1731 if self._aminor in used_devs:
1732 minor = self._aminor
1736 self._SetFromMinor(minor)
1737 return minor is not None
1740 """Assemble the drbd.
1743 - if we have a configured device, we try to ensure that it matches
1745 - if not, we create it from zero
1748 super(DRBD8, self).Assemble()
1751 if self.minor is None:
1752 # local device completely unconfigured
1753 self._FastAssemble()
1755 # we have to recheck the local and network status and try to fix
1757 self._SlowAssemble()
1759 def _SlowAssemble(self):
1760 """Assembles the DRBD device from a (partially) configured device.
1762 In case of partially attached (local device matches but no network
1763 setup), we perform the network attach. If successful, we re-test
1764 the attach if can return success.
1767 # TODO: Rewrite to not use a for loop just because there is 'break'
1768 # pylint: disable=W0631
1769 net_data = (self._lhost, self._lport, self._rhost, self._rport)
1770 for minor in (self._aminor,):
1771 info = self._GetDevInfo(self._GetShowData(minor))
1772 match_l = self._MatchesLocal(info)
1773 match_r = self._MatchesNet(info)
1775 if match_l and match_r:
1776 # everything matches
1779 if match_l and not match_r and "local_addr" not in info:
1780 # disk matches, but not attached to network, attach and recheck
1781 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1782 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1783 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1786 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1787 " show' disagrees", minor)
1789 if match_r and "local_dev" not in info:
1790 # no local disk, but network attached and it matches
1791 self._AssembleLocal(minor, self._children[0].dev_path,
1792 self._children[1].dev_path, self.size)
1793 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1796 _ThrowError("drbd%d: disk attach successful, but 'drbdsetup"
1797 " show' disagrees", minor)
1799 # this case must be considered only if we actually have local
1800 # storage, i.e. not in diskless mode, because all diskless
1801 # devices are equal from the point of view of local
1803 if (match_l and "local_dev" in info and
1804 not match_r and "local_addr" in info):
1805 # strange case - the device network part points to somewhere
1806 # else, even though its local storage is ours; as we own the
1807 # drbd space, we try to disconnect from the remote peer and
1808 # reconnect to our correct one
1810 self._ShutdownNet(minor)
1811 except errors.BlockDeviceError, err:
1812 _ThrowError("drbd%d: device has correct local storage, wrong"
1813 " remote peer and is unable to disconnect in order"
1814 " to attach to the correct peer: %s", minor, str(err))
1815 # note: _AssembleNet also handles the case when we don't want
1816 # local storage (i.e. one or more of the _[lr](host|port) is
1818 self._AssembleNet(minor, net_data, constants.DRBD_NET_PROTOCOL,
1819 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1820 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1823 _ThrowError("drbd%d: network attach successful, but 'drbdsetup"
1824 " show' disagrees", minor)
1829 self._SetFromMinor(minor)
1831 _ThrowError("drbd%d: cannot activate, unknown or unhandled reason",
1834 def _FastAssemble(self):
1835 """Assemble the drbd device from zero.
1837 This is run when in Assemble we detect our minor is unused.
1840 minor = self._aminor
1841 if self._children and self._children[0] and self._children[1]:
1842 self._AssembleLocal(minor, self._children[0].dev_path,
1843 self._children[1].dev_path, self.size)
1844 if self._lhost and self._lport and self._rhost and self._rport:
1845 self._AssembleNet(minor,
1846 (self._lhost, self._lport, self._rhost, self._rport),
1847 constants.DRBD_NET_PROTOCOL,
1848 hmac=constants.DRBD_HMAC_ALG, secret=self._secret)
1849 self._SetFromMinor(minor)
1852 def _ShutdownLocal(cls, minor):
1853 """Detach from the local device.
1855 I/Os will continue to be served from the remote device. If we
1856 don't have a remote device, this operation will fail.
1859 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1861 _ThrowError("drbd%d: can't detach local disk: %s", minor, result.output)
1864 def _ShutdownNet(cls, minor):
1865 """Disconnect from the remote peer.
1867 This fails if we don't have a local device.
1870 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1872 _ThrowError("drbd%d: can't shutdown network: %s", minor, result.output)
1875 def _ShutdownAll(cls, minor):
1876 """Deactivate the device.
1878 This will, of course, fail if the device is in use.
1881 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1883 _ThrowError("drbd%d: can't shutdown drbd device: %s",
1884 minor, result.output)
1887 """Shutdown the DRBD device.
1890 if self.minor is None and not self.Attach():
1891 logging.info("drbd%d: not attached during Shutdown()", self._aminor)
1895 self.dev_path = None
1896 self._ShutdownAll(minor)
1899 """Stub remove for DRBD devices.
1905 def Create(cls, unique_id, children, size):
1906 """Create a new DRBD8 device.
1908 Since DRBD devices are not created per se, just assembled, this
1909 function only initializes the metadata.
1912 if len(children) != 2:
1913 raise errors.ProgrammerError("Invalid setup for the drbd device")
1914 # check that the minor is unused
1915 aminor = unique_id[4]
1916 proc_info = cls._MassageProcData(cls._GetProcData())
1917 if aminor in proc_info:
1918 status = DRBD8Status(proc_info[aminor])
1919 in_use = status.is_in_use
1923 _ThrowError("drbd%d: minor is already in use at Create() time", aminor)
1926 if not meta.Attach():
1927 _ThrowError("drbd%d: can't attach to meta device '%s'",
1929 cls._CheckMetaSize(meta.dev_path)
1930 cls._InitMeta(aminor, meta.dev_path)
1931 return cls(unique_id, children, size)
1933 def Grow(self, amount, dryrun):
1934 """Resize the DRBD device and its backing storage.
1937 if self.minor is None:
1938 _ThrowError("drbd%d: Grow called while not attached", self._aminor)
1939 if len(self._children) != 2 or None in self._children:
1940 _ThrowError("drbd%d: cannot grow diskless device", self.minor)
1941 self._children[0].Grow(amount, dryrun)
1943 # DRBD does not support dry-run mode, so we'll return here
1945 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize", "-s",
1946 "%dm" % (self.size + amount)])
1948 _ThrowError("drbd%d: resize failed: %s", self.minor, result.output)
1951 class FileStorage(BlockDev):
1954 This class represents the a file storage backend device.
1956 The unique_id for the file device is a (file_driver, file_path) tuple.
1959 def __init__(self, unique_id, children, size):
1960 """Initalizes a file device backend.
1964 raise errors.BlockDeviceError("Invalid setup for file device")
1965 super(FileStorage, self).__init__(unique_id, children, size)
1966 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1967 raise ValueError("Invalid configuration data %s" % str(unique_id))
1968 self.driver = unique_id[0]
1969 self.dev_path = unique_id[1]
1973 """Assemble the device.
1975 Checks whether the file device exists, raises BlockDeviceError otherwise.
1978 if not os.path.exists(self.dev_path):
1979 _ThrowError("File device '%s' does not exist" % self.dev_path)
1982 """Shutdown the device.
1984 This is a no-op for the file type, as we don't deactivate
1985 the file on shutdown.
1990 def Open(self, force=False):
1991 """Make the device ready for I/O.
1993 This is a no-op for the file type.
1999 """Notifies that the device will no longer be used for I/O.
2001 This is a no-op for the file type.
2007 """Remove the file backing the block device.
2010 @return: True if the removal was successful
2014 os.remove(self.dev_path)
2015 except OSError, err:
2016 if err.errno != errno.ENOENT:
2017 _ThrowError("Can't remove file '%s': %s", self.dev_path, err)
2019 def Rename(self, new_id):
2020 """Renames the file.
2023 # TODO: implement rename for file-based storage
2024 _ThrowError("Rename is not supported for file-based storage")
2026 def Grow(self, amount, dryrun):
2029 @param amount: the amount (in mebibytes) to grow with
2032 # Check that the file exists
2034 current_size = self.GetActualSize()
2035 new_size = current_size + amount * 1024 * 1024
2036 assert new_size > current_size, "Cannot Grow with a negative amount"
2037 # We can't really simulate the growth
2041 f = open(self.dev_path, "a+")
2042 f.truncate(new_size)
2044 except EnvironmentError, err:
2045 _ThrowError("Error in file growth: %", str(err))
2048 """Attach to an existing file.
2050 Check if this file already exists.
2053 @return: True if file exists
2056 self.attached = os.path.exists(self.dev_path)
2057 return self.attached
2059 def GetActualSize(self):
2060 """Return the actual disk size.
2062 @note: the device needs to be active when this is called
2065 assert self.attached, "BlockDevice not attached in GetActualSize()"
2067 st = os.stat(self.dev_path)
2069 except OSError, err:
2070 _ThrowError("Can't stat %s: %s", self.dev_path, err)
2073 def Create(cls, unique_id, children, size):
2074 """Create a new file.
2076 @param size: the size of file in MiB
2078 @rtype: L{bdev.FileStorage}
2079 @return: an instance of FileStorage
2082 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
2083 raise ValueError("Invalid configuration data %s" % str(unique_id))
2084 dev_path = unique_id[1]
2086 fd = os.open(dev_path, os.O_RDWR | os.O_CREAT | os.O_EXCL)
2087 f = os.fdopen(fd, "w")
2088 f.truncate(size * 1024 * 1024)
2090 except EnvironmentError, err:
2091 if err.errno == errno.EEXIST:
2092 _ThrowError("File already existing: %s", dev_path)
2093 _ThrowError("Error in file creation: %", str(err))
2095 return FileStorage(unique_id, children, size)
2098 class PersistentBlockDevice(BlockDev):
2099 """A block device with persistent node
2101 May be either directly attached, or exposed through DM (e.g. dm-multipath).
2102 udev helpers are probably required to give persistent, human-friendly
2105 For the time being, pathnames are required to lie under /dev.
2108 def __init__(self, unique_id, children, size):
2109 """Attaches to a static block device.
2111 The unique_id is a path under /dev.
2114 super(PersistentBlockDevice, self).__init__(unique_id, children, size)
2115 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
2116 raise ValueError("Invalid configuration data %s" % str(unique_id))
2117 self.dev_path = unique_id[1]
2118 if not os.path.realpath(self.dev_path).startswith("/dev/"):
2119 raise ValueError("Full path '%s' lies outside /dev" %
2120 os.path.realpath(self.dev_path))
2121 # TODO: this is just a safety guard checking that we only deal with devices
2122 # we know how to handle. In the future this will be integrated with
2123 # external storage backends and possible values will probably be collected
2124 # from the cluster configuration.
2125 if unique_id[0] != constants.BLOCKDEV_DRIVER_MANUAL:
2126 raise ValueError("Got persistent block device of invalid type: %s" %
2129 self.major = self.minor = None
2133 def Create(cls, unique_id, children, size):
2134 """Create a new device
2136 This is a noop, we only return a PersistentBlockDevice instance
2139 return PersistentBlockDevice(unique_id, children, 0)
2149 def Rename(self, new_id):
2150 """Rename this device.
2153 _ThrowError("Rename is not supported for PersistentBlockDev storage")
2156 """Attach to an existing block device.
2160 self.attached = False
2162 st = os.stat(self.dev_path)
2163 except OSError, err:
2164 logging.error("Error stat()'ing %s: %s", self.dev_path, str(err))
2167 if not stat.S_ISBLK(st.st_mode):
2168 logging.error("%s is not a block device", self.dev_path)
2171 self.major = os.major(st.st_rdev)
2172 self.minor = os.minor(st.st_rdev)
2173 self.attached = True
2178 """Assemble the device.
2184 """Shutdown the device.
2189 def Open(self, force=False):
2190 """Make the device ready for I/O.
2196 """Notifies that the device will no longer be used for I/O.
2201 def Grow(self, amount, dryrun):
2202 """Grow the logical volume.
2205 _ThrowError("Grow is not supported for PersistentBlockDev storage")
2209 constants.LD_LV: LogicalVolume,
2210 constants.LD_DRBD8: DRBD8,
2211 constants.LD_BLOCKDEV: PersistentBlockDevice,
2214 if constants.ENABLE_FILE_STORAGE or constants.ENABLE_SHARED_FILE_STORAGE:
2215 DEV_MAP[constants.LD_FILE] = FileStorage
2218 def FindDevice(dev_type, unique_id, children, size):
2219 """Search for an existing, assembled device.
2221 This will succeed only if the device exists and is assembled, but it
2222 does not do any actions in order to activate the device.
2225 if dev_type not in DEV_MAP:
2226 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2227 device = DEV_MAP[dev_type](unique_id, children, size)
2228 if not device.attached:
2233 def Assemble(dev_type, unique_id, children, size):
2234 """Try to attach or assemble an existing device.
2236 This will attach to assemble the device, as needed, to bring it
2237 fully up. It must be safe to run on already-assembled devices.
2240 if dev_type not in DEV_MAP:
2241 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2242 device = DEV_MAP[dev_type](unique_id, children, size)
2247 def Create(dev_type, unique_id, children, size):
2251 if dev_type not in DEV_MAP:
2252 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
2253 device = DEV_MAP[dev_type].Create(unique_id, children, size)