4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Block device abstraction"""
27 import pyparsing as pyp
31 from ganeti import utils
32 from ganeti import errors
33 from ganeti import constants
36 class BlockDev(object):
37 """Block device abstract class.
39 A block device can be in the following states:
40 - not existing on the system, and by `Create()` it goes into:
41 - existing but not setup/not active, and by `Assemble()` goes into:
42 - active read-write and by `Open()` it goes into
43 - online (=used, or ready for use)
45 A device can also be online but read-only, however we are not using
46 the readonly state (LV has it, if needed in the future) and we are
47 usually looking at this like at a stack, so it's easier to
48 conceptualise the transition from not-existing to online and back
51 The many different states of the device are due to the fact that we
52 need to cover many device types:
53 - logical volumes are created, lvchange -a y $lv, and used
54 - drbd devices are attached to a local disk/remote peer and made primary
56 A block device is identified by three items:
57 - the /dev path of the device (dynamic)
58 - a unique ID of the device (static)
59 - it's major/minor pair (dynamic)
61 Not all devices implement both the first two as distinct items. LVM
62 logical volumes have their unique ID (the pair volume group, logical
63 volume name) in a 1-to-1 relation to the dev path. For DRBD devices,
64 the /dev path is again dynamic and the unique id is the pair (host1,
67 You can get to a device in two ways:
68 - creating the (real) device, which returns you
69 an attached instance (lvcreate)
70 - attaching of a python instance to an existing (real) device
72 The second point, the attachement to a device, is different
73 depending on whether the device is assembled or not. At init() time,
74 we search for a device with the same unique_id as us. If found,
75 good. It also means that the device is already assembled. If not,
76 after assembly we'll have our correct major/minor.
79 def __init__(self, unique_id, children):
80 self._children = children
82 self.unique_id = unique_id
88 """Assemble the device from its components.
90 Implementations of this method by child classes must ensure that:
91 - after the device has been assembled, it knows its major/minor
92 numbers; this allows other devices (usually parents) to probe
93 correctly for their children
94 - calling this method on an existing, in-use device is safe
95 - if the device is already configured (and in an OK state),
96 this method is idempotent
102 """Find a device which matches our config and attach to it.
105 raise NotImplementedError
108 """Notifies that the device will no longer be used for I/O.
111 raise NotImplementedError
114 def Create(cls, unique_id, children, size):
115 """Create the device.
117 If the device cannot be created, it will return None
118 instead. Error messages go to the logging system.
120 Note that for some devices, the unique_id is used, and for other,
121 the children. The idea is that these two, taken together, are
122 enough for both creation and assembly (later).
125 raise NotImplementedError
128 """Remove this device.
130 This makes sense only for some of the device types: LV and file
131 storeage. Also note that if the device can't attach, the removal
135 raise NotImplementedError
137 def Rename(self, new_id):
138 """Rename this device.
140 This may or may not make sense for a given device type.
143 raise NotImplementedError
145 def Open(self, force=False):
146 """Make the device ready for use.
148 This makes the device ready for I/O. For now, just the DRBD
151 The force parameter signifies that if the device has any kind of
152 --force thing, it should be used, we know what we are doing.
155 raise NotImplementedError
158 """Shut down the device, freeing its children.
160 This undoes the `Assemble()` work, except for the child
161 assembling; as such, the children on the device are still
162 assembled after this call.
165 raise NotImplementedError
167 def SetSyncSpeed(self, speed):
168 """Adjust the sync speed of the mirror.
170 In case this is not a mirroring device, this is no-op.
175 for child in self._children:
176 result = result and child.SetSyncSpeed(speed)
179 def GetSyncStatus(self):
180 """Returns the sync status of the device.
182 If this device is a mirroring device, this function returns the
183 status of the mirror.
185 If sync_percent is None, it means the device is not syncing.
187 If estimated_time is None, it means we can't estimate
188 the time needed, otherwise it's the time left in seconds.
190 If is_degraded is True, it means the device is missing
191 redundancy. This is usually a sign that something went wrong in
192 the device setup, if sync_percent is None.
194 The ldisk parameter represents the degradation of the local
195 data. This is only valid for some devices, the rest will always
196 return False (not degraded).
199 @return: (sync_percent, estimated_time, is_degraded, ldisk)
202 return None, None, False, False
205 def CombinedSyncStatus(self):
206 """Calculate the mirror status recursively for our children.
208 The return value is the same as for `GetSyncStatus()` except the
209 minimum percent and maximum time are calculated across our
213 min_percent, max_time, is_degraded, ldisk = self.GetSyncStatus()
215 for child in self._children:
216 c_percent, c_time, c_degraded, c_ldisk = child.GetSyncStatus()
217 if min_percent is None:
218 min_percent = c_percent
219 elif c_percent is not None:
220 min_percent = min(min_percent, c_percent)
223 elif c_time is not None:
224 max_time = max(max_time, c_time)
225 is_degraded = is_degraded or c_degraded
226 ldisk = ldisk or c_ldisk
227 return min_percent, max_time, is_degraded, ldisk
230 def SetInfo(self, text):
231 """Update metadata with info text.
233 Only supported for some device types.
236 for child in self._children:
239 def Grow(self, amount):
240 """Grow the block device.
242 @param amount: the amount (in mebibytes) to grow with
245 raise NotImplementedError
248 return ("<%s: unique_id: %s, children: %s, %s:%s, %s>" %
249 (self.__class__, self.unique_id, self._children,
250 self.major, self.minor, self.dev_path))
253 class LogicalVolume(BlockDev):
254 """Logical Volume block device.
257 def __init__(self, unique_id, children):
258 """Attaches to a LV device.
260 The unique_id is a tuple (vg_name, lv_name)
263 super(LogicalVolume, self).__init__(unique_id, children)
264 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
265 raise ValueError("Invalid configuration data %s" % str(unique_id))
266 self._vg_name, self._lv_name = unique_id
267 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
268 self._degraded = True
269 self.major = self.minor = None
273 def Create(cls, unique_id, children, size):
274 """Create a new logical volume.
277 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
278 raise ValueError("Invalid configuration data %s" % str(unique_id))
279 vg_name, lv_name = unique_id
280 pvs_info = cls.GetPVInfo(vg_name)
282 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
287 pvlist = [ pv[1] for pv in pvs_info ]
288 free_size = sum([ pv[0] for pv in pvs_info ])
290 # The size constraint should have been checked from the master before
291 # calling the create function.
293 raise errors.BlockDeviceError("Not enough free space: required %s,"
294 " available %s" % (size, free_size))
295 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-n%s" % lv_name,
298 raise errors.BlockDeviceError("%s - %s" % (result.fail_reason,
300 return LogicalVolume(unique_id, children)
303 def GetPVInfo(vg_name):
304 """Get the free space info for PVs in a volume group.
306 @param vg_name: the volume group name
309 @return: list of tuples (free_space, name) with free_space in mebibytes
312 command = ["pvs", "--noheadings", "--nosuffix", "--units=m",
313 "-opv_name,vg_name,pv_free,pv_attr", "--unbuffered",
315 result = utils.RunCmd(command)
317 logging.error("Can't get the PV information: %s - %s",
318 result.fail_reason, result.output)
321 for line in result.stdout.splitlines():
322 fields = line.strip().split(':')
324 logging.error("Can't parse pvs output: line '%s'", line)
326 # skip over pvs from another vg or ones which are not allocatable
327 if fields[1] != vg_name or fields[3][0] != 'a':
329 data.append((float(fields[2]), fields[0]))
334 """Remove this logical volume.
337 if not self.minor and not self.Attach():
338 # the LV does not exist
340 result = utils.RunCmd(["lvremove", "-f", "%s/%s" %
341 (self._vg_name, self._lv_name)])
343 logging.error("Can't lvremove: %s - %s",
344 result.fail_reason, result.output)
346 return not result.failed
348 def Rename(self, new_id):
349 """Rename this logical volume.
352 if not isinstance(new_id, (tuple, list)) or len(new_id) != 2:
353 raise errors.ProgrammerError("Invalid new logical id '%s'" % new_id)
354 new_vg, new_name = new_id
355 if new_vg != self._vg_name:
356 raise errors.ProgrammerError("Can't move a logical volume across"
357 " volume groups (from %s to to %s)" %
358 (self._vg_name, new_vg))
359 result = utils.RunCmd(["lvrename", new_vg, self._lv_name, new_name])
361 raise errors.BlockDeviceError("Failed to rename the logical volume: %s" %
363 self._lv_name = new_name
364 self.dev_path = "/dev/%s/%s" % (self._vg_name, self._lv_name)
367 """Attach to an existing LV.
369 This method will try to see if an existing and active LV exists
370 which matches our name. If so, its major/minor will be
374 self.attached = False
375 result = utils.RunCmd(["lvs", "--noheadings", "--separator=,",
376 "-olv_attr,lv_kernel_major,lv_kernel_minor",
379 logging.error("Can't find LV %s: %s, %s",
380 self.dev_path, result.fail_reason, result.output)
382 out = result.stdout.strip().rstrip(',')
385 logging.error("Can't parse LVS output, len(%s) != 3", str(out))
388 status, major, minor = out[:3]
390 logging.error("lvs lv_attr is not 6 characters (%s)", status)
396 except ValueError, err:
397 logging.error("lvs major/minor cannot be parsed: %s", str(err))
401 self._degraded = status[0] == 'v' # virtual volume, i.e. doesn't backing
407 """Assemble the device.
409 We alway run `lvchange -ay` on the LV to ensure it's active before
410 use, as there were cases when xenvg was not active after boot
411 (also possibly after disk issues).
414 result = utils.RunCmd(["lvchange", "-ay", self.dev_path])
416 logging.error("Can't activate lv %s: %s", self.dev_path, result.output)
421 """Shutdown the device.
423 This is a no-op for the LV device type, as we don't deactivate the
429 def GetSyncStatus(self):
430 """Returns the sync status of the device.
432 If this device is a mirroring device, this function returns the
433 status of the mirror.
435 For logical volumes, sync_percent and estimated_time are always
436 None (no recovery in progress, as we don't handle the mirrored LV
437 case). The is_degraded parameter is the inverse of the ldisk
440 For the ldisk parameter, we check if the logical volume has the
441 'virtual' type, which means it's not backed by existing storage
442 anymore (read from it return I/O error). This happens after a
443 physical disk failure and subsequent 'vgreduce --removemissing' on
446 The status was already read in Attach, so we just return it.
449 @return: (sync_percent, estimated_time, is_degraded, ldisk)
452 return None, None, self._degraded, self._degraded
454 def Open(self, force=False):
455 """Make the device ready for I/O.
457 This is a no-op for the LV device type.
463 """Notifies that the device will no longer be used for I/O.
465 This is a no-op for the LV device type.
470 def Snapshot(self, size):
471 """Create a snapshot copy of an lvm block device.
474 snap_name = self._lv_name + ".snap"
476 # remove existing snapshot if found
477 snap = LogicalVolume((self._vg_name, snap_name), None)
480 pvs_info = self.GetPVInfo(self._vg_name)
482 raise errors.BlockDeviceError("Can't compute PV info for vg %s" %
486 free_size, pv_name = pvs_info[0]
488 raise errors.BlockDeviceError("Not enough free space: required %s,"
489 " available %s" % (size, free_size))
491 result = utils.RunCmd(["lvcreate", "-L%dm" % size, "-s",
492 "-n%s" % snap_name, self.dev_path])
494 raise errors.BlockDeviceError("command: %s error: %s - %s" %
495 (result.cmd, result.fail_reason,
500 def SetInfo(self, text):
501 """Update metadata with info text.
504 BlockDev.SetInfo(self, text)
506 # Replace invalid characters
507 text = re.sub('^[^A-Za-z0-9_+.]', '_', text)
508 text = re.sub('[^-A-Za-z0-9_+.]', '_', text)
510 # Only up to 128 characters are allowed
513 result = utils.RunCmd(["lvchange", "--addtag", text,
516 raise errors.BlockDeviceError("Command: %s error: %s - %s" %
517 (result.cmd, result.fail_reason,
519 def Grow(self, amount):
520 """Grow the logical volume.
523 # we try multiple algorithms since the 'best' ones might not have
524 # space available in the right place, but later ones might (since
525 # they have less constraints); also note that only recent LVM
527 for alloc_policy in "contiguous", "cling", "normal":
528 result = utils.RunCmd(["lvextend", "--alloc", alloc_policy,
529 "-L", "+%dm" % amount, self.dev_path])
530 if not result.failed:
532 raise errors.BlockDeviceError("Can't grow LV %s: %s" %
533 (self.dev_path, result.output))
536 class DRBD8Status(object):
537 """A DRBD status representation class.
539 Note that this doesn't support unconfigured devices (cs:Unconfigured).
542 LINE_RE = re.compile(r"\s*[0-9]+:\s*cs:(\S+)\s+st:([^/]+)/(\S+)"
543 "\s+ds:([^/]+)/(\S+)\s+.*$")
544 SYNC_RE = re.compile(r"^.*\ssync'ed:\s*([0-9.]+)%.*"
545 "\sfinish: ([0-9]+):([0-9]+):([0-9]+)\s.*$")
547 def __init__(self, procline):
548 m = self.LINE_RE.match(procline)
550 raise errors.BlockDeviceError("Can't parse input data '%s'" % procline)
551 self.cstatus = m.group(1)
552 self.lrole = m.group(2)
553 self.rrole = m.group(3)
554 self.ldisk = m.group(4)
555 self.rdisk = m.group(5)
557 self.is_standalone = self.cstatus == "StandAlone"
558 self.is_wfconn = self.cstatus == "WFConnection"
559 self.is_connected = self.cstatus == "Connected"
560 self.is_primary = self.lrole == "Primary"
561 self.is_secondary = self.lrole == "Secondary"
562 self.peer_primary = self.rrole == "Primary"
563 self.peer_secondary = self.rrole == "Secondary"
564 self.both_primary = self.is_primary and self.peer_primary
565 self.both_secondary = self.is_secondary and self.peer_secondary
567 self.is_diskless = self.ldisk == "Diskless"
568 self.is_disk_uptodate = self.ldisk == "UpToDate"
570 m = self.SYNC_RE.match(procline)
572 self.sync_percent = float(m.group(1))
573 hours = int(m.group(2))
574 minutes = int(m.group(3))
575 seconds = int(m.group(4))
576 self.est_time = hours * 3600 + minutes * 60 + seconds
578 self.sync_percent = None
581 self.is_sync_target = self.peer_sync_source = self.cstatus == "SyncTarget"
582 self.peer_sync_target = self.is_sync_source = self.cstatus == "SyncSource"
583 self.is_resync = self.is_sync_target or self.is_sync_source
586 class BaseDRBD(BlockDev):
589 This class contains a few bits of common functionality between the
590 0.7 and 8.x versions of DRBD.
593 _VERSION_RE = re.compile(r"^version: (\d+)\.(\d+)\.(\d+)"
594 r" \(api:(\d+)/proto:(\d+)(?:-(\d+))?\)")
597 _ST_UNCONFIGURED = "Unconfigured"
598 _ST_WFCONNECTION = "WFConnection"
599 _ST_CONNECTED = "Connected"
601 _STATUS_FILE = "/proc/drbd"
604 def _GetProcData(filename=_STATUS_FILE):
605 """Return data from /proc/drbd.
608 stat = open(filename, "r")
610 data = stat.read().splitlines()
614 raise errors.BlockDeviceError("Can't read any data from %s" % filename)
618 def _MassageProcData(data):
619 """Transform the output of _GetProdData into a nicer form.
621 @return: a dictionary of minor: joined lines from /proc/drbd
625 lmatch = re.compile("^ *([0-9]+):.*$")
627 old_minor = old_line = None
629 lresult = lmatch.match(line)
630 if lresult is not None:
631 if old_minor is not None:
632 results[old_minor] = old_line
633 old_minor = int(lresult.group(1))
636 if old_minor is not None:
637 old_line += " " + line.strip()
639 if old_minor is not None:
640 results[old_minor] = old_line
644 def _GetVersion(cls):
645 """Return the DRBD version.
647 This will return a dict with keys:
653 - proto2 (only on drbd > 8.2.X)
656 proc_data = cls._GetProcData()
657 first_line = proc_data[0].strip()
658 version = cls._VERSION_RE.match(first_line)
660 raise errors.BlockDeviceError("Can't parse DRBD version from '%s'" %
663 values = version.groups()
664 retval = {'k_major': int(values[0]),
665 'k_minor': int(values[1]),
666 'k_point': int(values[2]),
667 'api': int(values[3]),
668 'proto': int(values[4]),
670 if values[5] is not None:
671 retval['proto2'] = values[5]
677 """Return the path to a drbd device for a given minor.
680 return "/dev/drbd%d" % minor
683 def _GetUsedDevs(cls):
684 """Compute the list of used DRBD devices.
687 data = cls._GetProcData()
690 valid_line = re.compile("^ *([0-9]+): cs:([^ ]+).*$")
692 match = valid_line.match(line)
695 minor = int(match.group(1))
696 state = match.group(2)
697 if state == cls._ST_UNCONFIGURED:
699 used_devs[minor] = state, line
703 def _SetFromMinor(self, minor):
704 """Set our parameters based on the given minor.
706 This sets our minor variable and our dev_path.
710 self.minor = self.dev_path = None
711 self.attached = False
714 self.dev_path = self._DevPath(minor)
718 def _CheckMetaSize(meta_device):
719 """Check if the given meta device looks like a valid one.
721 This currently only check the size, which must be around
725 result = utils.RunCmd(["blockdev", "--getsize", meta_device])
727 logging.error("Failed to get device size: %s - %s",
728 result.fail_reason, result.output)
731 sectors = int(result.stdout)
733 logging.error("Invalid output from blockdev: '%s'", result.stdout)
735 bytes = sectors * 512
736 if bytes < 128 * 1024 * 1024: # less than 128MiB
737 logging.error("Meta device too small (%.2fMib)", (bytes / 1024 / 1024))
739 if bytes > (128 + 32) * 1024 * 1024: # account for an extra (big) PE on LVM
740 logging.error("Meta device too big (%.2fMiB)", (bytes / 1024 / 1024))
744 def Rename(self, new_id):
747 This is not supported for drbd devices.
750 raise errors.ProgrammerError("Can't rename a drbd device")
753 class DRBD8(BaseDRBD):
754 """DRBD v8.x block device.
756 This implements the local host part of the DRBD device, i.e. it
757 doesn't do anything to the supposed peer. If you need a fully
758 connected DRBD pair, you need to use this class on both hosts.
760 The unique_id for the drbd device is the (local_ip, local_port,
761 remote_ip, remote_port) tuple, and it must have two children: the
762 data device and the meta_device. The meta device is checked for
763 valid size and is zeroed on create.
770 _NET_RECONFIG_TIMEOUT = 60
772 def __init__(self, unique_id, children):
773 if children and children.count(None) > 0:
775 super(DRBD8, self).__init__(unique_id, children)
776 self.major = self._DRBD_MAJOR
777 version = self._GetVersion()
778 if version['k_major'] != 8 :
779 raise errors.BlockDeviceError("Mismatch in DRBD kernel version and"
780 " requested ganeti usage: kernel is"
781 " %s.%s, ganeti wants 8.x" %
782 (version['k_major'], version['k_minor']))
784 if len(children) not in (0, 2):
785 raise ValueError("Invalid configuration data %s" % str(children))
786 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 6:
787 raise ValueError("Invalid configuration data %s" % str(unique_id))
788 (self._lhost, self._lport,
789 self._rhost, self._rport,
790 self._aminor, self._secret) = unique_id
791 if (self._lhost is not None and self._lhost == self._rhost and
792 self._lport == self._rport):
793 raise ValueError("Invalid configuration data, same local/remote %s" %
798 def _InitMeta(cls, minor, dev_path):
799 """Initialize a meta device.
801 This will not work if the given minor is in use.
804 result = utils.RunCmd(["drbdmeta", "--force", cls._DevPath(minor),
805 "v08", dev_path, "0", "create-md"])
807 raise errors.BlockDeviceError("Can't initialize meta device: %s" %
811 def _FindUnusedMinor(cls):
812 """Find an unused DRBD device.
814 This is specific to 8.x as the minors are allocated dynamically,
815 so non-existing numbers up to a max minor count are actually free.
818 data = cls._GetProcData()
820 unused_line = re.compile("^ *([0-9]+): cs:Unconfigured$")
821 used_line = re.compile("^ *([0-9]+): cs:")
824 match = unused_line.match(line)
826 return int(match.group(1))
827 match = used_line.match(line)
829 minor = int(match.group(1))
830 highest = max(highest, minor)
831 if highest is None: # there are no minors in use at all
833 if highest >= cls._MAX_MINORS:
834 logging.error("Error: no free drbd minors!")
835 raise errors.BlockDeviceError("Can't find a free DRBD minor")
839 def _IsValidMeta(cls, meta_device):
840 """Check if the given meta device looks like a valid one.
843 minor = cls._FindUnusedMinor()
844 minor_path = cls._DevPath(minor)
845 result = utils.RunCmd(["drbdmeta", minor_path,
846 "v08", meta_device, "0",
849 logging.error("Invalid meta device %s: %s", meta_device, result.output)
854 def _GetShowParser(cls):
855 """Return a parser for `drbd show` output.
857 This will either create or return an already-create parser for the
858 output of the command `drbd show`.
861 if cls._PARSE_SHOW is not None:
862 return cls._PARSE_SHOW
865 lbrace = pyp.Literal("{").suppress()
866 rbrace = pyp.Literal("}").suppress()
867 semi = pyp.Literal(";").suppress()
868 # this also converts the value to an int
869 number = pyp.Word(pyp.nums).setParseAction(lambda s, l, t: int(t[0]))
871 comment = pyp.Literal ("#") + pyp.Optional(pyp.restOfLine)
872 defa = pyp.Literal("_is_default").suppress()
873 dbl_quote = pyp.Literal('"').suppress()
875 keyword = pyp.Word(pyp.alphanums + '-')
878 value = pyp.Word(pyp.alphanums + '_-/.:')
879 quoted = dbl_quote + pyp.CharsNotIn('"') + dbl_quote
880 addr_port = (pyp.Word(pyp.nums + '.') + pyp.Literal(':').suppress() +
882 # meta device, extended syntax
883 meta_value = ((value ^ quoted) + pyp.Literal('[').suppress() +
884 number + pyp.Word(']').suppress())
887 stmt = (~rbrace + keyword + ~lbrace +
888 pyp.Optional(addr_port ^ value ^ quoted ^ meta_value) +
889 pyp.Optional(defa) + semi +
890 pyp.Optional(pyp.restOfLine).suppress())
893 section_name = pyp.Word(pyp.alphas + '_')
894 section = section_name + lbrace + pyp.ZeroOrMore(pyp.Group(stmt)) + rbrace
896 bnf = pyp.ZeroOrMore(pyp.Group(section ^ stmt))
899 cls._PARSE_SHOW = bnf
904 def _GetShowData(cls, minor):
905 """Return the `drbdsetup show` data for a minor.
908 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "show"])
910 logging.error("Can't display the drbd config: %s - %s",
911 result.fail_reason, result.output)
916 def _GetDevInfo(cls, out):
917 """Parse details about a given DRBD minor.
919 This return, if available, the local backing device (as a path)
920 and the local and remote (ip, port) information from a string
921 containing the output of the `drbdsetup show` command as returned
929 bnf = cls._GetShowParser()
933 results = bnf.parseString(out)
934 except pyp.ParseException, err:
935 raise errors.BlockDeviceError("Can't parse drbdsetup show output: %s" %
938 # and massage the results into our desired format
939 for section in results:
941 if sname == "_this_host":
942 for lst in section[1:]:
944 data["local_dev"] = lst[1]
945 elif lst[0] == "meta-disk":
946 data["meta_dev"] = lst[1]
947 data["meta_index"] = lst[2]
948 elif lst[0] == "address":
949 data["local_addr"] = tuple(lst[1:])
950 elif sname == "_remote_host":
951 for lst in section[1:]:
952 if lst[0] == "address":
953 data["remote_addr"] = tuple(lst[1:])
956 def _MatchesLocal(self, info):
957 """Test if our local config matches with an existing device.
959 The parameter should be as returned from `_GetDevInfo()`. This
960 method tests if our local backing device is the same as the one in
961 the info parameter, in effect testing if we look like the given
966 backend, meta = self._children
968 backend = meta = None
970 if backend is not None:
971 retval = ("local_dev" in info and info["local_dev"] == backend.dev_path)
973 retval = ("local_dev" not in info)
976 retval = retval and ("meta_dev" in info and
977 info["meta_dev"] == meta.dev_path)
978 retval = retval and ("meta_index" in info and
979 info["meta_index"] == 0)
981 retval = retval and ("meta_dev" not in info and
982 "meta_index" not in info)
985 def _MatchesNet(self, info):
986 """Test if our network config matches with an existing device.
988 The parameter should be as returned from `_GetDevInfo()`. This
989 method tests if our network configuration is the same as the one
990 in the info parameter, in effect testing if we look like the given
994 if (((self._lhost is None and not ("local_addr" in info)) and
995 (self._rhost is None and not ("remote_addr" in info)))):
998 if self._lhost is None:
1001 if not ("local_addr" in info and
1002 "remote_addr" in info):
1005 retval = (info["local_addr"] == (self._lhost, self._lport))
1006 retval = (retval and
1007 info["remote_addr"] == (self._rhost, self._rport))
1011 def _AssembleLocal(cls, minor, backend, meta):
1012 """Configure the local part of a DRBD device.
1014 This is the first thing that must be done on an unconfigured DRBD
1015 device. And it must be done only once.
1018 if not cls._IsValidMeta(meta):
1020 args = ["drbdsetup", cls._DevPath(minor), "disk",
1021 backend, meta, "0", "-e", "detach", "--create-device"]
1022 result = utils.RunCmd(args)
1024 logging.error("Can't attach local disk: %s", result.output)
1025 return not result.failed
1028 def _AssembleNet(cls, minor, net_info, protocol,
1029 dual_pri=False, hmac=None, secret=None):
1030 """Configure the network part of the device.
1033 lhost, lport, rhost, rport = net_info
1034 if None in net_info:
1035 # we don't want network connection and actually want to make
1037 return cls._ShutdownNet(minor)
1039 # Workaround for a race condition. When DRBD is doing its dance to
1040 # establish a connection with its peer, it also sends the
1041 # synchronization speed over the wire. In some cases setting the
1042 # sync speed only after setting up both sides can race with DRBD
1043 # connecting, hence we set it here before telling DRBD anything
1045 cls._SetMinorSyncSpeed(minor, constants.SYNC_SPEED)
1047 args = ["drbdsetup", cls._DevPath(minor), "net",
1048 "%s:%s" % (lhost, lport), "%s:%s" % (rhost, rport), protocol,
1049 "-A", "discard-zero-changes",
1056 args.extend(["-a", hmac, "-x", secret])
1057 result = utils.RunCmd(args)
1059 logging.error("Can't setup network for dbrd device: %s - %s",
1060 result.fail_reason, result.output)
1063 timeout = time.time() + 10
1065 while time.time() < timeout:
1066 info = cls._GetDevInfo(cls._GetShowData(minor))
1067 if not "local_addr" in info or not "remote_addr" in info:
1070 if (info["local_addr"] != (lhost, lport) or
1071 info["remote_addr"] != (rhost, rport)):
1077 logging.error("Timeout while configuring network")
1081 def AddChildren(self, devices):
1082 """Add a disk to the DRBD device.
1085 if self.minor is None:
1086 raise errors.BlockDeviceError("Can't attach to dbrd8 during AddChildren")
1087 if len(devices) != 2:
1088 raise errors.BlockDeviceError("Need two devices for AddChildren")
1089 info = self._GetDevInfo(self._GetShowData(self.minor))
1090 if "local_dev" in info:
1091 raise errors.BlockDeviceError("DRBD8 already attached to a local disk")
1092 backend, meta = devices
1093 if backend.dev_path is None or meta.dev_path is None:
1094 raise errors.BlockDeviceError("Children not ready during AddChildren")
1097 if not self._CheckMetaSize(meta.dev_path):
1098 raise errors.BlockDeviceError("Invalid meta device size")
1099 self._InitMeta(self._FindUnusedMinor(), meta.dev_path)
1100 if not self._IsValidMeta(meta.dev_path):
1101 raise errors.BlockDeviceError("Cannot initalize meta device")
1103 if not self._AssembleLocal(self.minor, backend.dev_path, meta.dev_path):
1104 raise errors.BlockDeviceError("Can't attach to local storage")
1105 self._children = devices
1107 def RemoveChildren(self, devices):
1108 """Detach the drbd device from local storage.
1111 if self.minor is None:
1112 raise errors.BlockDeviceError("Can't attach to drbd8 during"
1114 # early return if we don't actually have backing storage
1115 info = self._GetDevInfo(self._GetShowData(self.minor))
1116 if "local_dev" not in info:
1118 if len(self._children) != 2:
1119 raise errors.BlockDeviceError("We don't have two children: %s" %
1121 if self._children.count(None) == 2: # we don't actually have children :)
1122 logging.error("Requested detach while detached")
1124 if len(devices) != 2:
1125 raise errors.BlockDeviceError("We need two children in RemoveChildren")
1126 for child, dev in zip(self._children, devices):
1127 if dev != child.dev_path:
1128 raise errors.BlockDeviceError("Mismatch in local storage"
1129 " (%s != %s) in RemoveChildren" %
1130 (dev, child.dev_path))
1132 if not self._ShutdownLocal(self.minor):
1133 raise errors.BlockDeviceError("Can't detach from local storage")
1137 def _SetMinorSyncSpeed(cls, minor, kbytes):
1138 """Set the speed of the DRBD syncer.
1140 This is the low-level implementation.
1143 @param minor: the drbd minor whose settings we change
1145 @param kbytes: the speed in kbytes/second
1147 @return: the success of the operation
1150 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "syncer",
1151 "-r", "%d" % kbytes, "--create-device"])
1153 logging.error("Can't change syncer rate: %s - %s",
1154 result.fail_reason, result.output)
1155 return not result.failed
1157 def SetSyncSpeed(self, kbytes):
1158 """Set the speed of the DRBD syncer.
1161 @param kbytes: the speed in kbytes/second
1163 @return: the success of the operation
1166 if self.minor is None:
1167 logging.info("Not attached during SetSyncSpeed")
1169 children_result = super(DRBD8, self).SetSyncSpeed(kbytes)
1170 return self._SetMinorSyncSpeed(self.minor, kbytes) and children_result
1172 def GetProcStatus(self):
1173 """Return device data from /proc.
1176 if self.minor is None:
1177 raise errors.BlockDeviceError("GetStats() called while not attached")
1178 proc_info = self._MassageProcData(self._GetProcData())
1179 if self.minor not in proc_info:
1180 raise errors.BlockDeviceError("Can't find myself in /proc (minor %d)" %
1182 return DRBD8Status(proc_info[self.minor])
1184 def GetSyncStatus(self):
1185 """Returns the sync status of the device.
1188 If sync_percent is None, it means all is ok
1189 If estimated_time is None, it means we can't esimate
1190 the time needed, otherwise it's the time left in seconds.
1193 We set the is_degraded parameter to True on two conditions:
1194 network not connected or local disk missing.
1196 We compute the ldisk parameter based on wheter we have a local
1200 @return: (sync_percent, estimated_time, is_degraded, ldisk)
1203 if self.minor is None and not self.Attach():
1204 raise errors.BlockDeviceError("Can't attach to device in GetSyncStatus")
1205 stats = self.GetProcStatus()
1206 ldisk = not stats.is_disk_uptodate
1207 is_degraded = not stats.is_connected
1208 return stats.sync_percent, stats.est_time, is_degraded or ldisk, ldisk
1210 def Open(self, force=False):
1211 """Make the local state primary.
1213 If the 'force' parameter is given, the '-o' option is passed to
1214 drbdsetup. Since this is a potentially dangerous operation, the
1215 force flag should be only given after creation, when it actually
1219 if self.minor is None and not self.Attach():
1220 logging.error("DRBD cannot attach to a device during open")
1222 cmd = ["drbdsetup", self.dev_path, "primary"]
1225 result = utils.RunCmd(cmd)
1227 msg = ("Can't make drbd device primary: %s" % result.output)
1229 raise errors.BlockDeviceError(msg)
1232 """Make the local state secondary.
1234 This will, of course, fail if the device is in use.
1237 if self.minor is None and not self.Attach():
1238 logging.info("Instance not attached to a device")
1239 raise errors.BlockDeviceError("Can't find device")
1240 result = utils.RunCmd(["drbdsetup", self.dev_path, "secondary"])
1242 msg = ("Can't switch drbd device to"
1243 " secondary: %s" % result.output)
1245 raise errors.BlockDeviceError(msg)
1247 def DisconnectNet(self):
1248 """Removes network configuration.
1250 This method shutdowns the network side of the device.
1252 The method will wait up to a hardcoded timeout for the device to
1253 go into standalone after the 'disconnect' command before
1254 re-configuring it, as sometimes it takes a while for the
1255 disconnect to actually propagate and thus we might issue a 'net'
1256 command while the device is still connected. If the device will
1257 still be attached to the network and we time out, we raise an
1261 if self.minor is None:
1262 raise errors.BlockDeviceError("DRBD disk not attached in re-attach net")
1264 if None in (self._lhost, self._lport, self._rhost, self._rport):
1265 raise errors.BlockDeviceError("DRBD disk missing network info in"
1268 ever_disconnected = self._ShutdownNet(self.minor)
1269 timeout_limit = time.time() + self._NET_RECONFIG_TIMEOUT
1270 sleep_time = 0.100 # we start the retry time at 100 miliseconds
1271 while time.time() < timeout_limit:
1272 status = self.GetProcStatus()
1273 if status.is_standalone:
1275 # retry the disconnect, it seems possible that due to a
1276 # well-time disconnect on the peer, my disconnect command might
1277 # be ingored and forgotten
1278 ever_disconnected = self._ShutdownNet(self.minor) or ever_disconnected
1279 time.sleep(sleep_time)
1280 sleep_time = min(2, sleep_time * 1.5)
1282 if not status.is_standalone:
1283 if ever_disconnected:
1284 msg = ("Device did not react to the"
1285 " 'disconnect' command in a timely manner")
1287 msg = ("Can't shutdown network, even after multiple retries")
1288 raise errors.BlockDeviceError(msg)
1290 reconfig_time = time.time() - timeout_limit + self._NET_RECONFIG_TIMEOUT
1291 if reconfig_time > 15: # hardcoded alert limit
1292 logging.debug("DRBD8.DisconnectNet: detach took %.3f seconds",
1295 def AttachNet(self, multimaster):
1296 """Reconnects the network.
1298 This method connects the network side of the device with a
1299 specified multi-master flag. The device needs to be 'Standalone'
1300 but have valid network configuration data.
1303 - multimaster: init the network in dual-primary mode
1306 if self.minor is None:
1307 raise errors.BlockDeviceError("DRBD disk not attached in AttachNet")
1309 if None in (self._lhost, self._lport, self._rhost, self._rport):
1310 raise errors.BlockDeviceError("DRBD disk missing network info in"
1313 status = self.GetProcStatus()
1315 if not status.is_standalone:
1316 raise errors.BlockDeviceError("Device is not standalone in AttachNet")
1318 return self._AssembleNet(self.minor,
1319 (self._lhost, self._lport,
1320 self._rhost, self._rport),
1321 "C", dual_pri=multimaster)
1324 """Check if our minor is configured.
1326 This doesn't do any device configurations - it only checks if the
1327 minor is in a state different from Unconfigured.
1329 Note that this function will not change the state of the system in
1330 any way (except in case of side-effects caused by reading from
1334 used_devs = self._GetUsedDevs()
1335 if self._aminor in used_devs:
1336 minor = self._aminor
1340 self._SetFromMinor(minor)
1341 return minor is not None
1344 """Assemble the drbd.
1347 - if we have a configured device, we try to ensure that it matches
1349 - if not, we create it from zero
1352 result = super(DRBD8, self).Assemble()
1357 if self.minor is None:
1358 # local device completely unconfigured
1359 return self._FastAssemble()
1361 # we have to recheck the local and network status and try to fix
1363 return self._SlowAssemble()
1365 def _SlowAssemble(self):
1366 """Assembles the DRBD device from a (partially) configured device.
1368 In case of partially attached (local device matches but no network
1369 setup), we perform the network attach. If successful, we re-test
1370 the attach if can return success.
1373 for minor in (self._aminor,):
1374 info = self._GetDevInfo(self._GetShowData(minor))
1375 match_l = self._MatchesLocal(info)
1376 match_r = self._MatchesNet(info)
1377 if match_l and match_r:
1379 if match_l and not match_r and "local_addr" not in info:
1380 res_r = self._AssembleNet(minor,
1381 (self._lhost, self._lport,
1382 self._rhost, self._rport),
1383 constants.DRBD_NET_PROTOCOL,
1384 hmac=constants.DRBD_HMAC_ALG,
1388 if self._MatchesNet(self._GetDevInfo(self._GetShowData(minor))):
1390 # the weakest case: we find something that is only net attached
1391 # even though we were passed some children at init time
1392 if match_r and "local_dev" not in info:
1395 # this case must be considered only if we actually have local
1396 # storage, i.e. not in diskless mode, because all diskless
1397 # devices are equal from the point of view of local
1399 if (match_l and "local_dev" in info and
1400 not match_r and "local_addr" in info):
1401 # strange case - the device network part points to somewhere
1402 # else, even though its local storage is ours; as we own the
1403 # drbd space, we try to disconnect from the remote peer and
1404 # reconnect to our correct one
1405 if not self._ShutdownNet(minor):
1406 raise errors.BlockDeviceError("Device has correct local storage,"
1407 " wrong remote peer and is unable to"
1408 " disconnect in order to attach to"
1409 " the correct peer")
1410 # note: _AssembleNet also handles the case when we don't want
1411 # local storage (i.e. one or more of the _[lr](host|port) is
1413 if (self._AssembleNet(minor, (self._lhost, self._lport,
1414 self._rhost, self._rport),
1415 constants.DRBD_NET_PROTOCOL,
1416 hmac=constants.DRBD_HMAC_ALG,
1417 secret=self._secret) and
1418 self._MatchesNet(self._GetDevInfo(self._GetShowData(minor)))):
1424 self._SetFromMinor(minor)
1425 return minor is not None
1427 def _FastAssemble(self):
1428 """Assemble the drbd device from zero.
1430 This is run when in Assemble we detect our minor is unused.
1433 # TODO: maybe completely tear-down the minor (drbdsetup ... down)
1434 # before attaching our own?
1435 minor = self._aminor
1436 need_localdev_teardown = False
1437 if self._children and self._children[0] and self._children[1]:
1438 result = self._AssembleLocal(minor, self._children[0].dev_path,
1439 self._children[1].dev_path)
1442 need_localdev_teardown = True
1443 if self._lhost and self._lport and self._rhost and self._rport:
1444 result = self._AssembleNet(minor,
1445 (self._lhost, self._lport,
1446 self._rhost, self._rport),
1447 constants.DRBD_NET_PROTOCOL,
1448 hmac=constants.DRBD_HMAC_ALG,
1449 secret=self._secret)
1451 if need_localdev_teardown:
1452 # we will ignore failures from this
1453 logging.error("net setup failed, tearing down local device")
1454 self._ShutdownAll(minor)
1456 self._SetFromMinor(minor)
1460 def _ShutdownLocal(cls, minor):
1461 """Detach from the local device.
1463 I/Os will continue to be served from the remote device. If we
1464 don't have a remote device, this operation will fail.
1467 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "detach"])
1469 logging.error("Can't detach local device: %s", result.output)
1470 return not result.failed
1473 def _ShutdownNet(cls, minor):
1474 """Disconnect from the remote peer.
1476 This fails if we don't have a local device.
1479 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "disconnect"])
1481 logging.error("Can't shutdown network: %s", result.output)
1482 return not result.failed
1485 def _ShutdownAll(cls, minor):
1486 """Deactivate the device.
1488 This will, of course, fail if the device is in use.
1491 result = utils.RunCmd(["drbdsetup", cls._DevPath(minor), "down"])
1493 logging.error("Can't shutdown drbd device: %s", result.output)
1494 return not result.failed
1497 """Shutdown the DRBD device.
1500 if self.minor is None and not self.Attach():
1501 logging.info("DRBD device not attached to a device during Shutdown")
1503 if not self._ShutdownAll(self.minor):
1506 self.dev_path = None
1510 """Stub remove for DRBD devices.
1513 return self.Shutdown()
1516 def Create(cls, unique_id, children, size):
1517 """Create a new DRBD8 device.
1519 Since DRBD devices are not created per se, just assembled, this
1520 function only initializes the metadata.
1523 if len(children) != 2:
1524 raise errors.ProgrammerError("Invalid setup for the drbd device")
1527 if not meta.Attach():
1528 raise errors.BlockDeviceError("Can't attach to meta device")
1529 if not cls._CheckMetaSize(meta.dev_path):
1530 raise errors.BlockDeviceError("Invalid meta device size")
1531 cls._InitMeta(cls._FindUnusedMinor(), meta.dev_path)
1532 if not cls._IsValidMeta(meta.dev_path):
1533 raise errors.BlockDeviceError("Cannot initalize meta device")
1534 return cls(unique_id, children)
1536 def Grow(self, amount):
1537 """Resize the DRBD device and its backing storage.
1540 if self.minor is None:
1541 raise errors.ProgrammerError("drbd8: Grow called while not attached")
1542 if len(self._children) != 2 or None in self._children:
1543 raise errors.BlockDeviceError("Cannot grow diskless DRBD8 device")
1544 self._children[0].Grow(amount)
1545 result = utils.RunCmd(["drbdsetup", self.dev_path, "resize"])
1547 raise errors.BlockDeviceError("resize failed for %s: %s" %
1548 (self.dev_path, result.output))
1552 class FileStorage(BlockDev):
1555 This class represents the a file storage backend device.
1557 The unique_id for the file device is a (file_driver, file_path) tuple.
1560 def __init__(self, unique_id, children):
1561 """Initalizes a file device backend.
1565 raise errors.BlockDeviceError("Invalid setup for file device")
1566 super(FileStorage, self).__init__(unique_id, children)
1567 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1568 raise ValueError("Invalid configuration data %s" % str(unique_id))
1569 self.driver = unique_id[0]
1570 self.dev_path = unique_id[1]
1574 """Assemble the device.
1576 Checks whether the file device exists, raises BlockDeviceError otherwise.
1579 if not os.path.exists(self.dev_path):
1580 raise errors.BlockDeviceError("File device '%s' does not exist." %
1585 """Shutdown the device.
1587 This is a no-op for the file type, as we don't deacivate
1588 the file on shutdown.
1593 def Open(self, force=False):
1594 """Make the device ready for I/O.
1596 This is a no-op for the file type.
1602 """Notifies that the device will no longer be used for I/O.
1604 This is a no-op for the file type.
1610 """Remove the file backing the block device.
1613 @return: True if the removal was successful
1616 if not os.path.exists(self.dev_path):
1619 os.remove(self.dev_path)
1621 except OSError, err:
1622 logging.error("Can't remove file '%s': %s", self.dev_path, err)
1626 """Attach to an existing file.
1628 Check if this file already exists.
1631 @return: True if file exists
1634 self.attached = os.path.exists(self.dev_path)
1635 return self.attached
1638 def Create(cls, unique_id, children, size):
1639 """Create a new file.
1641 @param size: the size of file in MiB
1643 @rtype: L{bdev.FileStorage}
1644 @return: an instance of FileStorage
1647 if not isinstance(unique_id, (tuple, list)) or len(unique_id) != 2:
1648 raise ValueError("Invalid configuration data %s" % str(unique_id))
1649 dev_path = unique_id[1]
1651 f = open(dev_path, 'w')
1652 except IOError, err:
1653 raise errors.BlockDeviceError("Could not create '%'" % err)
1655 f.truncate(size * 1024 * 1024)
1658 return FileStorage(unique_id, children)
1662 constants.LD_LV: LogicalVolume,
1663 constants.LD_DRBD8: DRBD8,
1664 constants.LD_FILE: FileStorage,
1668 def FindDevice(dev_type, unique_id, children):
1669 """Search for an existing, assembled device.
1671 This will succeed only if the device exists and is assembled, but it
1672 does not do any actions in order to activate the device.
1675 if dev_type not in DEV_MAP:
1676 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1677 device = DEV_MAP[dev_type](unique_id, children)
1678 if not device.attached:
1683 def Assemble(dev_type, unique_id, children):
1684 """Try to attach or assemble an existing device.
1686 This will attach to assemble the device, as needed, to bring it
1687 fully up. It must be safe to run on already-assembled devices.
1690 if dev_type not in DEV_MAP:
1691 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1692 device = DEV_MAP[dev_type](unique_id, children)
1693 if not device.Assemble():
1694 raise errors.BlockDeviceError("Can't find a valid block device for"
1696 (dev_type, unique_id, children))
1700 def Create(dev_type, unique_id, children, size):
1704 if dev_type not in DEV_MAP:
1705 raise errors.ProgrammerError("Invalid block device type '%s'" % dev_type)
1706 device = DEV_MAP[dev_type].Create(unique_id, children, size)