4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
29 # Allow wildcard import in pylint: disable=W0401
41 from ganeti import errors
42 from ganeti import constants
43 from ganeti import compat
44 from ganeti import pathutils
46 from ganeti.utils.algo import *
47 from ganeti.utils.filelock import *
48 from ganeti.utils.hash import *
49 from ganeti.utils.io import *
50 from ganeti.utils.log import *
51 from ganeti.utils.lvm import *
52 from ganeti.utils.mlock import *
53 from ganeti.utils.nodesetup import *
54 from ganeti.utils.process import *
55 from ganeti.utils.retry import *
56 from ganeti.utils.storage import *
57 from ganeti.utils.text import *
58 from ganeti.utils.wrapper import *
59 from ganeti.utils.x509 import *
62 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
64 UUID_RE = re.compile(constants.UUID_REGEX)
67 def ForceDictType(target, key_types, allowed_values=None):
68 """Force the values of a dict to have certain types.
71 @param target: the dict to update
73 @param key_types: dict mapping target dict keys to types
74 in constants.ENFORCEABLE_TYPES
75 @type allowed_values: list
76 @keyword allowed_values: list of specially allowed values
79 if allowed_values is None:
82 if not isinstance(target, dict):
83 msg = "Expected dictionary, got '%s'" % target
84 raise errors.TypeEnforcementError(msg)
87 if key not in key_types:
88 msg = "Unknown parameter '%s'" % key
89 raise errors.TypeEnforcementError(msg)
91 if target[key] in allowed_values:
94 ktype = key_types[key]
95 if ktype not in constants.ENFORCEABLE_TYPES:
96 msg = "'%s' has non-enforceable type %s" % (key, ktype)
97 raise errors.ProgrammerError(msg)
99 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
100 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
102 elif not isinstance(target[key], basestring):
103 if isinstance(target[key], bool) and not target[key]:
106 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
107 raise errors.TypeEnforcementError(msg)
108 elif ktype == constants.VTYPE_BOOL:
109 if isinstance(target[key], basestring) and target[key]:
110 if target[key].lower() == constants.VALUE_FALSE:
112 elif target[key].lower() == constants.VALUE_TRUE:
115 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
116 raise errors.TypeEnforcementError(msg)
121 elif ktype == constants.VTYPE_SIZE:
123 target[key] = ParseUnit(target[key])
124 except errors.UnitParseError, err:
125 msg = "'%s' (value %s) is not a valid size. error: %s" % \
126 (key, target[key], err)
127 raise errors.TypeEnforcementError(msg)
128 elif ktype == constants.VTYPE_INT:
130 target[key] = int(target[key])
131 except (ValueError, TypeError):
132 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
133 raise errors.TypeEnforcementError(msg)
136 def ValidateServiceName(name):
137 """Validate the given service name.
139 @type name: number or string
140 @param name: Service name or port specification
145 except (ValueError, TypeError):
146 # Non-numeric service name
147 valid = _VALID_SERVICE_NAME_RE.match(name)
149 # Numeric port (protocols other than TCP or UDP might need adjustments
151 valid = (numport >= 0 and numport < (1 << 16))
154 raise errors.OpPrereqError("Invalid service name '%s'" % name,
160 def _ComputeMissingKeys(key_path, options, defaults):
161 """Helper functions to compute which keys a invalid.
163 @param key_path: The current key path (if any)
164 @param options: The user provided options
165 @param defaults: The default dictionary
166 @return: A list of invalid keys
169 defaults_keys = frozenset(defaults.keys())
171 for key, value in options.items():
173 new_path = "%s/%s" % (key_path, key)
177 if key not in defaults_keys:
178 invalid.append(new_path)
179 elif isinstance(value, dict):
180 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
185 def VerifyDictOptions(options, defaults):
186 """Verify a dict has only keys set which also are in the defaults dict.
188 @param options: The user provided options
189 @param defaults: The default dictionary
190 @raise error.OpPrereqError: If one of the keys is not supported
193 invalid = _ComputeMissingKeys("", options, defaults)
196 raise errors.OpPrereqError("Provided option keys not supported: %s" %
197 CommaJoin(invalid), errors.ECODE_INVAL)
200 def ListVolumeGroups():
201 """List volume groups and their size
205 Dictionary with keys volume name and values
206 the size of the volume
209 command = "vgs --noheadings --units m --nosuffix -o name,size"
210 result = RunCmd(command)
215 for line in result.stdout.splitlines():
217 name, size = line.split()
218 size = int(float(size))
219 except (IndexError, ValueError), err:
220 logging.error("Invalid output from vgs (%s): %s", err, line)
228 def BridgeExists(bridge):
229 """Check whether the given bridge exists in the system
232 @param bridge: the bridge name to check
234 @return: True if it does
237 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
240 def TryConvert(fn, val):
241 """Try to convert a value ignoring errors.
243 This function tries to apply function I{fn} to I{val}. If no
244 C{ValueError} or C{TypeError} exceptions are raised, it will return
245 the result, else it will return the original value. Any other
246 exceptions are propagated to the caller.
249 @param fn: function to apply to the value
250 @param val: the value to be converted
251 @return: The converted value if the conversion was successful,
252 otherwise the original value.
257 except (ValueError, TypeError):
262 def ParseCpuMask(cpu_mask):
263 """Parse a CPU mask definition and return the list of CPU IDs.
265 CPU mask format: comma-separated list of CPU IDs
266 or dash-separated ID ranges
267 Example: "0-2,5" -> "0,1,2,5"
270 @param cpu_mask: CPU mask definition
272 @return: list of CPU IDs
278 for range_def in cpu_mask.split(","):
279 boundaries = range_def.split("-")
280 n_elements = len(boundaries)
282 raise errors.ParseError("Invalid CPU ID range definition"
283 " (only one hyphen allowed): %s" % range_def)
285 lower = int(boundaries[0])
286 except (ValueError, TypeError), err:
287 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
288 " CPU ID range: %s" % str(err))
290 higher = int(boundaries[-1])
291 except (ValueError, TypeError), err:
292 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
293 " CPU ID range: %s" % str(err))
295 raise errors.ParseError("Invalid CPU ID range definition"
296 " (%d > %d): %s" % (lower, higher, range_def))
297 cpu_list.extend(range(lower, higher + 1))
301 def ParseMultiCpuMask(cpu_mask):
302 """Parse a multiple CPU mask definition and return the list of CPU IDs.
304 CPU mask format: colon-separated list of comma-separated list of CPU IDs
305 or dash-separated ID ranges, with optional "all" as CPU value
306 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
309 @param cpu_mask: multiple CPU mask definition
310 @rtype: list of lists of int
311 @return: list of lists of CPU IDs
317 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
318 if range_def == constants.CPU_PINNING_ALL:
319 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
321 # Uniquify and sort the list before adding
322 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
327 def GetHomeDir(user, default=None):
328 """Try to get the homedir of the given user.
330 The user can be passed either as a string (denoting the name) or as
331 an integer (denoting the user id). If the user is not found, the
332 C{default} argument is returned, which defaults to C{None}.
336 if isinstance(user, basestring):
337 result = pwd.getpwnam(user)
338 elif isinstance(user, (int, long)):
339 result = pwd.getpwuid(user)
341 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
348 def FirstFree(seq, base=0):
349 """Returns the first non-existing integer from seq.
351 The seq argument should be a sorted list of positive integers. The
352 first time the index of an element is smaller than the element
353 value, the index will be returned.
355 The base argument is used to start at a different offset,
356 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
358 Example: C{[0, 1, 3]} will return I{2}.
361 @param seq: the sequence to be analyzed.
363 @param base: use this value as the base index of the sequence
365 @return: the first non-used index in the sequence
368 for idx, elem in enumerate(seq):
369 assert elem >= base, "Passed element is higher than base offset"
370 if elem > idx + base:
376 def SingleWaitForFdCondition(fdobj, event, timeout):
377 """Waits for a condition to occur on the socket.
379 Immediately returns at the first interruption.
381 @type fdobj: integer or object supporting a fileno() method
382 @param fdobj: entity to wait for events on
384 @param event: ORed condition (see select module)
385 @type timeout: float or None
386 @param timeout: Timeout in seconds
388 @return: None for timeout, otherwise occured conditions
391 check = (event | select.POLLPRI |
392 select.POLLNVAL | select.POLLHUP | select.POLLERR)
394 if timeout is not None:
395 # Poller object expects milliseconds
398 poller = select.poll()
399 poller.register(fdobj, event)
401 # TODO: If the main thread receives a signal and we have no timeout, we
402 # could wait forever. This should check a global "quit" flag or something
404 io_events = poller.poll(timeout)
405 except select.error, err:
406 if err[0] != errno.EINTR:
409 if io_events and io_events[0][1] & check:
410 return io_events[0][1]
415 class FdConditionWaiterHelper(object):
416 """Retry helper for WaitForFdCondition.
418 This class contains the retried and wait functions that make sure
419 WaitForFdCondition can continue waiting until the timeout is actually
424 def __init__(self, timeout):
425 self.timeout = timeout
427 def Poll(self, fdobj, event):
428 result = SingleWaitForFdCondition(fdobj, event, self.timeout)
434 def UpdateTimeout(self, timeout):
435 self.timeout = timeout
438 def WaitForFdCondition(fdobj, event, timeout):
439 """Waits for a condition to occur on the socket.
441 Retries until the timeout is expired, even if interrupted.
443 @type fdobj: integer or object supporting a fileno() method
444 @param fdobj: entity to wait for events on
446 @param event: ORed condition (see select module)
447 @type timeout: float or None
448 @param timeout: Timeout in seconds
450 @return: None for timeout, otherwise occured conditions
453 if timeout is not None:
454 retrywaiter = FdConditionWaiterHelper(timeout)
456 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
457 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
462 while result is None:
463 result = SingleWaitForFdCondition(fdobj, event, timeout)
467 def EnsureDaemon(name):
468 """Check for and start daemon if not alive.
471 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
473 logging.error("Can't start daemon '%s', failure %s, output: %s",
474 name, result.fail_reason, result.output)
480 def StopDaemon(name):
484 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
486 logging.error("Can't stop daemon '%s', failure %s, output: %s",
487 name, result.fail_reason, result.output)
493 def SplitTime(value):
494 """Splits time as floating point number into a tuple.
496 @param value: Time in seconds
497 @type value: int or float
498 @return: Tuple containing (seconds, microseconds)
501 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
503 assert 0 <= seconds, \
504 "Seconds must be larger than or equal to 0, but are %s" % seconds
505 assert 0 <= microseconds <= 999999, \
506 "Microseconds must be 0-999999, but are %s" % microseconds
508 return (int(seconds), int(microseconds))
511 def MergeTime(timetuple):
512 """Merges a tuple into time as a floating point number.
514 @param timetuple: Time as tuple, (seconds, microseconds)
515 @type timetuple: tuple
516 @return: Time as a floating point number expressed in seconds
519 (seconds, microseconds) = timetuple
521 assert 0 <= seconds, \
522 "Seconds must be larger than or equal to 0, but are %s" % seconds
523 assert 0 <= microseconds <= 999999, \
524 "Microseconds must be 0-999999, but are %s" % microseconds
526 return float(seconds) + (float(microseconds) * 0.000001)
530 """Return the current timestamp expressed as number of nanoseconds since the
533 @return: nanoseconds since the Unix epoch
536 return int(time.time() * 1000000000)
539 def FindMatch(data, name):
540 """Tries to find an item in a dictionary matching a name.
542 Callers have to ensure the data names aren't contradictory (e.g. a regexp
543 that matches a string). If the name isn't a direct key, all regular
544 expression objects in the dictionary are matched against it.
547 @param data: Dictionary containing data
549 @param name: Name to look for
550 @rtype: tuple; (value in dictionary, matched groups as list)
554 return (data[name], [])
556 for key, value in data.items():
558 if hasattr(key, "match"):
561 return (value, list(m.groups()))
566 def GetMounts(filename=constants.PROC_MOUNTS):
567 """Returns the list of mounted filesystems.
569 This function is Linux-specific.
571 @param filename: path of mounts file (/proc/mounts by default)
572 @rtype: list of tuples
573 @return: list of mount entries (device, mountpoint, fstype, options)
576 # TODO(iustin): investigate non-Linux options (e.g. via mount output)
578 mountlines = ReadFile(filename).splitlines()
579 for line in mountlines:
580 device, mountpoint, fstype, options, _ = line.split(None, 4)
581 data.append((device, mountpoint, fstype, options))
586 def SignalHandled(signums):
587 """Signal Handled decoration.
589 This special decorator installs a signal handler and then calls the target
590 function. The function must accept a 'signal_handlers' keyword argument,
591 which will contain a dict indexed by signal number, with SignalHandler
594 The decorator can be safely stacked with iself, to handle multiple signals
595 with different handlers.
598 @param signums: signals to intercept
602 def sig_function(*args, **kwargs):
603 assert "signal_handlers" not in kwargs or \
604 kwargs["signal_handlers"] is None or \
605 isinstance(kwargs["signal_handlers"], dict), \
606 "Wrong signal_handlers parameter in original function call"
607 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
608 signal_handlers = kwargs["signal_handlers"]
611 kwargs["signal_handlers"] = signal_handlers
612 sighandler = SignalHandler(signums)
615 signal_handlers[sig] = sighandler
616 return fn(*args, **kwargs)
623 def TimeoutExpired(epoch, timeout, _time_fn=time.time):
624 """Checks whether a timeout has expired.
627 return _time_fn() > (epoch + timeout)
630 class SignalWakeupFd(object):
632 # This is only supported in Python 2.5 and above (some distributions
633 # backported it to Python 2.4)
634 _set_wakeup_fd_fn = signal.set_wakeup_fd
635 except AttributeError:
638 def _SetWakeupFd(self, _): # pylint: disable=R0201
642 def _SetWakeupFd(self, fd):
643 return self._set_wakeup_fd_fn(fd)
646 """Initializes this class.
649 (read_fd, write_fd) = os.pipe()
651 # Once these succeeded, the file descriptors will be closed automatically.
652 # Buffer size 0 is important, otherwise .read() with a specified length
653 # might buffer data and the file descriptors won't be marked readable.
654 self._read_fh = os.fdopen(read_fd, "r", 0)
655 self._write_fh = os.fdopen(write_fd, "w", 0)
657 self._previous = self._SetWakeupFd(self._write_fh.fileno())
660 self.fileno = self._read_fh.fileno
661 self.read = self._read_fh.read
664 """Restores the previous wakeup file descriptor.
667 if hasattr(self, "_previous") and self._previous is not None:
668 self._SetWakeupFd(self._previous)
669 self._previous = None
672 """Notifies the wakeup file descriptor.
675 self._write_fh.write(chr(0))
678 """Called before object deletion.
684 class SignalHandler(object):
685 """Generic signal handler class.
687 It automatically restores the original handler when deconstructed or
688 when L{Reset} is called. You can either pass your own handler
689 function in or query the L{called} attribute to detect whether the
693 @ivar signum: the signals we handle
694 @type called: boolean
695 @ivar called: tracks whether any of the signals have been raised
698 def __init__(self, signum, handler_fn=None, wakeup=None):
699 """Constructs a new SignalHandler instance.
701 @type signum: int or list of ints
702 @param signum: Single signal number or set of signal numbers
703 @type handler_fn: callable
704 @param handler_fn: Signal handling function
707 assert handler_fn is None or callable(handler_fn)
709 self.signum = set(signum)
712 self._handler_fn = handler_fn
713 self._wakeup = wakeup
717 for signum in self.signum:
719 prev_handler = signal.signal(signum, self._HandleSignal)
721 self._previous[signum] = prev_handler
723 # Restore previous handler
724 signal.signal(signum, prev_handler)
729 # Here we have a race condition: a handler may have already been called,
730 # but there's not much we can do about it at this point.
737 """Restore previous handler.
739 This will reset all the signals to their previous handlers.
742 for signum, prev_handler in self._previous.items():
743 signal.signal(signum, prev_handler)
744 # If successful, remove from dict
745 del self._previous[signum]
748 """Unsets the L{called} flag.
750 This function can be used in case a signal may arrive several times.
755 def _HandleSignal(self, signum, frame):
756 """Actual signal handling function.
759 # This is not nice and not absolutely atomic, but it appears to be the only
760 # solution in Python -- there are no atomic types.
764 # Notify whoever is interested in signals
765 self._wakeup.Notify()
768 self._handler_fn(signum, frame)
771 class FieldSet(object):
772 """A simple field set.
774 Among the features are:
775 - checking if a string is among a list of static string or regex objects
776 - checking if a whole list of string matches
777 - returning the matching groups from a regex match
779 Internally, all fields are held as regular expression objects.
782 def __init__(self, *items):
783 self.items = [re.compile("^%s$" % value) for value in items]
785 def Extend(self, other_set):
786 """Extend the field set with the items from another one"""
787 self.items.extend(other_set.items)
789 def Matches(self, field):
790 """Checks if a field matches the current set
793 @param field: the string to match
794 @return: either None or a regular expression match object
797 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
801 def NonMatching(self, items):
802 """Returns the list of fields not matching the current set
805 @param items: the list of fields to check
807 @return: list of non-matching fields
810 return [val for val in items if not self.Matches(val)]
813 def ValidateDeviceNames(kind, container):
814 """Validate instance device names.
816 Check that a device container contains only unique and valid names.
819 @param kind: One-word item description
820 @type container: list
821 @param container: Container containing the devices
826 for device in container:
827 if isinstance(device, dict):
829 name = device.get(constants.INIC_NAME, None)
831 name = device.get(constants.IDISK_NAME, None)
833 raise errors.OpPrereqError("Invalid container kind '%s'" % kind,
837 # Check that a device name is not the UUID of another device
838 valid.append(device.uuid)
842 except (ValueError, TypeError):
845 raise errors.OpPrereqError("Invalid name '%s'. Purely numeric %s names"
846 " are not allowed" % (name, kind),
849 if name is not None and name.lower() != constants.VALUE_NONE:
851 raise errors.OpPrereqError("%s name '%s' already used" % (kind, name),
852 errors.ECODE_NOTUNIQUE)