4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
29 # Allow wildcard import in pylint: disable=W0401
41 from ganeti import errors
42 from ganeti import constants
43 from ganeti import compat
44 from ganeti import pathutils
46 from ganeti.utils.algo import *
47 from ganeti.utils.filelock import *
48 from ganeti.utils.hash import *
49 from ganeti.utils.io import *
50 from ganeti.utils.log import *
51 from ganeti.utils.lvm import *
52 from ganeti.utils.mlock import *
53 from ganeti.utils.nodesetup import *
54 from ganeti.utils.process import *
55 from ganeti.utils.retry import *
56 from ganeti.utils.text import *
57 from ganeti.utils.wrapper import *
58 from ganeti.utils.x509 import *
61 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
63 UUID_RE = re.compile(constants.UUID_REGEX)
66 def ForceDictType(target, key_types, allowed_values=None):
67 """Force the values of a dict to have certain types.
70 @param target: the dict to update
72 @param key_types: dict mapping target dict keys to types
73 in constants.ENFORCEABLE_TYPES
74 @type allowed_values: list
75 @keyword allowed_values: list of specially allowed values
78 if allowed_values is None:
81 if not isinstance(target, dict):
82 msg = "Expected dictionary, got '%s'" % target
83 raise errors.TypeEnforcementError(msg)
86 if key not in key_types:
87 msg = "Unknown parameter '%s'" % key
88 raise errors.TypeEnforcementError(msg)
90 if target[key] in allowed_values:
93 ktype = key_types[key]
94 if ktype not in constants.ENFORCEABLE_TYPES:
95 msg = "'%s' has non-enforceable type %s" % (key, ktype)
96 raise errors.ProgrammerError(msg)
98 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
99 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
101 elif not isinstance(target[key], basestring):
102 if isinstance(target[key], bool) and not target[key]:
105 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
106 raise errors.TypeEnforcementError(msg)
107 elif ktype == constants.VTYPE_BOOL:
108 if isinstance(target[key], basestring) and target[key]:
109 if target[key].lower() == constants.VALUE_FALSE:
111 elif target[key].lower() == constants.VALUE_TRUE:
114 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
115 raise errors.TypeEnforcementError(msg)
120 elif ktype == constants.VTYPE_SIZE:
122 target[key] = ParseUnit(target[key])
123 except errors.UnitParseError, err:
124 msg = "'%s' (value %s) is not a valid size. error: %s" % \
125 (key, target[key], err)
126 raise errors.TypeEnforcementError(msg)
127 elif ktype == constants.VTYPE_INT:
129 target[key] = int(target[key])
130 except (ValueError, TypeError):
131 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
132 raise errors.TypeEnforcementError(msg)
135 def ValidateServiceName(name):
136 """Validate the given service name.
138 @type name: number or string
139 @param name: Service name or port specification
144 except (ValueError, TypeError):
145 # Non-numeric service name
146 valid = _VALID_SERVICE_NAME_RE.match(name)
148 # Numeric port (protocols other than TCP or UDP might need adjustments
150 valid = (numport >= 0 and numport < (1 << 16))
153 raise errors.OpPrereqError("Invalid service name '%s'" % name,
159 def _ComputeMissingKeys(key_path, options, defaults):
160 """Helper functions to compute which keys a invalid.
162 @param key_path: The current key path (if any)
163 @param options: The user provided options
164 @param defaults: The default dictionary
165 @return: A list of invalid keys
168 defaults_keys = frozenset(defaults.keys())
170 for key, value in options.items():
172 new_path = "%s/%s" % (key_path, key)
176 if key not in defaults_keys:
177 invalid.append(new_path)
178 elif isinstance(value, dict):
179 invalid.extend(_ComputeMissingKeys(new_path, value, defaults[key]))
184 def VerifyDictOptions(options, defaults):
185 """Verify a dict has only keys set which also are in the defaults dict.
187 @param options: The user provided options
188 @param defaults: The default dictionary
189 @raise error.OpPrereqError: If one of the keys is not supported
192 invalid = _ComputeMissingKeys("", options, defaults)
195 raise errors.OpPrereqError("Provided option keys not supported: %s" %
196 CommaJoin(invalid), errors.ECODE_INVAL)
199 def ListVolumeGroups():
200 """List volume groups and their size
204 Dictionary with keys volume name and values
205 the size of the volume
208 command = "vgs --noheadings --units m --nosuffix -o name,size"
209 result = RunCmd(command)
214 for line in result.stdout.splitlines():
216 name, size = line.split()
217 size = int(float(size))
218 except (IndexError, ValueError), err:
219 logging.error("Invalid output from vgs (%s): %s", err, line)
227 def BridgeExists(bridge):
228 """Check whether the given bridge exists in the system
231 @param bridge: the bridge name to check
233 @return: True if it does
236 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
239 def TryConvert(fn, val):
240 """Try to convert a value ignoring errors.
242 This function tries to apply function I{fn} to I{val}. If no
243 C{ValueError} or C{TypeError} exceptions are raised, it will return
244 the result, else it will return the original value. Any other
245 exceptions are propagated to the caller.
248 @param fn: function to apply to the value
249 @param val: the value to be converted
250 @return: The converted value if the conversion was successful,
251 otherwise the original value.
256 except (ValueError, TypeError):
261 def ParseCpuMask(cpu_mask):
262 """Parse a CPU mask definition and return the list of CPU IDs.
264 CPU mask format: comma-separated list of CPU IDs
265 or dash-separated ID ranges
266 Example: "0-2,5" -> "0,1,2,5"
269 @param cpu_mask: CPU mask definition
271 @return: list of CPU IDs
277 for range_def in cpu_mask.split(","):
278 boundaries = range_def.split("-")
279 n_elements = len(boundaries)
281 raise errors.ParseError("Invalid CPU ID range definition"
282 " (only one hyphen allowed): %s" % range_def)
284 lower = int(boundaries[0])
285 except (ValueError, TypeError), err:
286 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
287 " CPU ID range: %s" % str(err))
289 higher = int(boundaries[-1])
290 except (ValueError, TypeError), err:
291 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
292 " CPU ID range: %s" % str(err))
294 raise errors.ParseError("Invalid CPU ID range definition"
295 " (%d > %d): %s" % (lower, higher, range_def))
296 cpu_list.extend(range(lower, higher + 1))
300 def ParseMultiCpuMask(cpu_mask):
301 """Parse a multiple CPU mask definition and return the list of CPU IDs.
303 CPU mask format: colon-separated list of comma-separated list of CPU IDs
304 or dash-separated ID ranges, with optional "all" as CPU value
305 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
308 @param cpu_mask: multiple CPU mask definition
309 @rtype: list of lists of int
310 @return: list of lists of CPU IDs
316 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
317 if range_def == constants.CPU_PINNING_ALL:
318 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
320 # Uniquify and sort the list before adding
321 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
326 def GetHomeDir(user, default=None):
327 """Try to get the homedir of the given user.
329 The user can be passed either as a string (denoting the name) or as
330 an integer (denoting the user id). If the user is not found, the
331 C{default} argument is returned, which defaults to C{None}.
335 if isinstance(user, basestring):
336 result = pwd.getpwnam(user)
337 elif isinstance(user, (int, long)):
338 result = pwd.getpwuid(user)
340 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
347 def FirstFree(seq, base=0):
348 """Returns the first non-existing integer from seq.
350 The seq argument should be a sorted list of positive integers. The
351 first time the index of an element is smaller than the element
352 value, the index will be returned.
354 The base argument is used to start at a different offset,
355 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
357 Example: C{[0, 1, 3]} will return I{2}.
360 @param seq: the sequence to be analyzed.
362 @param base: use this value as the base index of the sequence
364 @return: the first non-used index in the sequence
367 for idx, elem in enumerate(seq):
368 assert elem >= base, "Passed element is higher than base offset"
369 if elem > idx + base:
375 def SingleWaitForFdCondition(fdobj, event, timeout):
376 """Waits for a condition to occur on the socket.
378 Immediately returns at the first interruption.
380 @type fdobj: integer or object supporting a fileno() method
381 @param fdobj: entity to wait for events on
383 @param event: ORed condition (see select module)
384 @type timeout: float or None
385 @param timeout: Timeout in seconds
387 @return: None for timeout, otherwise occured conditions
390 check = (event | select.POLLPRI |
391 select.POLLNVAL | select.POLLHUP | select.POLLERR)
393 if timeout is not None:
394 # Poller object expects milliseconds
397 poller = select.poll()
398 poller.register(fdobj, event)
400 # TODO: If the main thread receives a signal and we have no timeout, we
401 # could wait forever. This should check a global "quit" flag or something
403 io_events = poller.poll(timeout)
404 except select.error, err:
405 if err[0] != errno.EINTR:
408 if io_events and io_events[0][1] & check:
409 return io_events[0][1]
414 class FdConditionWaiterHelper(object):
415 """Retry helper for WaitForFdCondition.
417 This class contains the retried and wait functions that make sure
418 WaitForFdCondition can continue waiting until the timeout is actually
423 def __init__(self, timeout):
424 self.timeout = timeout
426 def Poll(self, fdobj, event):
427 result = SingleWaitForFdCondition(fdobj, event, self.timeout)
433 def UpdateTimeout(self, timeout):
434 self.timeout = timeout
437 def WaitForFdCondition(fdobj, event, timeout):
438 """Waits for a condition to occur on the socket.
440 Retries until the timeout is expired, even if interrupted.
442 @type fdobj: integer or object supporting a fileno() method
443 @param fdobj: entity to wait for events on
445 @param event: ORed condition (see select module)
446 @type timeout: float or None
447 @param timeout: Timeout in seconds
449 @return: None for timeout, otherwise occured conditions
452 if timeout is not None:
453 retrywaiter = FdConditionWaiterHelper(timeout)
455 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
456 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
461 while result is None:
462 result = SingleWaitForFdCondition(fdobj, event, timeout)
466 def EnsureDaemon(name):
467 """Check for and start daemon if not alive.
470 result = RunCmd([pathutils.DAEMON_UTIL, "check-and-start", name])
472 logging.error("Can't start daemon '%s', failure %s, output: %s",
473 name, result.fail_reason, result.output)
479 def StopDaemon(name):
483 result = RunCmd([pathutils.DAEMON_UTIL, "stop", name])
485 logging.error("Can't stop daemon '%s', failure %s, output: %s",
486 name, result.fail_reason, result.output)
492 def SplitTime(value):
493 """Splits time as floating point number into a tuple.
495 @param value: Time in seconds
496 @type value: int or float
497 @return: Tuple containing (seconds, microseconds)
500 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
502 assert 0 <= seconds, \
503 "Seconds must be larger than or equal to 0, but are %s" % seconds
504 assert 0 <= microseconds <= 999999, \
505 "Microseconds must be 0-999999, but are %s" % microseconds
507 return (int(seconds), int(microseconds))
510 def MergeTime(timetuple):
511 """Merges a tuple into time as a floating point number.
513 @param timetuple: Time as tuple, (seconds, microseconds)
514 @type timetuple: tuple
515 @return: Time as a floating point number expressed in seconds
518 (seconds, microseconds) = timetuple
520 assert 0 <= seconds, \
521 "Seconds must be larger than or equal to 0, but are %s" % seconds
522 assert 0 <= microseconds <= 999999, \
523 "Microseconds must be 0-999999, but are %s" % microseconds
525 return float(seconds) + (float(microseconds) * 0.000001)
528 def FindMatch(data, name):
529 """Tries to find an item in a dictionary matching a name.
531 Callers have to ensure the data names aren't contradictory (e.g. a regexp
532 that matches a string). If the name isn't a direct key, all regular
533 expression objects in the dictionary are matched against it.
536 @param data: Dictionary containing data
538 @param name: Name to look for
539 @rtype: tuple; (value in dictionary, matched groups as list)
543 return (data[name], [])
545 for key, value in data.items():
547 if hasattr(key, "match"):
550 return (value, list(m.groups()))
555 def GetMounts(filename=constants.PROC_MOUNTS):
556 """Returns the list of mounted filesystems.
558 This function is Linux-specific.
560 @param filename: path of mounts file (/proc/mounts by default)
561 @rtype: list of tuples
562 @return: list of mount entries (device, mountpoint, fstype, options)
565 # TODO(iustin): investigate non-Linux options (e.g. via mount output)
567 mountlines = ReadFile(filename).splitlines()
568 for line in mountlines:
569 device, mountpoint, fstype, options, _ = line.split(None, 4)
570 data.append((device, mountpoint, fstype, options))
575 def SignalHandled(signums):
576 """Signal Handled decoration.
578 This special decorator installs a signal handler and then calls the target
579 function. The function must accept a 'signal_handlers' keyword argument,
580 which will contain a dict indexed by signal number, with SignalHandler
583 The decorator can be safely stacked with iself, to handle multiple signals
584 with different handlers.
587 @param signums: signals to intercept
591 def sig_function(*args, **kwargs):
592 assert "signal_handlers" not in kwargs or \
593 kwargs["signal_handlers"] is None or \
594 isinstance(kwargs["signal_handlers"], dict), \
595 "Wrong signal_handlers parameter in original function call"
596 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
597 signal_handlers = kwargs["signal_handlers"]
600 kwargs["signal_handlers"] = signal_handlers
601 sighandler = SignalHandler(signums)
604 signal_handlers[sig] = sighandler
605 return fn(*args, **kwargs)
612 def TimeoutExpired(epoch, timeout, _time_fn=time.time):
613 """Checks whether a timeout has expired.
616 return _time_fn() > (epoch + timeout)
619 class SignalWakeupFd(object):
621 # This is only supported in Python 2.5 and above (some distributions
622 # backported it to Python 2.4)
623 _set_wakeup_fd_fn = signal.set_wakeup_fd
624 except AttributeError:
627 def _SetWakeupFd(self, _): # pylint: disable=R0201
631 def _SetWakeupFd(self, fd):
632 return self._set_wakeup_fd_fn(fd)
635 """Initializes this class.
638 (read_fd, write_fd) = os.pipe()
640 # Once these succeeded, the file descriptors will be closed automatically.
641 # Buffer size 0 is important, otherwise .read() with a specified length
642 # might buffer data and the file descriptors won't be marked readable.
643 self._read_fh = os.fdopen(read_fd, "r", 0)
644 self._write_fh = os.fdopen(write_fd, "w", 0)
646 self._previous = self._SetWakeupFd(self._write_fh.fileno())
649 self.fileno = self._read_fh.fileno
650 self.read = self._read_fh.read
653 """Restores the previous wakeup file descriptor.
656 if hasattr(self, "_previous") and self._previous is not None:
657 self._SetWakeupFd(self._previous)
658 self._previous = None
661 """Notifies the wakeup file descriptor.
664 self._write_fh.write("\0")
667 """Called before object deletion.
673 class SignalHandler(object):
674 """Generic signal handler class.
676 It automatically restores the original handler when deconstructed or
677 when L{Reset} is called. You can either pass your own handler
678 function in or query the L{called} attribute to detect whether the
682 @ivar signum: the signals we handle
683 @type called: boolean
684 @ivar called: tracks whether any of the signals have been raised
687 def __init__(self, signum, handler_fn=None, wakeup=None):
688 """Constructs a new SignalHandler instance.
690 @type signum: int or list of ints
691 @param signum: Single signal number or set of signal numbers
692 @type handler_fn: callable
693 @param handler_fn: Signal handling function
696 assert handler_fn is None or callable(handler_fn)
698 self.signum = set(signum)
701 self._handler_fn = handler_fn
702 self._wakeup = wakeup
706 for signum in self.signum:
708 prev_handler = signal.signal(signum, self._HandleSignal)
710 self._previous[signum] = prev_handler
712 # Restore previous handler
713 signal.signal(signum, prev_handler)
718 # Here we have a race condition: a handler may have already been called,
719 # but there's not much we can do about it at this point.
726 """Restore previous handler.
728 This will reset all the signals to their previous handlers.
731 for signum, prev_handler in self._previous.items():
732 signal.signal(signum, prev_handler)
733 # If successful, remove from dict
734 del self._previous[signum]
737 """Unsets the L{called} flag.
739 This function can be used in case a signal may arrive several times.
744 def _HandleSignal(self, signum, frame):
745 """Actual signal handling function.
748 # This is not nice and not absolutely atomic, but it appears to be the only
749 # solution in Python -- there are no atomic types.
753 # Notify whoever is interested in signals
754 self._wakeup.Notify()
757 self._handler_fn(signum, frame)
760 class FieldSet(object):
761 """A simple field set.
763 Among the features are:
764 - checking if a string is among a list of static string or regex objects
765 - checking if a whole list of string matches
766 - returning the matching groups from a regex match
768 Internally, all fields are held as regular expression objects.
771 def __init__(self, *items):
772 self.items = [re.compile("^%s$" % value) for value in items]
774 def Extend(self, other_set):
775 """Extend the field set with the items from another one"""
776 self.items.extend(other_set.items)
778 def Matches(self, field):
779 """Checks if a field matches the current set
782 @param field: the string to match
783 @return: either None or a regular expression match object
786 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
790 def NonMatching(self, items):
791 """Returns the list of fields not matching the current set
794 @param items: the list of fields to check
796 @return: list of non-matching fields
799 return [val for val in items if not self.Matches(val)]
802 def ValidateDeviceNames(kind, container):
803 """Validate instance device names.
805 Check that a device container contains only unique and valid names.
808 @param kind: One-word item description
809 @type container: list
810 @param container: Container containing the devices
815 for device in container:
816 if isinstance(device, dict):
818 name = device.get(constants.INIC_NAME, None)
820 name = device.get(constants.IDISK_NAME, None)
822 raise errors.OpPrereqError("Invalid container kind '%s'" % kind,
826 # Check that a device name is not the UUID of another device
827 valid.append(device.uuid)
831 except (ValueError, TypeError):
834 raise errors.OpPrereqError("Invalid name '%s'. Purely numeric %s names"
835 " are not allowed" % (name, kind),
838 if name is not None and name.lower() != constants.VALUE_NONE:
840 raise errors.OpPrereqError("%s name '%s' already used" % (kind, name),
841 errors.ECODE_NOTUNIQUE)