4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
29 # Allow wildcard import in pylint: disable=W0401
40 from ganeti import errors
41 from ganeti import constants
42 from ganeti import compat
44 from ganeti.utils.algo import *
45 from ganeti.utils.filelock import *
46 from ganeti.utils.hash import *
47 from ganeti.utils.io import *
48 from ganeti.utils.log import *
49 from ganeti.utils.mlock import *
50 from ganeti.utils.nodesetup import *
51 from ganeti.utils.process import *
52 from ganeti.utils.retry import *
53 from ganeti.utils.text import *
54 from ganeti.utils.wrapper import *
55 from ganeti.utils.x509 import *
58 _VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
60 UUID_RE = re.compile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-"
61 "[a-f0-9]{4}-[a-f0-9]{12}$")
64 def ForceDictType(target, key_types, allowed_values=None):
65 """Force the values of a dict to have certain types.
68 @param target: the dict to update
70 @param key_types: dict mapping target dict keys to types
71 in constants.ENFORCEABLE_TYPES
72 @type allowed_values: list
73 @keyword allowed_values: list of specially allowed values
76 if allowed_values is None:
79 if not isinstance(target, dict):
80 msg = "Expected dictionary, got '%s'" % target
81 raise errors.TypeEnforcementError(msg)
84 if key not in key_types:
85 msg = "Unknown parameter '%s'" % key
86 raise errors.TypeEnforcementError(msg)
88 if target[key] in allowed_values:
91 ktype = key_types[key]
92 if ktype not in constants.ENFORCEABLE_TYPES:
93 msg = "'%s' has non-enforceable type %s" % (key, ktype)
94 raise errors.ProgrammerError(msg)
96 if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
97 if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
99 elif not isinstance(target[key], basestring):
100 if isinstance(target[key], bool) and not target[key]:
103 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
104 raise errors.TypeEnforcementError(msg)
105 elif ktype == constants.VTYPE_BOOL:
106 if isinstance(target[key], basestring) and target[key]:
107 if target[key].lower() == constants.VALUE_FALSE:
109 elif target[key].lower() == constants.VALUE_TRUE:
112 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
113 raise errors.TypeEnforcementError(msg)
118 elif ktype == constants.VTYPE_SIZE:
120 target[key] = ParseUnit(target[key])
121 except errors.UnitParseError, err:
122 msg = "'%s' (value %s) is not a valid size. error: %s" % \
123 (key, target[key], err)
124 raise errors.TypeEnforcementError(msg)
125 elif ktype == constants.VTYPE_INT:
127 target[key] = int(target[key])
128 except (ValueError, TypeError):
129 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
130 raise errors.TypeEnforcementError(msg)
133 def ValidateServiceName(name):
134 """Validate the given service name.
136 @type name: number or string
137 @param name: Service name or port specification
142 except (ValueError, TypeError):
143 # Non-numeric service name
144 valid = _VALID_SERVICE_NAME_RE.match(name)
146 # Numeric port (protocols other than TCP or UDP might need adjustments
148 valid = (numport >= 0 and numport < (1 << 16))
151 raise errors.OpPrereqError("Invalid service name '%s'" % name,
157 def ListVolumeGroups():
158 """List volume groups and their size
162 Dictionary with keys volume name and values
163 the size of the volume
166 command = "vgs --noheadings --units m --nosuffix -o name,size"
167 result = RunCmd(command)
172 for line in result.stdout.splitlines():
174 name, size = line.split()
175 size = int(float(size))
176 except (IndexError, ValueError), err:
177 logging.error("Invalid output from vgs (%s): %s", err, line)
185 def BridgeExists(bridge):
186 """Check whether the given bridge exists in the system
189 @param bridge: the bridge name to check
191 @return: True if it does
194 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
197 def TryConvert(fn, val):
198 """Try to convert a value ignoring errors.
200 This function tries to apply function I{fn} to I{val}. If no
201 C{ValueError} or C{TypeError} exceptions are raised, it will return
202 the result, else it will return the original value. Any other
203 exceptions are propagated to the caller.
206 @param fn: function to apply to the value
207 @param val: the value to be converted
208 @return: The converted value if the conversion was successful,
209 otherwise the original value.
214 except (ValueError, TypeError):
219 def ParseCpuMask(cpu_mask):
220 """Parse a CPU mask definition and return the list of CPU IDs.
222 CPU mask format: comma-separated list of CPU IDs
223 or dash-separated ID ranges
224 Example: "0-2,5" -> "0,1,2,5"
227 @param cpu_mask: CPU mask definition
229 @return: list of CPU IDs
235 for range_def in cpu_mask.split(","):
236 boundaries = range_def.split("-")
237 n_elements = len(boundaries)
239 raise errors.ParseError("Invalid CPU ID range definition"
240 " (only one hyphen allowed): %s" % range_def)
242 lower = int(boundaries[0])
243 except (ValueError, TypeError), err:
244 raise errors.ParseError("Invalid CPU ID value for lower boundary of"
245 " CPU ID range: %s" % str(err))
247 higher = int(boundaries[-1])
248 except (ValueError, TypeError), err:
249 raise errors.ParseError("Invalid CPU ID value for higher boundary of"
250 " CPU ID range: %s" % str(err))
252 raise errors.ParseError("Invalid CPU ID range definition"
253 " (%d > %d): %s" % (lower, higher, range_def))
254 cpu_list.extend(range(lower, higher + 1))
258 def ParseMultiCpuMask(cpu_mask):
259 """Parse a multiple CPU mask definition and return the list of CPU IDs.
261 CPU mask format: colon-separated list of comma-separated list of CPU IDs
262 or dash-separated ID ranges, with optional "all" as CPU value
263 Example: "0-2,5:all:1,5,6:2" -> [ [ 0,1,2,5 ], [ -1 ], [ 1, 5, 6 ], [ 2 ] ]
266 @param cpu_mask: multiple CPU mask definition
267 @rtype: list of lists of int
268 @return: list of lists of CPU IDs
274 for range_def in cpu_mask.split(constants.CPU_PINNING_SEP):
275 if range_def == constants.CPU_PINNING_ALL:
276 cpu_list.append([constants.CPU_PINNING_ALL_VAL, ])
278 # Uniquify and sort the list before adding
279 cpu_list.append(sorted(set(ParseCpuMask(range_def))))
284 def GetHomeDir(user, default=None):
285 """Try to get the homedir of the given user.
287 The user can be passed either as a string (denoting the name) or as
288 an integer (denoting the user id). If the user is not found, the
289 'default' argument is returned, which defaults to None.
293 if isinstance(user, basestring):
294 result = pwd.getpwnam(user)
295 elif isinstance(user, (int, long)):
296 result = pwd.getpwuid(user)
298 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
305 def FirstFree(seq, base=0):
306 """Returns the first non-existing integer from seq.
308 The seq argument should be a sorted list of positive integers. The
309 first time the index of an element is smaller than the element
310 value, the index will be returned.
312 The base argument is used to start at a different offset,
313 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
315 Example: C{[0, 1, 3]} will return I{2}.
318 @param seq: the sequence to be analyzed.
320 @param base: use this value as the base index of the sequence
322 @return: the first non-used index in the sequence
325 for idx, elem in enumerate(seq):
326 assert elem >= base, "Passed element is higher than base offset"
327 if elem > idx + base:
333 def SingleWaitForFdCondition(fdobj, event, timeout):
334 """Waits for a condition to occur on the socket.
336 Immediately returns at the first interruption.
338 @type fdobj: integer or object supporting a fileno() method
339 @param fdobj: entity to wait for events on
341 @param event: ORed condition (see select module)
342 @type timeout: float or None
343 @param timeout: Timeout in seconds
345 @return: None for timeout, otherwise occured conditions
348 check = (event | select.POLLPRI |
349 select.POLLNVAL | select.POLLHUP | select.POLLERR)
351 if timeout is not None:
352 # Poller object expects milliseconds
355 poller = select.poll()
356 poller.register(fdobj, event)
358 # TODO: If the main thread receives a signal and we have no timeout, we
359 # could wait forever. This should check a global "quit" flag or something
361 io_events = poller.poll(timeout)
362 except select.error, err:
363 if err[0] != errno.EINTR:
366 if io_events and io_events[0][1] & check:
367 return io_events[0][1]
372 class FdConditionWaiterHelper(object):
373 """Retry helper for WaitForFdCondition.
375 This class contains the retried and wait functions that make sure
376 WaitForFdCondition can continue waiting until the timeout is actually
381 def __init__(self, timeout):
382 self.timeout = timeout
384 def Poll(self, fdobj, event):
385 result = SingleWaitForFdCondition(fdobj, event, self.timeout)
391 def UpdateTimeout(self, timeout):
392 self.timeout = timeout
395 def WaitForFdCondition(fdobj, event, timeout):
396 """Waits for a condition to occur on the socket.
398 Retries until the timeout is expired, even if interrupted.
400 @type fdobj: integer or object supporting a fileno() method
401 @param fdobj: entity to wait for events on
403 @param event: ORed condition (see select module)
404 @type timeout: float or None
405 @param timeout: Timeout in seconds
407 @return: None for timeout, otherwise occured conditions
410 if timeout is not None:
411 retrywaiter = FdConditionWaiterHelper(timeout)
413 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
414 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
419 while result is None:
420 result = SingleWaitForFdCondition(fdobj, event, timeout)
424 def EnsureDaemon(name):
425 """Check for and start daemon if not alive.
428 result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
430 logging.error("Can't start daemon '%s', failure %s, output: %s",
431 name, result.fail_reason, result.output)
437 def StopDaemon(name):
441 result = RunCmd([constants.DAEMON_UTIL, "stop", name])
443 logging.error("Can't stop daemon '%s', failure %s, output: %s",
444 name, result.fail_reason, result.output)
450 def CheckVolumeGroupSize(vglist, vgname, minsize):
451 """Checks if the volume group list is valid.
453 The function will check if a given volume group is in the list of
454 volume groups and has a minimum size.
457 @param vglist: dictionary of volume group names and their size
459 @param vgname: the volume group we should check
461 @param minsize: the minimum size we accept
463 @return: None for success, otherwise the error message
466 vgsize = vglist.get(vgname, None)
468 return "volume group '%s' missing" % vgname
469 elif vgsize < minsize:
470 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
471 (vgname, minsize, vgsize))
475 def SplitTime(value):
476 """Splits time as floating point number into a tuple.
478 @param value: Time in seconds
479 @type value: int or float
480 @return: Tuple containing (seconds, microseconds)
483 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
485 assert 0 <= seconds, \
486 "Seconds must be larger than or equal to 0, but are %s" % seconds
487 assert 0 <= microseconds <= 999999, \
488 "Microseconds must be 0-999999, but are %s" % microseconds
490 return (int(seconds), int(microseconds))
493 def MergeTime(timetuple):
494 """Merges a tuple into time as a floating point number.
496 @param timetuple: Time as tuple, (seconds, microseconds)
497 @type timetuple: tuple
498 @return: Time as a floating point number expressed in seconds
501 (seconds, microseconds) = timetuple
503 assert 0 <= seconds, \
504 "Seconds must be larger than or equal to 0, but are %s" % seconds
505 assert 0 <= microseconds <= 999999, \
506 "Microseconds must be 0-999999, but are %s" % microseconds
508 return float(seconds) + (float(microseconds) * 0.000001)
511 def FindMatch(data, name):
512 """Tries to find an item in a dictionary matching a name.
514 Callers have to ensure the data names aren't contradictory (e.g. a regexp
515 that matches a string). If the name isn't a direct key, all regular
516 expression objects in the dictionary are matched against it.
519 @param data: Dictionary containing data
521 @param name: Name to look for
522 @rtype: tuple; (value in dictionary, matched groups as list)
526 return (data[name], [])
528 for key, value in data.items():
530 if hasattr(key, "match"):
533 return (value, list(m.groups()))
538 def GetMounts(filename=constants.PROC_MOUNTS):
539 """Returns the list of mounted filesystems.
541 This function is Linux-specific.
543 @param filename: path of mounts file (/proc/mounts by default)
544 @rtype: list of tuples
545 @return: list of mount entries (device, mountpoint, fstype, options)
548 # TODO(iustin): investigate non-Linux options (e.g. via mount output)
550 mountlines = ReadFile(filename).splitlines()
551 for line in mountlines:
552 device, mountpoint, fstype, options, _ = line.split(None, 4)
553 data.append((device, mountpoint, fstype, options))
558 def SignalHandled(signums):
559 """Signal Handled decoration.
561 This special decorator installs a signal handler and then calls the target
562 function. The function must accept a 'signal_handlers' keyword argument,
563 which will contain a dict indexed by signal number, with SignalHandler
566 The decorator can be safely stacked with iself, to handle multiple signals
567 with different handlers.
570 @param signums: signals to intercept
574 def sig_function(*args, **kwargs):
575 assert "signal_handlers" not in kwargs or \
576 kwargs["signal_handlers"] is None or \
577 isinstance(kwargs["signal_handlers"], dict), \
578 "Wrong signal_handlers parameter in original function call"
579 if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
580 signal_handlers = kwargs["signal_handlers"]
583 kwargs["signal_handlers"] = signal_handlers
584 sighandler = SignalHandler(signums)
587 signal_handlers[sig] = sighandler
588 return fn(*args, **kwargs)
595 class SignalWakeupFd(object):
597 # This is only supported in Python 2.5 and above (some distributions
598 # backported it to Python 2.4)
599 _set_wakeup_fd_fn = signal.set_wakeup_fd
600 except AttributeError:
602 def _SetWakeupFd(self, _): # pylint: disable=R0201
605 def _SetWakeupFd(self, fd):
606 return self._set_wakeup_fd_fn(fd)
609 """Initializes this class.
612 (read_fd, write_fd) = os.pipe()
614 # Once these succeeded, the file descriptors will be closed automatically.
615 # Buffer size 0 is important, otherwise .read() with a specified length
616 # might buffer data and the file descriptors won't be marked readable.
617 self._read_fh = os.fdopen(read_fd, "r", 0)
618 self._write_fh = os.fdopen(write_fd, "w", 0)
620 self._previous = self._SetWakeupFd(self._write_fh.fileno())
623 self.fileno = self._read_fh.fileno
624 self.read = self._read_fh.read
627 """Restores the previous wakeup file descriptor.
630 if hasattr(self, "_previous") and self._previous is not None:
631 self._SetWakeupFd(self._previous)
632 self._previous = None
635 """Notifies the wakeup file descriptor.
638 self._write_fh.write("\0")
641 """Called before object deletion.
647 class SignalHandler(object):
648 """Generic signal handler class.
650 It automatically restores the original handler when deconstructed or
651 when L{Reset} is called. You can either pass your own handler
652 function in or query the L{called} attribute to detect whether the
656 @ivar signum: the signals we handle
657 @type called: boolean
658 @ivar called: tracks whether any of the signals have been raised
661 def __init__(self, signum, handler_fn=None, wakeup=None):
662 """Constructs a new SignalHandler instance.
664 @type signum: int or list of ints
665 @param signum: Single signal number or set of signal numbers
666 @type handler_fn: callable
667 @param handler_fn: Signal handling function
670 assert handler_fn is None or callable(handler_fn)
672 self.signum = set(signum)
675 self._handler_fn = handler_fn
676 self._wakeup = wakeup
680 for signum in self.signum:
682 prev_handler = signal.signal(signum, self._HandleSignal)
684 self._previous[signum] = prev_handler
686 # Restore previous handler
687 signal.signal(signum, prev_handler)
692 # Here we have a race condition: a handler may have already been called,
693 # but there's not much we can do about it at this point.
700 """Restore previous handler.
702 This will reset all the signals to their previous handlers.
705 for signum, prev_handler in self._previous.items():
706 signal.signal(signum, prev_handler)
707 # If successful, remove from dict
708 del self._previous[signum]
711 """Unsets the L{called} flag.
713 This function can be used in case a signal may arrive several times.
718 def _HandleSignal(self, signum, frame):
719 """Actual signal handling function.
722 # This is not nice and not absolutely atomic, but it appears to be the only
723 # solution in Python -- there are no atomic types.
727 # Notify whoever is interested in signals
728 self._wakeup.Notify()
731 self._handler_fn(signum, frame)
734 class FieldSet(object):
735 """A simple field set.
737 Among the features are:
738 - checking if a string is among a list of static string or regex objects
739 - checking if a whole list of string matches
740 - returning the matching groups from a regex match
742 Internally, all fields are held as regular expression objects.
745 def __init__(self, *items):
746 self.items = [re.compile("^%s$" % value) for value in items]
748 def Extend(self, other_set):
749 """Extend the field set with the items from another one"""
750 self.items.extend(other_set.items)
752 def Matches(self, field):
753 """Checks if a field matches the current set
756 @param field: the string to match
757 @return: either None or a regular expression match object
760 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
764 def NonMatching(self, items):
765 """Returns the list of fields not matching the current set
768 @param items: the list of fields to check
770 @return: list of non-matching fields
773 return [val for val in items if not self.Matches(val)]