4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
45 import logging.handlers
49 from cStringIO import StringIO
52 from hashlib import sha1
57 from ganeti import errors
58 from ganeti import constants
62 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
66 #: when set to True, L{RunCmd} is disabled
69 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
72 class RunResult(object):
73 """Holds the result of running external programs.
76 @ivar exit_code: the exit code of the program, or None (if the program
78 @type signal: int or None
79 @ivar signal: the signal that caused the program to finish, or None
80 (if the program wasn't terminated by a signal)
82 @ivar stdout: the standard output of the program
84 @ivar stderr: the standard error of the program
86 @ivar failed: True in case the program was
87 terminated by a signal or exited with a non-zero exit code
88 @ivar fail_reason: a string detailing the termination reason
91 __slots__ = ["exit_code", "signal", "stdout", "stderr",
92 "failed", "fail_reason", "cmd"]
95 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
97 self.exit_code = exit_code
101 self.failed = (signal_ is not None or exit_code != 0)
103 if self.signal is not None:
104 self.fail_reason = "terminated by signal %s" % self.signal
105 elif self.exit_code is not None:
106 self.fail_reason = "exited with exit code %s" % self.exit_code
108 self.fail_reason = "unable to determine termination reason"
111 logging.debug("Command '%s' failed (%s); output: %s",
112 self.cmd, self.fail_reason, self.output)
114 def _GetOutput(self):
115 """Returns the combined stdout and stderr for easier usage.
118 return self.stdout + self.stderr
120 output = property(_GetOutput, None, None, "Return full output")
123 def _BuildCmdEnvironment(env):
124 """Builds the environment for an external program.
127 cmd_env = os.environ.copy()
128 cmd_env["LC_ALL"] = "C"
134 def RunCmd(cmd, env=None, output=None, cwd="/"):
135 """Execute a (shell) command.
137 The command should not read from its standard input, as it will be
140 @type cmd: string or list
141 @param cmd: Command to run
143 @param env: Additional environment variables
145 @param output: if desired, the output of the command can be
146 saved in a file instead of the RunResult instance; this
147 parameter denotes the file name (if not None)
149 @param cwd: if specified, will be used as the working
150 directory for the command; the default will be /
152 @return: RunResult instance
153 @raise errors.ProgrammerError: if we call this when forks are disabled
157 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
159 if isinstance(cmd, basestring):
163 cmd = [str(val) for val in cmd]
164 strcmd = ShellQuoteArgs(cmd)
168 logging.debug("RunCmd %s, output file '%s'", strcmd, output)
170 logging.debug("RunCmd %s", strcmd)
172 cmd_env = _BuildCmdEnvironment(env)
176 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
178 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
181 if err.errno == errno.ENOENT:
182 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
194 return RunResult(exitcode, signal_, out, err, strcmd)
197 def StartDaemon(cmd, env=None, cwd="/", output=None, output_fd=None,
199 """Start a daemon process after forking twice.
201 @type cmd: string or list
202 @param cmd: Command to run
204 @param env: Additional environment variables
206 @param cwd: Working directory for the program
208 @param output: Path to file in which to save the output
210 @param output_fd: File descriptor for output
211 @type pidfile: string
212 @param pidfile: Process ID file
214 @return: Daemon process ID
215 @raise errors.ProgrammerError: if we call this when forks are disabled
219 raise errors.ProgrammerError("utils.StartDaemon() called with fork()"
222 if output and not (bool(output) ^ (output_fd is not None)):
223 raise errors.ProgrammerError("Only one of 'output' and 'output_fd' can be"
226 if isinstance(cmd, basestring):
227 cmd = ["/bin/sh", "-c", cmd]
229 strcmd = ShellQuoteArgs(cmd)
232 logging.debug("StartDaemon %s, output file '%s'", strcmd, output)
234 logging.debug("StartDaemon %s", strcmd)
236 cmd_env = _BuildCmdEnvironment(env)
238 # Create pipe for sending PID back
239 (pidpipe_read, pidpipe_write) = os.pipe()
242 # Create pipe for sending error messages
243 (errpipe_read, errpipe_write) = os.pipe()
250 # Child process, won't return
251 _RunCmdDaemonChild(errpipe_read, errpipe_write,
252 pidpipe_read, pidpipe_write,
254 output, output_fd, pidfile)
256 # Well, maybe child process failed
259 _CloseFDNoErr(errpipe_write)
261 # Wait for daemon to be started (or an error message to arrive) and read
262 # up to 100 KB as an error message
263 errormsg = RetryOnSignal(os.read, errpipe_read, 100 * 1024)
265 _CloseFDNoErr(errpipe_read)
267 _CloseFDNoErr(pidpipe_write)
269 # Read up to 128 bytes for PID
270 pidtext = RetryOnSignal(os.read, pidpipe_read, 128)
272 _CloseFDNoErr(pidpipe_read)
274 # Try to avoid zombies by waiting for child process
281 raise errors.OpExecError("Error when starting daemon process: %r" %
286 except (ValueError, TypeError), err:
287 raise errors.OpExecError("Error while trying to parse PID %r: %s" %
291 def _RunCmdDaemonChild(errpipe_read, errpipe_write,
292 pidpipe_read, pidpipe_write,
294 output, fd_output, pidfile):
295 """Child process for starting daemon.
299 # Close parent's side
300 _CloseFDNoErr(errpipe_read)
301 _CloseFDNoErr(pidpipe_read)
303 # First child process
308 # And fork for the second time
311 # Exit first child process
312 os._exit(0) # pylint: disable-msg=W0212
314 # Make sure pipe is closed on execv* (and thereby notifies original process)
315 SetCloseOnExecFlag(errpipe_write, True)
317 # List of file descriptors to be left open
318 noclose_fds = [errpipe_write]
323 # TODO: Atomic replace with another locked file instead of writing into
325 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
327 # Lock the PID file (and fail if not possible to do so). Any code
328 # wanting to send a signal to the daemon should try to lock the PID
329 # file before reading it. If acquiring the lock succeeds, the daemon is
330 # no longer running and the signal should not be sent.
333 os.write(fd_pidfile, "%d\n" % os.getpid())
334 except Exception, err:
335 raise Exception("Creating and locking PID file failed: %s" % err)
337 # Keeping the file open to hold the lock
338 noclose_fds.append(fd_pidfile)
340 SetCloseOnExecFlag(fd_pidfile, False)
345 fd_devnull = os.open(os.devnull, os.O_RDWR)
347 assert not output or (bool(output) ^ (fd_output is not None))
349 if fd_output is not None:
354 # TODO: Implement flag to set append=yes/no
355 fd_output = os.open(output, os.O_WRONLY | os.O_CREAT, 0600)
356 except EnvironmentError, err:
357 raise Exception("Opening output file failed: %s" % err)
359 fd_output = fd_devnull
361 # Redirect standard I/O
362 os.dup2(fd_devnull, 0)
363 os.dup2(fd_output, 1)
364 os.dup2(fd_output, 2)
366 # Send daemon PID to parent
367 RetryOnSignal(os.write, pidpipe_write, str(os.getpid()))
369 # Close all file descriptors except stdio and error message pipe
370 CloseFDs(noclose_fds=noclose_fds)
372 # Change working directory
376 os.execvp(args[0], args)
378 os.execvpe(args[0], args, env)
379 except: # pylint: disable-msg=W0702
381 # Report errors to original process
382 buf = str(sys.exc_info()[1])
384 RetryOnSignal(os.write, errpipe_write, buf)
385 except: # pylint: disable-msg=W0702
386 # Ignore errors in error handling
389 os._exit(1) # pylint: disable-msg=W0212
392 def _RunCmdPipe(cmd, env, via_shell, cwd):
393 """Run a command and return its output.
395 @type cmd: string or list
396 @param cmd: Command to run
398 @param env: The environment to use
399 @type via_shell: bool
400 @param via_shell: if we should run via the shell
402 @param cwd: the working directory for the program
404 @return: (out, err, status)
407 poller = select.poll()
408 child = subprocess.Popen(cmd, shell=via_shell,
409 stderr=subprocess.PIPE,
410 stdout=subprocess.PIPE,
411 stdin=subprocess.PIPE,
412 close_fds=True, env=env,
416 poller.register(child.stdout, select.POLLIN)
417 poller.register(child.stderr, select.POLLIN)
421 child.stdout.fileno(): (out, child.stdout),
422 child.stderr.fileno(): (err, child.stderr),
425 SetNonblockFlag(fd, True)
428 pollresult = RetryOnSignal(poller.poll)
430 for fd, event in pollresult:
431 if event & select.POLLIN or event & select.POLLPRI:
432 data = fdmap[fd][1].read()
433 # no data from read signifies EOF (the same as POLLHUP)
435 poller.unregister(fd)
438 fdmap[fd][0].write(data)
439 if (event & select.POLLNVAL or event & select.POLLHUP or
440 event & select.POLLERR):
441 poller.unregister(fd)
447 status = child.wait()
448 return out, err, status
451 def _RunCmdFile(cmd, env, via_shell, output, cwd):
452 """Run a command and save its output to a file.
454 @type cmd: string or list
455 @param cmd: Command to run
457 @param env: The environment to use
458 @type via_shell: bool
459 @param via_shell: if we should run via the shell
461 @param output: the filename in which to save the output
463 @param cwd: the working directory for the program
465 @return: the exit status
468 fh = open(output, "a")
470 child = subprocess.Popen(cmd, shell=via_shell,
471 stderr=subprocess.STDOUT,
473 stdin=subprocess.PIPE,
474 close_fds=True, env=env,
478 status = child.wait()
484 def SetCloseOnExecFlag(fd, enable):
485 """Sets or unsets the close-on-exec flag on a file descriptor.
488 @param fd: File descriptor
490 @param enable: Whether to set or unset it.
493 flags = fcntl.fcntl(fd, fcntl.F_GETFD)
496 flags |= fcntl.FD_CLOEXEC
498 flags &= ~fcntl.FD_CLOEXEC
500 fcntl.fcntl(fd, fcntl.F_SETFD, flags)
503 def SetNonblockFlag(fd, enable):
504 """Sets or unsets the O_NONBLOCK flag on on a file descriptor.
507 @param fd: File descriptor
509 @param enable: Whether to set or unset it
512 flags = fcntl.fcntl(fd, fcntl.F_GETFL)
515 flags |= os.O_NONBLOCK
517 flags &= ~os.O_NONBLOCK
519 fcntl.fcntl(fd, fcntl.F_SETFL, flags)
522 def RetryOnSignal(fn, *args, **kwargs):
523 """Calls a function again if it failed due to EINTR.
528 return fn(*args, **kwargs)
529 except EnvironmentError, err:
530 if err.errno != errno.EINTR:
532 except select.error, err:
533 if not (err.args and err.args[0] == errno.EINTR):
537 def RemoveFile(filename):
538 """Remove a file ignoring some errors.
540 Remove a file, ignoring non-existing ones or directories. Other
544 @param filename: the file to be removed
550 if err.errno not in (errno.ENOENT, errno.EISDIR):
554 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
558 @param old: Original path
562 @param mkdir: Whether to create target directory if it doesn't exist
563 @type mkdir_mode: int
564 @param mkdir_mode: Mode for newly created directories
568 return os.rename(old, new)
570 # In at least one use case of this function, the job queue, directory
571 # creation is very rare. Checking for the directory before renaming is not
573 if mkdir and err.errno == errno.ENOENT:
574 # Create directory and try again
575 dirname = os.path.dirname(new)
577 os.makedirs(dirname, mode=mkdir_mode)
579 # Ignore EEXIST. This is only handled in os.makedirs as included in
580 # Python 2.5 and above.
581 if err.errno != errno.EEXIST or not os.path.exists(dirname):
584 return os.rename(old, new)
589 def ResetTempfileModule():
590 """Resets the random name generator of the tempfile module.
592 This function should be called after C{os.fork} in the child process to
593 ensure it creates a newly seeded random generator. Otherwise it would
594 generate the same random parts as the parent process. If several processes
595 race for the creation of a temporary file, this could lead to one not getting
599 # pylint: disable-msg=W0212
600 if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
601 tempfile._once_lock.acquire()
603 # Reset random name generator
604 tempfile._name_sequence = None
606 tempfile._once_lock.release()
608 logging.critical("The tempfile module misses at least one of the"
609 " '_once_lock' and '_name_sequence' attributes")
612 def _FingerprintFile(filename):
613 """Compute the fingerprint of a file.
615 If the file does not exist, a None will be returned
619 @param filename: the filename to checksum
621 @return: the hex digest of the sha checksum of the contents
625 if not (os.path.exists(filename) and os.path.isfile(filename)):
638 return fp.hexdigest()
641 def FingerprintFiles(files):
642 """Compute fingerprints for a list of files.
645 @param files: the list of filename to fingerprint
647 @return: a dictionary filename: fingerprint, holding only
653 for filename in files:
654 cksum = _FingerprintFile(filename)
656 ret[filename] = cksum
661 def ForceDictType(target, key_types, allowed_values=None):
662 """Force the values of a dict to have certain types.
665 @param target: the dict to update
666 @type key_types: dict
667 @param key_types: dict mapping target dict keys to types
668 in constants.ENFORCEABLE_TYPES
669 @type allowed_values: list
670 @keyword allowed_values: list of specially allowed values
673 if allowed_values is None:
676 if not isinstance(target, dict):
677 msg = "Expected dictionary, got '%s'" % target
678 raise errors.TypeEnforcementError(msg)
681 if key not in key_types:
682 msg = "Unknown key '%s'" % key
683 raise errors.TypeEnforcementError(msg)
685 if target[key] in allowed_values:
688 ktype = key_types[key]
689 if ktype not in constants.ENFORCEABLE_TYPES:
690 msg = "'%s' has non-enforceable type %s" % (key, ktype)
691 raise errors.ProgrammerError(msg)
693 if ktype == constants.VTYPE_STRING:
694 if not isinstance(target[key], basestring):
695 if isinstance(target[key], bool) and not target[key]:
698 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
699 raise errors.TypeEnforcementError(msg)
700 elif ktype == constants.VTYPE_BOOL:
701 if isinstance(target[key], basestring) and target[key]:
702 if target[key].lower() == constants.VALUE_FALSE:
704 elif target[key].lower() == constants.VALUE_TRUE:
707 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
708 raise errors.TypeEnforcementError(msg)
713 elif ktype == constants.VTYPE_SIZE:
715 target[key] = ParseUnit(target[key])
716 except errors.UnitParseError, err:
717 msg = "'%s' (value %s) is not a valid size. error: %s" % \
718 (key, target[key], err)
719 raise errors.TypeEnforcementError(msg)
720 elif ktype == constants.VTYPE_INT:
722 target[key] = int(target[key])
723 except (ValueError, TypeError):
724 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
725 raise errors.TypeEnforcementError(msg)
728 def IsProcessAlive(pid):
729 """Check if a given pid exists on the system.
731 @note: zombie status is not handled, so zombie processes
732 will be returned as alive
734 @param pid: the process ID to check
736 @return: True if the process exists
743 os.stat("/proc/%d/status" % pid)
745 except EnvironmentError, err:
746 if err.errno in (errno.ENOENT, errno.ENOTDIR):
751 def ReadPidFile(pidfile):
752 """Read a pid from a file.
754 @type pidfile: string
755 @param pidfile: path to the file containing the pid
757 @return: The process id, if the file exists and contains a valid PID,
762 raw_data = ReadFile(pidfile)
763 except EnvironmentError, err:
764 if err.errno != errno.ENOENT:
765 logging.exception("Can't read pid file")
770 except (TypeError, ValueError), err:
771 logging.info("Can't parse pid file contents", exc_info=True)
777 def MatchNameComponent(key, name_list, case_sensitive=True):
778 """Try to match a name against a list.
780 This function will try to match a name like test1 against a list
781 like C{['test1.example.com', 'test2.example.com', ...]}. Against
782 this list, I{'test1'} as well as I{'test1.example'} will match, but
783 not I{'test1.ex'}. A multiple match will be considered as no match
784 at all (e.g. I{'test1'} against C{['test1.example.com',
785 'test1.example.org']}), except when the key fully matches an entry
786 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
789 @param key: the name to be searched
790 @type name_list: list
791 @param name_list: the list of strings against which to search the key
792 @type case_sensitive: boolean
793 @param case_sensitive: whether to provide a case-sensitive match
796 @return: None if there is no match I{or} if there are multiple matches,
797 otherwise the element from the list which matches
804 if not case_sensitive:
805 re_flags |= re.IGNORECASE
807 mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
810 for name in name_list:
811 if mo.match(name) is not None:
812 names_filtered.append(name)
813 if not case_sensitive and key == name.upper():
814 string_matches.append(name)
816 if len(string_matches) == 1:
817 return string_matches[0]
818 if len(names_filtered) == 1:
819 return names_filtered[0]
824 """Class implementing resolver and hostname functionality
827 def __init__(self, name=None):
828 """Initialize the host name object.
830 If the name argument is not passed, it will use this system's
835 name = self.SysName()
838 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
839 self.ip = self.ipaddrs[0]
842 """Returns the hostname without domain.
845 return self.name.split('.')[0]
849 """Return the current system's name.
851 This is simply a wrapper over C{socket.gethostname()}.
854 return socket.gethostname()
857 def LookupHostname(hostname):
861 @param hostname: hostname to look up
864 @return: a tuple (name, aliases, ipaddrs) as returned by
865 C{socket.gethostbyname_ex}
866 @raise errors.ResolverError: in case of errors in resolving
870 result = socket.gethostbyname_ex(hostname)
871 except socket.gaierror, err:
872 # hostname not found in DNS
873 raise errors.ResolverError(hostname, err.args[0], err.args[1])
878 def GetHostInfo(name=None):
879 """Lookup host name and raise an OpPrereqError for failures"""
882 return HostInfo(name)
883 except errors.ResolverError, err:
884 raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
885 (err[0], err[2]), errors.ECODE_RESOLVER)
888 def ListVolumeGroups():
889 """List volume groups and their size
893 Dictionary with keys volume name and values
894 the size of the volume
897 command = "vgs --noheadings --units m --nosuffix -o name,size"
898 result = RunCmd(command)
903 for line in result.stdout.splitlines():
905 name, size = line.split()
906 size = int(float(size))
907 except (IndexError, ValueError), err:
908 logging.error("Invalid output from vgs (%s): %s", err, line)
916 def BridgeExists(bridge):
917 """Check whether the given bridge exists in the system
920 @param bridge: the bridge name to check
922 @return: True if it does
925 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
928 def NiceSort(name_list):
929 """Sort a list of strings based on digit and non-digit groupings.
931 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
932 will sort the list in the logical order C{['a1', 'a2', 'a10',
935 The sort algorithm breaks each name in groups of either only-digits
936 or no-digits. Only the first eight such groups are considered, and
937 after that we just use what's left of the string.
939 @type name_list: list
940 @param name_list: the names to be sorted
942 @return: a copy of the name list sorted with our algorithm
945 _SORTER_BASE = "(\D+|\d+)"
946 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
947 _SORTER_BASE, _SORTER_BASE,
948 _SORTER_BASE, _SORTER_BASE,
949 _SORTER_BASE, _SORTER_BASE)
950 _SORTER_RE = re.compile(_SORTER_FULL)
951 _SORTER_NODIGIT = re.compile("^\D*$")
953 """Attempts to convert a variable to integer."""
954 if val is None or _SORTER_NODIGIT.match(val):
959 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
960 for name in name_list]
962 return [tup[1] for tup in to_sort]
965 def TryConvert(fn, val):
966 """Try to convert a value ignoring errors.
968 This function tries to apply function I{fn} to I{val}. If no
969 C{ValueError} or C{TypeError} exceptions are raised, it will return
970 the result, else it will return the original value. Any other
971 exceptions are propagated to the caller.
974 @param fn: function to apply to the value
975 @param val: the value to be converted
976 @return: The converted value if the conversion was successful,
977 otherwise the original value.
982 except (ValueError, TypeError):
988 """Verifies the syntax of an IPv4 address.
990 This function checks if the IPv4 address passes is valid or not based
991 on syntax (not IP range, class calculations, etc.).
994 @param ip: the address to be checked
995 @rtype: a regular expression match object
996 @return: a regular expression match object, or None if the
1000 unit = "(0|[1-9]\d{0,2})"
1001 #TODO: convert and return only boolean
1002 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
1005 def IsValidShellParam(word):
1006 """Verifies is the given word is safe from the shell's p.o.v.
1008 This means that we can pass this to a command via the shell and be
1009 sure that it doesn't alter the command line and is passed as such to
1012 Note that we are overly restrictive here, in order to be on the safe
1016 @param word: the word to check
1018 @return: True if the word is 'safe'
1021 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
1024 def BuildShellCmd(template, *args):
1025 """Build a safe shell command line from the given arguments.
1027 This function will check all arguments in the args list so that they
1028 are valid shell parameters (i.e. they don't contain shell
1029 metacharacters). If everything is ok, it will return the result of
1033 @param template: the string holding the template for the
1036 @return: the expanded command line
1040 if not IsValidShellParam(word):
1041 raise errors.ProgrammerError("Shell argument '%s' contains"
1042 " invalid characters" % word)
1043 return template % args
1046 def FormatUnit(value, units):
1047 """Formats an incoming number of MiB with the appropriate unit.
1050 @param value: integer representing the value in MiB (1048576)
1052 @param units: the type of formatting we should do:
1053 - 'h' for automatic scaling
1058 @return: the formatted value (with suffix)
1061 if units not in ('m', 'g', 't', 'h'):
1062 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
1066 if units == 'm' or (units == 'h' and value < 1024):
1069 return "%d%s" % (round(value, 0), suffix)
1071 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
1074 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
1079 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
1082 def ParseUnit(input_string):
1083 """Tries to extract number and scale from the given string.
1085 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
1086 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
1087 is always an int in MiB.
1090 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
1092 raise errors.UnitParseError("Invalid format")
1094 value = float(m.groups()[0])
1096 unit = m.groups()[1]
1098 lcunit = unit.lower()
1102 if lcunit in ('m', 'mb', 'mib'):
1103 # Value already in MiB
1106 elif lcunit in ('g', 'gb', 'gib'):
1109 elif lcunit in ('t', 'tb', 'tib'):
1110 value *= 1024 * 1024
1113 raise errors.UnitParseError("Unknown unit: %s" % unit)
1115 # Make sure we round up
1116 if int(value) < value:
1119 # Round up to the next multiple of 4
1122 value += 4 - value % 4
1127 def AddAuthorizedKey(file_name, key):
1128 """Adds an SSH public key to an authorized_keys file.
1130 @type file_name: str
1131 @param file_name: path to authorized_keys file
1133 @param key: string containing key
1136 key_fields = key.split()
1138 f = open(file_name, 'a+')
1142 # Ignore whitespace changes
1143 if line.split() == key_fields:
1145 nl = line.endswith('\n')
1149 f.write(key.rstrip('\r\n'))
1156 def RemoveAuthorizedKey(file_name, key):
1157 """Removes an SSH public key from an authorized_keys file.
1159 @type file_name: str
1160 @param file_name: path to authorized_keys file
1162 @param key: string containing key
1165 key_fields = key.split()
1167 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1169 out = os.fdopen(fd, 'w')
1171 f = open(file_name, 'r')
1174 # Ignore whitespace changes while comparing lines
1175 if line.split() != key_fields:
1179 os.rename(tmpname, file_name)
1189 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1190 """Sets the name of an IP address and hostname in /etc/hosts.
1192 @type file_name: str
1193 @param file_name: path to the file to modify (usually C{/etc/hosts})
1195 @param ip: the IP address
1197 @param hostname: the hostname to be added
1199 @param aliases: the list of aliases to add for the hostname
1202 # FIXME: use WriteFile + fn rather than duplicating its efforts
1203 # Ensure aliases are unique
1204 aliases = UniqueSequence([hostname] + aliases)[1:]
1206 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1208 out = os.fdopen(fd, 'w')
1210 f = open(file_name, 'r')
1213 fields = line.split()
1214 if fields and not fields[0].startswith('#') and ip == fields[0]:
1218 out.write("%s\t%s" % (ip, hostname))
1220 out.write(" %s" % ' '.join(aliases))
1225 os.chmod(tmpname, 0644)
1226 os.rename(tmpname, file_name)
1236 def AddHostToEtcHosts(hostname):
1237 """Wrapper around SetEtcHostsEntry.
1240 @param hostname: a hostname that will be resolved and added to
1241 L{constants.ETC_HOSTS}
1244 hi = HostInfo(name=hostname)
1245 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1248 def RemoveEtcHostsEntry(file_name, hostname):
1249 """Removes a hostname from /etc/hosts.
1251 IP addresses without names are removed from the file.
1253 @type file_name: str
1254 @param file_name: path to the file to modify (usually C{/etc/hosts})
1256 @param hostname: the hostname to be removed
1259 # FIXME: use WriteFile + fn rather than duplicating its efforts
1260 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1262 out = os.fdopen(fd, 'w')
1264 f = open(file_name, 'r')
1267 fields = line.split()
1268 if len(fields) > 1 and not fields[0].startswith('#'):
1270 if hostname in names:
1271 while hostname in names:
1272 names.remove(hostname)
1274 out.write("%s %s\n" % (fields[0], ' '.join(names)))
1281 os.chmod(tmpname, 0644)
1282 os.rename(tmpname, file_name)
1292 def RemoveHostFromEtcHosts(hostname):
1293 """Wrapper around RemoveEtcHostsEntry.
1296 @param hostname: hostname that will be resolved and its
1297 full and shot name will be removed from
1298 L{constants.ETC_HOSTS}
1301 hi = HostInfo(name=hostname)
1302 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1303 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1306 def CreateBackup(file_name):
1307 """Creates a backup of a file.
1309 @type file_name: str
1310 @param file_name: file to be backed up
1312 @return: the path to the newly created backup
1313 @raise errors.ProgrammerError: for invalid file names
1316 if not os.path.isfile(file_name):
1317 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1320 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1321 dir_name = os.path.dirname(file_name)
1323 fsrc = open(file_name, 'rb')
1325 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1326 fdst = os.fdopen(fd, 'wb')
1328 shutil.copyfileobj(fsrc, fdst)
1337 def ShellQuote(value):
1338 """Quotes shell argument according to POSIX.
1341 @param value: the argument to be quoted
1343 @return: the quoted value
1346 if _re_shell_unquoted.match(value):
1349 return "'%s'" % value.replace("'", "'\\''")
1352 def ShellQuoteArgs(args):
1353 """Quotes a list of shell arguments.
1356 @param args: list of arguments to be quoted
1358 @return: the quoted arguments concatenated with spaces
1361 return ' '.join([ShellQuote(i) for i in args])
1364 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1365 """Simple ping implementation using TCP connect(2).
1367 Check if the given IP is reachable by doing attempting a TCP connect
1371 @param target: the IP or hostname to ping
1373 @param port: the port to connect to
1375 @param timeout: the timeout on the connection attempt
1376 @type live_port_needed: boolean
1377 @param live_port_needed: whether a closed port will cause the
1378 function to return failure, as if there was a timeout
1379 @type source: str or None
1380 @param source: if specified, will cause the connect to be made
1381 from this specific source address; failures to bind other
1382 than C{EADDRNOTAVAIL} will be ignored
1385 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1389 if source is not None:
1391 sock.bind((source, 0))
1392 except socket.error, (errcode, _):
1393 if errcode == errno.EADDRNOTAVAIL:
1396 sock.settimeout(timeout)
1399 sock.connect((target, port))
1402 except socket.timeout:
1404 except socket.error, (errcode, _):
1405 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1410 def OwnIpAddress(address):
1411 """Check if the current host has the the given IP address.
1413 Currently this is done by TCP-pinging the address from the loopback
1416 @type address: string
1417 @param address: the address to check
1419 @return: True if we own the address
1422 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1423 source=constants.LOCALHOST_IP_ADDRESS)
1426 def ListVisibleFiles(path):
1427 """Returns a list of visible files in a directory.
1430 @param path: the directory to enumerate
1432 @return: the list of all files not starting with a dot
1435 files = [i for i in os.listdir(path) if not i.startswith(".")]
1440 def GetHomeDir(user, default=None):
1441 """Try to get the homedir of the given user.
1443 The user can be passed either as a string (denoting the name) or as
1444 an integer (denoting the user id). If the user is not found, the
1445 'default' argument is returned, which defaults to None.
1449 if isinstance(user, basestring):
1450 result = pwd.getpwnam(user)
1451 elif isinstance(user, (int, long)):
1452 result = pwd.getpwuid(user)
1454 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1458 return result.pw_dir
1462 """Returns a random UUID.
1464 @note: This is a Linux-specific method as it uses the /proc
1469 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1472 def GenerateSecret(numbytes=20):
1473 """Generates a random secret.
1475 This will generate a pseudo-random secret returning an hex string
1476 (so that it can be used where an ASCII string is needed).
1478 @param numbytes: the number of bytes which will be represented by the returned
1479 string (defaulting to 20, the length of a SHA1 hash)
1481 @return: an hex representation of the pseudo-random sequence
1484 return os.urandom(numbytes).encode('hex')
1487 def EnsureDirs(dirs):
1488 """Make required directories, if they don't exist.
1490 @param dirs: list of tuples (dir_name, dir_mode)
1491 @type dirs: list of (string, integer)
1494 for dir_name, dir_mode in dirs:
1496 os.mkdir(dir_name, dir_mode)
1497 except EnvironmentError, err:
1498 if err.errno != errno.EEXIST:
1499 raise errors.GenericError("Cannot create needed directory"
1500 " '%s': %s" % (dir_name, err))
1501 if not os.path.isdir(dir_name):
1502 raise errors.GenericError("%s is not a directory" % dir_name)
1505 def ReadFile(file_name, size=-1):
1509 @param size: Read at most size bytes (if negative, entire file)
1511 @return: the (possibly partial) content of the file
1514 f = open(file_name, "r")
1521 def WriteFile(file_name, fn=None, data=None,
1522 mode=None, uid=-1, gid=-1,
1523 atime=None, mtime=None, close=True,
1524 dry_run=False, backup=False,
1525 prewrite=None, postwrite=None):
1526 """(Over)write a file atomically.
1528 The file_name and either fn (a function taking one argument, the
1529 file descriptor, and which should write the data to it) or data (the
1530 contents of the file) must be passed. The other arguments are
1531 optional and allow setting the file mode, owner and group, and the
1532 mtime/atime of the file.
1534 If the function doesn't raise an exception, it has succeeded and the
1535 target file has the new contents. If the function has raised an
1536 exception, an existing target file should be unmodified and the
1537 temporary file should be removed.
1539 @type file_name: str
1540 @param file_name: the target filename
1542 @param fn: content writing function, called with
1543 file descriptor as parameter
1545 @param data: contents of the file
1547 @param mode: file mode
1549 @param uid: the owner of the file
1551 @param gid: the group of the file
1553 @param atime: a custom access time to be set on the file
1555 @param mtime: a custom modification time to be set on the file
1556 @type close: boolean
1557 @param close: whether to close file after writing it
1558 @type prewrite: callable
1559 @param prewrite: function to be called before writing content
1560 @type postwrite: callable
1561 @param postwrite: function to be called after writing content
1564 @return: None if the 'close' parameter evaluates to True,
1565 otherwise the file descriptor
1567 @raise errors.ProgrammerError: if any of the arguments are not valid
1570 if not os.path.isabs(file_name):
1571 raise errors.ProgrammerError("Path passed to WriteFile is not"
1572 " absolute: '%s'" % file_name)
1574 if [fn, data].count(None) != 1:
1575 raise errors.ProgrammerError("fn or data required")
1577 if [atime, mtime].count(None) == 1:
1578 raise errors.ProgrammerError("Both atime and mtime must be either"
1581 if backup and not dry_run and os.path.isfile(file_name):
1582 CreateBackup(file_name)
1584 dir_name, base_name = os.path.split(file_name)
1585 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1587 # here we need to make sure we remove the temp file, if any error
1588 # leaves it in place
1590 if uid != -1 or gid != -1:
1591 os.chown(new_name, uid, gid)
1593 os.chmod(new_name, mode)
1594 if callable(prewrite):
1596 if data is not None:
1600 if callable(postwrite):
1603 if atime is not None and mtime is not None:
1604 os.utime(new_name, (atime, mtime))
1606 os.rename(new_name, file_name)
1615 RemoveFile(new_name)
1620 def FirstFree(seq, base=0):
1621 """Returns the first non-existing integer from seq.
1623 The seq argument should be a sorted list of positive integers. The
1624 first time the index of an element is smaller than the element
1625 value, the index will be returned.
1627 The base argument is used to start at a different offset,
1628 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1630 Example: C{[0, 1, 3]} will return I{2}.
1633 @param seq: the sequence to be analyzed.
1635 @param base: use this value as the base index of the sequence
1637 @return: the first non-used index in the sequence
1640 for idx, elem in enumerate(seq):
1641 assert elem >= base, "Passed element is higher than base offset"
1642 if elem > idx + base:
1648 def all(seq, pred=bool): # pylint: disable-msg=W0622
1649 "Returns True if pred(x) is True for every element in the iterable"
1650 for _ in itertools.ifilterfalse(pred, seq):
1655 def any(seq, pred=bool): # pylint: disable-msg=W0622
1656 "Returns True if pred(x) is True for at least one element in the iterable"
1657 for _ in itertools.ifilter(pred, seq):
1662 def UniqueSequence(seq):
1663 """Returns a list with unique elements.
1665 Element order is preserved.
1668 @param seq: the sequence with the source elements
1670 @return: list of unique elements from seq
1674 return [i for i in seq if i not in seen and not seen.add(i)]
1677 def NormalizeAndValidateMac(mac):
1678 """Normalizes and check if a MAC address is valid.
1680 Checks whether the supplied MAC address is formally correct, only
1681 accepts colon separated format. Normalize it to all lower.
1684 @param mac: the MAC to be validated
1686 @return: returns the normalized and validated MAC.
1688 @raise errors.OpPrereqError: If the MAC isn't valid
1691 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1692 if not mac_check.match(mac):
1693 raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1694 mac, errors.ECODE_INVAL)
1699 def TestDelay(duration):
1700 """Sleep for a fixed amount of time.
1702 @type duration: float
1703 @param duration: the sleep duration
1705 @return: False for negative value, True otherwise
1709 return False, "Invalid sleep duration"
1710 time.sleep(duration)
1714 def _CloseFDNoErr(fd, retries=5):
1715 """Close a file descriptor ignoring errors.
1718 @param fd: the file descriptor
1720 @param retries: how many retries to make, in case we get any
1721 other error than EBADF
1726 except OSError, err:
1727 if err.errno != errno.EBADF:
1729 _CloseFDNoErr(fd, retries - 1)
1730 # else either it's closed already or we're out of retries, so we
1731 # ignore this and go on
1734 def CloseFDs(noclose_fds=None):
1735 """Close file descriptors.
1737 This closes all file descriptors above 2 (i.e. except
1740 @type noclose_fds: list or None
1741 @param noclose_fds: if given, it denotes a list of file descriptor
1742 that should not be closed
1745 # Default maximum for the number of available file descriptors.
1746 if 'SC_OPEN_MAX' in os.sysconf_names:
1748 MAXFD = os.sysconf('SC_OPEN_MAX')
1755 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1756 if (maxfd == resource.RLIM_INFINITY):
1759 # Iterate through and close all file descriptors (except the standard ones)
1760 for fd in range(3, maxfd):
1761 if noclose_fds and fd in noclose_fds:
1766 def Daemonize(logfile):
1767 """Daemonize the current process.
1769 This detaches the current process from the controlling terminal and
1770 runs it in the background as a daemon.
1773 @param logfile: the logfile to which we should redirect stdout/stderr
1775 @return: the value zero
1778 # pylint: disable-msg=W0212
1779 # yes, we really want os._exit
1785 if (pid == 0): # The first child.
1788 pid = os.fork() # Fork a second child.
1789 if (pid == 0): # The second child.
1793 # exit() or _exit()? See below.
1794 os._exit(0) # Exit parent (the first child) of the second child.
1796 os._exit(0) # Exit parent of the first child.
1800 i = os.open("/dev/null", os.O_RDONLY) # stdin
1801 assert i == 0, "Can't close/reopen stdin"
1802 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1803 assert i == 1, "Can't close/reopen stdout"
1804 # Duplicate standard output to standard error.
1809 def DaemonPidFileName(name):
1810 """Compute a ganeti pid file absolute path
1813 @param name: the daemon name
1815 @return: the full path to the pidfile corresponding to the given
1819 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1822 def WritePidFile(name):
1823 """Write the current process pidfile.
1825 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1828 @param name: the daemon name to use
1829 @raise errors.GenericError: if the pid file already exists and
1830 points to a live process
1834 pidfilename = DaemonPidFileName(name)
1835 if IsProcessAlive(ReadPidFile(pidfilename)):
1836 raise errors.GenericError("%s contains a live process" % pidfilename)
1838 WriteFile(pidfilename, data="%d\n" % pid)
1841 def RemovePidFile(name):
1842 """Remove the current process pidfile.
1844 Any errors are ignored.
1847 @param name: the daemon name used to derive the pidfile name
1850 pidfilename = DaemonPidFileName(name)
1851 # TODO: we could check here that the file contains our pid
1853 RemoveFile(pidfilename)
1854 except: # pylint: disable-msg=W0702
1858 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1860 """Kill a process given by its pid.
1863 @param pid: The PID to terminate.
1865 @param signal_: The signal to send, by default SIGTERM
1867 @param timeout: The timeout after which, if the process is still alive,
1868 a SIGKILL will be sent. If not positive, no such checking
1870 @type waitpid: boolean
1871 @param waitpid: If true, we should waitpid on this process after
1872 sending signals, since it's our own child and otherwise it
1873 would remain as zombie
1876 def _helper(pid, signal_, wait):
1877 """Simple helper to encapsulate the kill/waitpid sequence"""
1878 os.kill(pid, signal_)
1881 os.waitpid(pid, os.WNOHANG)
1886 # kill with pid=0 == suicide
1887 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1889 if not IsProcessAlive(pid):
1892 _helper(pid, signal_, waitpid)
1897 def _CheckProcess():
1898 if not IsProcessAlive(pid):
1902 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1912 # Wait up to $timeout seconds
1913 Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1914 except RetryTimeout:
1917 if IsProcessAlive(pid):
1918 # Kill process if it's still alive
1919 _helper(pid, signal.SIGKILL, waitpid)
1922 def FindFile(name, search_path, test=os.path.exists):
1923 """Look for a filesystem object in a given path.
1925 This is an abstract method to search for filesystem object (files,
1926 dirs) under a given search path.
1929 @param name: the name to look for
1930 @type search_path: str
1931 @param search_path: location to start at
1932 @type test: callable
1933 @param test: a function taking one argument that should return True
1934 if the a given object is valid; the default value is
1935 os.path.exists, causing only existing files to be returned
1937 @return: full path to the object if found, None otherwise
1940 # validate the filename mask
1941 if constants.EXT_PLUGIN_MASK.match(name) is None:
1942 logging.critical("Invalid value passed for external script name: '%s'",
1946 for dir_name in search_path:
1947 item_name = os.path.sep.join([dir_name, name])
1948 # check the user test and that we're indeed resolving to the given
1950 if test(item_name) and os.path.basename(item_name) == name:
1955 def CheckVolumeGroupSize(vglist, vgname, minsize):
1956 """Checks if the volume group list is valid.
1958 The function will check if a given volume group is in the list of
1959 volume groups and has a minimum size.
1962 @param vglist: dictionary of volume group names and their size
1964 @param vgname: the volume group we should check
1966 @param minsize: the minimum size we accept
1968 @return: None for success, otherwise the error message
1971 vgsize = vglist.get(vgname, None)
1973 return "volume group '%s' missing" % vgname
1974 elif vgsize < minsize:
1975 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1976 (vgname, minsize, vgsize))
1980 def SplitTime(value):
1981 """Splits time as floating point number into a tuple.
1983 @param value: Time in seconds
1984 @type value: int or float
1985 @return: Tuple containing (seconds, microseconds)
1988 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1990 assert 0 <= seconds, \
1991 "Seconds must be larger than or equal to 0, but are %s" % seconds
1992 assert 0 <= microseconds <= 999999, \
1993 "Microseconds must be 0-999999, but are %s" % microseconds
1995 return (int(seconds), int(microseconds))
1998 def MergeTime(timetuple):
1999 """Merges a tuple into time as a floating point number.
2001 @param timetuple: Time as tuple, (seconds, microseconds)
2002 @type timetuple: tuple
2003 @return: Time as a floating point number expressed in seconds
2006 (seconds, microseconds) = timetuple
2008 assert 0 <= seconds, \
2009 "Seconds must be larger than or equal to 0, but are %s" % seconds
2010 assert 0 <= microseconds <= 999999, \
2011 "Microseconds must be 0-999999, but are %s" % microseconds
2013 return float(seconds) + (float(microseconds) * 0.000001)
2016 def GetDaemonPort(daemon_name):
2017 """Get the daemon port for this cluster.
2019 Note that this routine does not read a ganeti-specific file, but
2020 instead uses C{socket.getservbyname} to allow pre-customization of
2021 this parameter outside of Ganeti.
2023 @type daemon_name: string
2024 @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
2028 if daemon_name not in constants.DAEMONS_PORTS:
2029 raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
2031 (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
2033 port = socket.getservbyname(daemon_name, proto)
2034 except socket.error:
2040 def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
2041 multithreaded=False, syslog=constants.SYSLOG_USAGE):
2042 """Configures the logging module.
2045 @param logfile: the filename to which we should log
2046 @type debug: integer
2047 @param debug: if greater than zero, enable debug messages, otherwise
2048 only those at C{INFO} and above level
2049 @type stderr_logging: boolean
2050 @param stderr_logging: whether we should also log to the standard error
2052 @param program: the name under which we should log messages
2053 @type multithreaded: boolean
2054 @param multithreaded: if True, will add the thread name to the log file
2055 @type syslog: string
2056 @param syslog: one of 'no', 'yes', 'only':
2057 - if no, syslog is not used
2058 - if yes, syslog is used (in addition to file-logging)
2059 - if only, only syslog is used
2060 @raise EnvironmentError: if we can't open the log file and
2061 syslog/stderr logging is disabled
2064 fmt = "%(asctime)s: " + program + " pid=%(process)d"
2065 sft = program + "[%(process)d]:"
2067 fmt += "/%(threadName)s"
2068 sft += " (%(threadName)s)"
2070 fmt += " %(module)s:%(lineno)s"
2071 # no debug info for syslog loggers
2072 fmt += " %(levelname)s %(message)s"
2073 # yes, we do want the textual level, as remote syslog will probably
2074 # lose the error level, and it's easier to grep for it
2075 sft += " %(levelname)s %(message)s"
2076 formatter = logging.Formatter(fmt)
2077 sys_fmt = logging.Formatter(sft)
2079 root_logger = logging.getLogger("")
2080 root_logger.setLevel(logging.NOTSET)
2082 # Remove all previously setup handlers
2083 for handler in root_logger.handlers:
2085 root_logger.removeHandler(handler)
2088 stderr_handler = logging.StreamHandler()
2089 stderr_handler.setFormatter(formatter)
2091 stderr_handler.setLevel(logging.NOTSET)
2093 stderr_handler.setLevel(logging.CRITICAL)
2094 root_logger.addHandler(stderr_handler)
2096 if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
2097 facility = logging.handlers.SysLogHandler.LOG_DAEMON
2098 syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
2100 syslog_handler.setFormatter(sys_fmt)
2101 # Never enable debug over syslog
2102 syslog_handler.setLevel(logging.INFO)
2103 root_logger.addHandler(syslog_handler)
2105 if syslog != constants.SYSLOG_ONLY:
2106 # this can fail, if the logging directories are not setup or we have
2107 # a permisssion problem; in this case, it's best to log but ignore
2108 # the error if stderr_logging is True, and if false we re-raise the
2109 # exception since otherwise we could run but without any logs at all
2111 logfile_handler = logging.FileHandler(logfile)
2112 logfile_handler.setFormatter(formatter)
2114 logfile_handler.setLevel(logging.DEBUG)
2116 logfile_handler.setLevel(logging.INFO)
2117 root_logger.addHandler(logfile_handler)
2118 except EnvironmentError:
2119 if stderr_logging or syslog == constants.SYSLOG_YES:
2120 logging.exception("Failed to enable logging to file '%s'", logfile)
2122 # we need to re-raise the exception
2126 def IsNormAbsPath(path):
2127 """Check whether a path is absolute and also normalized
2129 This avoids things like /dir/../../other/path to be valid.
2132 return os.path.normpath(path) == path and os.path.isabs(path)
2135 def TailFile(fname, lines=20):
2136 """Return the last lines from a file.
2138 @note: this function will only read and parse the last 4KB of
2139 the file; if the lines are very long, it could be that less
2140 than the requested number of lines are returned
2142 @param fname: the file name
2144 @param lines: the (maximum) number of lines to return
2147 fd = open(fname, "r")
2151 pos = max(0, pos-4096)
2153 raw_data = fd.read()
2157 rows = raw_data.splitlines()
2158 return rows[-lines:]
2161 def SafeEncode(text):
2162 """Return a 'safe' version of a source string.
2164 This function mangles the input string and returns a version that
2165 should be safe to display/encode as ASCII. To this end, we first
2166 convert it to ASCII using the 'backslashreplace' encoding which
2167 should get rid of any non-ASCII chars, and then we process it
2168 through a loop copied from the string repr sources in the python; we
2169 don't use string_escape anymore since that escape single quotes and
2170 backslashes too, and that is too much; and that escaping is not
2171 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2173 @type text: str or unicode
2174 @param text: input data
2176 @return: a safe version of text
2179 if isinstance(text, unicode):
2180 # only if unicode; if str already, we handle it below
2181 text = text.encode('ascii', 'backslashreplace')
2191 elif c < 32 or c >= 127: # non-printable
2192 resu += "\\x%02x" % (c & 0xff)
2198 def UnescapeAndSplit(text, sep=","):
2199 """Split and unescape a string based on a given separator.
2201 This function splits a string based on a separator where the
2202 separator itself can be escape in order to be an element of the
2203 elements. The escaping rules are (assuming coma being the
2205 - a plain , separates the elements
2206 - a sequence \\\\, (double backslash plus comma) is handled as a
2207 backslash plus a separator comma
2208 - a sequence \, (backslash plus comma) is handled as a
2212 @param text: the string to split
2214 @param text: the separator
2216 @return: a list of strings
2219 # we split the list by sep (with no escaping at this stage)
2220 slist = text.split(sep)
2221 # next, we revisit the elements and if any of them ended with an odd
2222 # number of backslashes, then we join it with the next
2226 if e1.endswith("\\"):
2227 num_b = len(e1) - len(e1.rstrip("\\"))
2230 # here the backslashes remain (all), and will be reduced in
2232 rlist.append(e1 + sep + e2)
2235 # finally, replace backslash-something with something
2236 rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2240 def CommaJoin(names):
2241 """Nicely join a set of identifiers.
2243 @param names: set, list or tuple
2244 @return: a string with the formatted results
2247 return ", ".join([str(val) for val in names])
2250 def BytesToMebibyte(value):
2251 """Converts bytes to mebibytes.
2254 @param value: Value in bytes
2256 @return: Value in mebibytes
2259 return int(round(value / (1024.0 * 1024.0), 0))
2262 def CalculateDirectorySize(path):
2263 """Calculates the size of a directory recursively.
2266 @param path: Path to directory
2268 @return: Size in mebibytes
2273 for (curpath, _, files) in os.walk(path):
2274 for filename in files:
2275 st = os.lstat(os.path.join(curpath, filename))
2278 return BytesToMebibyte(size)
2281 def GetFilesystemStats(path):
2282 """Returns the total and free space on a filesystem.
2285 @param path: Path on filesystem to be examined
2287 @return: tuple of (Total space, Free space) in mebibytes
2290 st = os.statvfs(path)
2292 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2293 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2294 return (tsize, fsize)
2297 def RunInSeparateProcess(fn):
2298 """Runs a function in a separate process.
2300 Note: Only boolean return values are supported.
2303 @param fn: Function to be called
2304 @rtype: tuple of (int/None, int/None)
2305 @return: Exit code and signal number
2312 # In case the function uses temporary files
2313 ResetTempfileModule()
2316 result = int(bool(fn()))
2317 assert result in (0, 1)
2318 except: # pylint: disable-msg=W0702
2319 logging.exception("Error while calling function in separate process")
2320 # 0 and 1 are reserved for the return value
2323 os._exit(result) # pylint: disable-msg=W0212
2327 # Avoid zombies and check exit code
2328 (_, status) = os.waitpid(pid, 0)
2330 if os.WIFSIGNALED(status):
2332 signum = os.WTERMSIG(status)
2334 exitcode = os.WEXITSTATUS(status)
2337 if not (exitcode in (0, 1) and signum is None):
2338 raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2341 return bool(exitcode)
2344 def LockedMethod(fn):
2345 """Synchronized object access decorator.
2347 This decorator is intended to protect access to an object using the
2348 object's own lock which is hardcoded to '_lock'.
2351 def _LockDebug(*args, **kwargs):
2353 logging.debug(*args, **kwargs)
2355 def wrapper(self, *args, **kwargs):
2356 # pylint: disable-msg=W0212
2357 assert hasattr(self, '_lock')
2359 _LockDebug("Waiting for %s", lock)
2362 _LockDebug("Acquired %s", lock)
2363 result = fn(self, *args, **kwargs)
2365 _LockDebug("Releasing %s", lock)
2367 _LockDebug("Released %s", lock)
2373 """Locks a file using POSIX locks.
2376 @param fd: the file descriptor we need to lock
2380 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2381 except IOError, err:
2382 if err.errno == errno.EAGAIN:
2383 raise errors.LockError("File already locked")
2387 def FormatTime(val):
2388 """Formats a time value.
2390 @type val: float or None
2391 @param val: the timestamp as returned by time.time()
2392 @return: a string value or N/A if we don't have a valid timestamp
2395 if val is None or not isinstance(val, (int, float)):
2397 # these two codes works on Linux, but they are not guaranteed on all
2399 return time.strftime("%F %T", time.localtime(val))
2402 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2403 """Reads the watcher pause file.
2405 @type filename: string
2406 @param filename: Path to watcher pause file
2407 @type now: None, float or int
2408 @param now: Current time as Unix timestamp
2409 @type remove_after: int
2410 @param remove_after: Remove watcher pause file after specified amount of
2411 seconds past the pause end time
2418 value = ReadFile(filename)
2419 except IOError, err:
2420 if err.errno != errno.ENOENT:
2424 if value is not None:
2428 logging.warning(("Watcher pause file (%s) contains invalid value,"
2429 " removing it"), filename)
2430 RemoveFile(filename)
2433 if value is not None:
2434 # Remove file if it's outdated
2435 if now > (value + remove_after):
2436 RemoveFile(filename)
2445 class RetryTimeout(Exception):
2446 """Retry loop timed out.
2451 class RetryAgain(Exception):
2457 class _RetryDelayCalculator(object):
2458 """Calculator for increasing delays.
2468 def __init__(self, start, factor, limit):
2469 """Initializes this class.
2472 @param start: Initial delay
2474 @param factor: Factor for delay increase
2475 @type limit: float or None
2476 @param limit: Upper limit for delay or None for no limit
2480 assert factor >= 1.0
2481 assert limit is None or limit >= 0.0
2484 self._factor = factor
2490 """Returns current delay and calculates the next one.
2493 current = self._next
2495 # Update for next run
2496 if self._limit is None or self._next < self._limit:
2497 self._next = min(self._limit, self._next * self._factor)
2502 #: Special delay to specify whole remaining timeout
2503 RETRY_REMAINING_TIME = object()
2506 def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2507 _time_fn=time.time):
2508 """Call a function repeatedly until it succeeds.
2510 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2511 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2512 total of C{timeout} seconds, this function throws L{RetryTimeout}.
2514 C{delay} can be one of the following:
2515 - callable returning the delay length as a float
2516 - Tuple of (start, factor, limit)
2517 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2518 useful when overriding L{wait_fn} to wait for an external event)
2519 - A static delay as a number (int or float)
2522 @param fn: Function to be called
2523 @param delay: Either a callable (returning the delay), a tuple of (start,
2524 factor, limit) (see L{_RetryDelayCalculator}),
2525 L{RETRY_REMAINING_TIME} or a number (int or float)
2526 @type timeout: float
2527 @param timeout: Total timeout
2528 @type wait_fn: callable
2529 @param wait_fn: Waiting function
2530 @return: Return value of function
2534 assert callable(wait_fn)
2535 assert callable(_time_fn)
2540 end_time = _time_fn() + timeout
2543 # External function to calculate delay
2546 elif isinstance(delay, (tuple, list)):
2547 # Increasing delay with optional upper boundary
2548 (start, factor, limit) = delay
2549 calc_delay = _RetryDelayCalculator(start, factor, limit)
2551 elif delay is RETRY_REMAINING_TIME:
2552 # Always use the remaining time
2557 calc_delay = lambda: delay
2559 assert calc_delay is None or callable(calc_delay)
2563 # pylint: disable-msg=W0142
2568 remaining_time = end_time - _time_fn()
2570 if remaining_time < 0.0:
2571 raise RetryTimeout()
2573 assert remaining_time >= 0.0
2575 if calc_delay is None:
2576 wait_fn(remaining_time)
2578 current_delay = calc_delay()
2579 if current_delay > 0.0:
2580 wait_fn(current_delay)
2583 def GetClosedTempfile(*args, **kwargs):
2584 """Creates a temporary file and returns its path.
2587 (fd, path) = tempfile.mkstemp(*args, **kwargs)
2592 def GenerateSelfSignedX509Cert(common_name, validity):
2593 """Generates a self-signed X509 certificate.
2595 @type common_name: string
2596 @param common_name: commonName value
2598 @param validity: Validity for certificate in seconds
2601 # Create private and public key
2602 key = OpenSSL.crypto.PKey()
2603 key.generate_key(OpenSSL.crypto.TYPE_RSA, constants.RSA_KEY_BITS)
2605 # Create self-signed certificate
2606 cert = OpenSSL.crypto.X509()
2608 cert.get_subject().CN = common_name
2609 cert.set_serial_number(1)
2610 cert.gmtime_adj_notBefore(0)
2611 cert.gmtime_adj_notAfter(validity)
2612 cert.set_issuer(cert.get_subject())
2613 cert.set_pubkey(key)
2614 cert.sign(key, constants.X509_CERT_SIGN_DIGEST)
2616 key_pem = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key)
2617 cert_pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)
2619 return (key_pem, cert_pem)
2622 def GenerateSelfSignedSslCert(filename, validity=(5 * 365)):
2623 """Legacy function to generate self-signed X509 certificate.
2626 (key_pem, cert_pem) = GenerateSelfSignedX509Cert(None,
2627 validity * 24 * 60 * 60)
2629 WriteFile(filename, mode=0400, data=key_pem + cert_pem)
2632 class FileLock(object):
2633 """Utility class for file locks.
2636 def __init__(self, filename):
2637 """Constructor for FileLock.
2639 This will open the file denoted by the I{filename} argument.
2642 @param filename: path to the file to be locked
2645 self.filename = filename
2646 self.fd = open(self.filename, "w")
2652 """Close the file and release the lock.
2655 if hasattr(self, "fd") and self.fd:
2659 def _flock(self, flag, blocking, timeout, errmsg):
2660 """Wrapper for fcntl.flock.
2663 @param flag: operation flag
2664 @type blocking: bool
2665 @param blocking: whether the operation should be done in blocking mode.
2666 @type timeout: None or float
2667 @param timeout: for how long the operation should be retried (implies
2669 @type errmsg: string
2670 @param errmsg: error message in case operation fails.
2673 assert self.fd, "Lock was closed"
2674 assert timeout is None or timeout >= 0, \
2675 "If specified, timeout must be positive"
2677 if timeout is not None:
2678 flag |= fcntl.LOCK_NB
2679 timeout_end = time.time() + timeout
2681 # Blocking doesn't have effect with timeout
2683 flag |= fcntl.LOCK_NB
2686 # TODO: Convert to utils.Retry
2691 fcntl.flock(self.fd, flag)
2693 except IOError, err:
2694 if err.errno in (errno.EAGAIN, ):
2695 if timeout_end is not None and time.time() < timeout_end:
2696 # Wait before trying again
2697 time.sleep(max(0.1, min(1.0, timeout)))
2699 raise errors.LockError(errmsg)
2701 logging.exception("fcntl.flock failed")
2704 def Exclusive(self, blocking=False, timeout=None):
2705 """Locks the file in exclusive mode.
2707 @type blocking: boolean
2708 @param blocking: whether to block and wait until we
2709 can lock the file or return immediately
2710 @type timeout: int or None
2711 @param timeout: if not None, the duration to wait for the lock
2715 self._flock(fcntl.LOCK_EX, blocking, timeout,
2716 "Failed to lock %s in exclusive mode" % self.filename)
2718 def Shared(self, blocking=False, timeout=None):
2719 """Locks the file in shared mode.
2721 @type blocking: boolean
2722 @param blocking: whether to block and wait until we
2723 can lock the file or return immediately
2724 @type timeout: int or None
2725 @param timeout: if not None, the duration to wait for the lock
2729 self._flock(fcntl.LOCK_SH, blocking, timeout,
2730 "Failed to lock %s in shared mode" % self.filename)
2732 def Unlock(self, blocking=True, timeout=None):
2733 """Unlocks the file.
2735 According to C{flock(2)}, unlocking can also be a nonblocking
2738 To make a non-blocking request, include LOCK_NB with any of the above
2741 @type blocking: boolean
2742 @param blocking: whether to block and wait until we
2743 can lock the file or return immediately
2744 @type timeout: int or None
2745 @param timeout: if not None, the duration to wait for the lock
2749 self._flock(fcntl.LOCK_UN, blocking, timeout,
2750 "Failed to unlock %s" % self.filename)
2753 def SignalHandled(signums):
2754 """Signal Handled decoration.
2756 This special decorator installs a signal handler and then calls the target
2757 function. The function must accept a 'signal_handlers' keyword argument,
2758 which will contain a dict indexed by signal number, with SignalHandler
2761 The decorator can be safely stacked with iself, to handle multiple signals
2762 with different handlers.
2765 @param signums: signals to intercept
2769 def sig_function(*args, **kwargs):
2770 assert 'signal_handlers' not in kwargs or \
2771 kwargs['signal_handlers'] is None or \
2772 isinstance(kwargs['signal_handlers'], dict), \
2773 "Wrong signal_handlers parameter in original function call"
2774 if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2775 signal_handlers = kwargs['signal_handlers']
2777 signal_handlers = {}
2778 kwargs['signal_handlers'] = signal_handlers
2779 sighandler = SignalHandler(signums)
2782 signal_handlers[sig] = sighandler
2783 return fn(*args, **kwargs)
2790 class SignalHandler(object):
2791 """Generic signal handler class.
2793 It automatically restores the original handler when deconstructed or
2794 when L{Reset} is called. You can either pass your own handler
2795 function in or query the L{called} attribute to detect whether the
2799 @ivar signum: the signals we handle
2800 @type called: boolean
2801 @ivar called: tracks whether any of the signals have been raised
2804 def __init__(self, signum, handler_fn=None):
2805 """Constructs a new SignalHandler instance.
2807 @type signum: int or list of ints
2808 @param signum: Single signal number or set of signal numbers
2809 @type handler_fn: callable
2810 @param handler_fn: Signal handling function
2813 assert handler_fn is None or callable(handler_fn)
2815 self.signum = set(signum)
2818 self._handler_fn = handler_fn
2822 for signum in self.signum:
2824 prev_handler = signal.signal(signum, self._HandleSignal)
2826 self._previous[signum] = prev_handler
2828 # Restore previous handler
2829 signal.signal(signum, prev_handler)
2832 # Reset all handlers
2834 # Here we have a race condition: a handler may have already been called,
2835 # but there's not much we can do about it at this point.
2842 """Restore previous handler.
2844 This will reset all the signals to their previous handlers.
2847 for signum, prev_handler in self._previous.items():
2848 signal.signal(signum, prev_handler)
2849 # If successful, remove from dict
2850 del self._previous[signum]
2853 """Unsets the L{called} flag.
2855 This function can be used in case a signal may arrive several times.
2860 def _HandleSignal(self, signum, frame):
2861 """Actual signal handling function.
2864 # This is not nice and not absolutely atomic, but it appears to be the only
2865 # solution in Python -- there are no atomic types.
2868 if self._handler_fn:
2869 self._handler_fn(signum, frame)
2872 class FieldSet(object):
2873 """A simple field set.
2875 Among the features are:
2876 - checking if a string is among a list of static string or regex objects
2877 - checking if a whole list of string matches
2878 - returning the matching groups from a regex match
2880 Internally, all fields are held as regular expression objects.
2883 def __init__(self, *items):
2884 self.items = [re.compile("^%s$" % value) for value in items]
2886 def Extend(self, other_set):
2887 """Extend the field set with the items from another one"""
2888 self.items.extend(other_set.items)
2890 def Matches(self, field):
2891 """Checks if a field matches the current set
2894 @param field: the string to match
2895 @return: either None or a regular expression match object
2898 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2902 def NonMatching(self, items):
2903 """Returns the list of fields not matching the current set
2906 @param items: the list of fields to check
2908 @return: list of non-matching fields
2911 return [val for val in items if not self.Matches(val)]