4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
44 import logging.handlers
49 from cStringIO import StringIO
52 from hashlib import sha1
57 from ganeti import errors
58 from ganeti import constants
62 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
66 #: when set to True, L{RunCmd} is disabled
69 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
72 class RunResult(object):
73 """Holds the result of running external programs.
76 @ivar exit_code: the exit code of the program, or None (if the program
78 @type signal: int or None
79 @ivar signal: the signal that caused the program to finish, or None
80 (if the program wasn't terminated by a signal)
82 @ivar stdout: the standard output of the program
84 @ivar stderr: the standard error of the program
86 @ivar failed: True in case the program was
87 terminated by a signal or exited with a non-zero exit code
88 @ivar fail_reason: a string detailing the termination reason
91 __slots__ = ["exit_code", "signal", "stdout", "stderr",
92 "failed", "fail_reason", "cmd"]
95 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
97 self.exit_code = exit_code
101 self.failed = (signal_ is not None or exit_code != 0)
103 if self.signal is not None:
104 self.fail_reason = "terminated by signal %s" % self.signal
105 elif self.exit_code is not None:
106 self.fail_reason = "exited with exit code %s" % self.exit_code
108 self.fail_reason = "unable to determine termination reason"
111 logging.debug("Command '%s' failed (%s); output: %s",
112 self.cmd, self.fail_reason, self.output)
114 def _GetOutput(self):
115 """Returns the combined stdout and stderr for easier usage.
118 return self.stdout + self.stderr
120 output = property(_GetOutput, None, None, "Return full output")
123 def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
124 """Execute a (shell) command.
126 The command should not read from its standard input, as it will be
129 @type cmd: string or list
130 @param cmd: Command to run
132 @param env: Additional environment
134 @param output: if desired, the output of the command can be
135 saved in a file instead of the RunResult instance; this
136 parameter denotes the file name (if not None)
138 @param cwd: if specified, will be used as the working
139 directory for the command; the default will be /
140 @type reset_env: boolean
141 @param reset_env: whether to reset or keep the default os environment
143 @return: RunResult instance
144 @raise errors.ProgrammerError: if we call this when forks are disabled
148 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
150 if isinstance(cmd, list):
151 cmd = [str(val) for val in cmd]
152 strcmd = " ".join(cmd)
157 logging.debug("RunCmd '%s'", strcmd)
160 cmd_env = os.environ.copy()
161 cmd_env["LC_ALL"] = "C"
170 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
172 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
175 if err.errno == errno.ENOENT:
176 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
188 return RunResult(exitcode, signal_, out, err, strcmd)
191 def _RunCmdPipe(cmd, env, via_shell, cwd):
192 """Run a command and return its output.
194 @type cmd: string or list
195 @param cmd: Command to run
197 @param env: The environment to use
198 @type via_shell: bool
199 @param via_shell: if we should run via the shell
201 @param cwd: the working directory for the program
203 @return: (out, err, status)
206 poller = select.poll()
207 child = subprocess.Popen(cmd, shell=via_shell,
208 stderr=subprocess.PIPE,
209 stdout=subprocess.PIPE,
210 stdin=subprocess.PIPE,
211 close_fds=True, env=env,
215 poller.register(child.stdout, select.POLLIN)
216 poller.register(child.stderr, select.POLLIN)
220 child.stdout.fileno(): (out, child.stdout),
221 child.stderr.fileno(): (err, child.stderr),
224 status = fcntl.fcntl(fd, fcntl.F_GETFL)
225 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
229 pollresult = poller.poll()
230 except EnvironmentError, eerr:
231 if eerr.errno == errno.EINTR:
234 except select.error, serr:
235 if serr[0] == errno.EINTR:
239 for fd, event in pollresult:
240 if event & select.POLLIN or event & select.POLLPRI:
241 data = fdmap[fd][1].read()
242 # no data from read signifies EOF (the same as POLLHUP)
244 poller.unregister(fd)
247 fdmap[fd][0].write(data)
248 if (event & select.POLLNVAL or event & select.POLLHUP or
249 event & select.POLLERR):
250 poller.unregister(fd)
256 status = child.wait()
257 return out, err, status
260 def _RunCmdFile(cmd, env, via_shell, output, cwd):
261 """Run a command and save its output to a file.
263 @type cmd: string or list
264 @param cmd: Command to run
266 @param env: The environment to use
267 @type via_shell: bool
268 @param via_shell: if we should run via the shell
270 @param output: the filename in which to save the output
272 @param cwd: the working directory for the program
274 @return: the exit status
277 fh = open(output, "a")
279 child = subprocess.Popen(cmd, shell=via_shell,
280 stderr=subprocess.STDOUT,
282 stdin=subprocess.PIPE,
283 close_fds=True, env=env,
287 status = child.wait()
293 def RunParts(dir_name, env=None, reset_env=False):
294 """Run Scripts or programs in a directory
296 @type dir_name: string
297 @param dir_name: absolute path to a directory
299 @param env: The environment to use
300 @type reset_env: boolean
301 @param reset_env: whether to reset or keep the default os environment
302 @rtype: list of tuples
303 @return: list of (name, (one of RUNDIR_STATUS), RunResult)
309 dir_contents = ListVisibleFiles(dir_name)
311 logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
314 for relname in sorted(dir_contents):
315 fname = PathJoin(dir_name, relname)
316 if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
317 constants.EXT_PLUGIN_MASK.match(relname) is not None):
318 rr.append((relname, constants.RUNPARTS_SKIP, None))
321 result = RunCmd([fname], env=env, reset_env=reset_env)
322 except Exception, err: # pylint: disable-msg=W0703
323 rr.append((relname, constants.RUNPARTS_ERR, str(err)))
325 rr.append((relname, constants.RUNPARTS_RUN, result))
330 def RemoveFile(filename):
331 """Remove a file ignoring some errors.
333 Remove a file, ignoring non-existing ones or directories. Other
337 @param filename: the file to be removed
343 if err.errno not in (errno.ENOENT, errno.EISDIR):
347 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
351 @param old: Original path
355 @param mkdir: Whether to create target directory if it doesn't exist
356 @type mkdir_mode: int
357 @param mkdir_mode: Mode for newly created directories
361 return os.rename(old, new)
363 # In at least one use case of this function, the job queue, directory
364 # creation is very rare. Checking for the directory before renaming is not
366 if mkdir and err.errno == errno.ENOENT:
367 # Create directory and try again
368 Makedirs(os.path.dirname(new))
370 return os.rename(old, new)
375 def Makedirs(path, mode=0750):
376 """Super-mkdir; create a leaf directory and all intermediate ones.
378 This is a wrapper around C{os.makedirs} adding error handling not implemented
383 os.makedirs(path, mode)
385 # Ignore EEXIST. This is only handled in os.makedirs as included in
386 # Python 2.5 and above.
387 if err.errno != errno.EEXIST or not os.path.exists(path):
391 def ResetTempfileModule():
392 """Resets the random name generator of the tempfile module.
394 This function should be called after C{os.fork} in the child process to
395 ensure it creates a newly seeded random generator. Otherwise it would
396 generate the same random parts as the parent process. If several processes
397 race for the creation of a temporary file, this could lead to one not getting
401 # pylint: disable-msg=W0212
402 if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
403 tempfile._once_lock.acquire()
405 # Reset random name generator
406 tempfile._name_sequence = None
408 tempfile._once_lock.release()
410 logging.critical("The tempfile module misses at least one of the"
411 " '_once_lock' and '_name_sequence' attributes")
414 def _FingerprintFile(filename):
415 """Compute the fingerprint of a file.
417 If the file does not exist, a None will be returned
421 @param filename: the filename to checksum
423 @return: the hex digest of the sha checksum of the contents
427 if not (os.path.exists(filename) and os.path.isfile(filename)):
440 return fp.hexdigest()
443 def FingerprintFiles(files):
444 """Compute fingerprints for a list of files.
447 @param files: the list of filename to fingerprint
449 @return: a dictionary filename: fingerprint, holding only
455 for filename in files:
456 cksum = _FingerprintFile(filename)
458 ret[filename] = cksum
463 def ForceDictType(target, key_types, allowed_values=None):
464 """Force the values of a dict to have certain types.
467 @param target: the dict to update
468 @type key_types: dict
469 @param key_types: dict mapping target dict keys to types
470 in constants.ENFORCEABLE_TYPES
471 @type allowed_values: list
472 @keyword allowed_values: list of specially allowed values
475 if allowed_values is None:
478 if not isinstance(target, dict):
479 msg = "Expected dictionary, got '%s'" % target
480 raise errors.TypeEnforcementError(msg)
483 if key not in key_types:
484 msg = "Unknown key '%s'" % key
485 raise errors.TypeEnforcementError(msg)
487 if target[key] in allowed_values:
490 ktype = key_types[key]
491 if ktype not in constants.ENFORCEABLE_TYPES:
492 msg = "'%s' has non-enforceable type %s" % (key, ktype)
493 raise errors.ProgrammerError(msg)
495 if ktype == constants.VTYPE_STRING:
496 if not isinstance(target[key], basestring):
497 if isinstance(target[key], bool) and not target[key]:
500 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
501 raise errors.TypeEnforcementError(msg)
502 elif ktype == constants.VTYPE_BOOL:
503 if isinstance(target[key], basestring) and target[key]:
504 if target[key].lower() == constants.VALUE_FALSE:
506 elif target[key].lower() == constants.VALUE_TRUE:
509 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
510 raise errors.TypeEnforcementError(msg)
515 elif ktype == constants.VTYPE_SIZE:
517 target[key] = ParseUnit(target[key])
518 except errors.UnitParseError, err:
519 msg = "'%s' (value %s) is not a valid size. error: %s" % \
520 (key, target[key], err)
521 raise errors.TypeEnforcementError(msg)
522 elif ktype == constants.VTYPE_INT:
524 target[key] = int(target[key])
525 except (ValueError, TypeError):
526 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
527 raise errors.TypeEnforcementError(msg)
530 def IsProcessAlive(pid):
531 """Check if a given pid exists on the system.
533 @note: zombie status is not handled, so zombie processes
534 will be returned as alive
536 @param pid: the process ID to check
538 @return: True if the process exists
545 os.stat("/proc/%d/status" % pid)
547 except EnvironmentError, err:
548 if err.errno in (errno.ENOENT, errno.ENOTDIR):
553 def ReadPidFile(pidfile):
554 """Read a pid from a file.
556 @type pidfile: string
557 @param pidfile: path to the file containing the pid
559 @return: The process id, if the file exists and contains a valid PID,
564 raw_data = ReadFile(pidfile)
565 except EnvironmentError, err:
566 if err.errno != errno.ENOENT:
567 logging.exception("Can't read pid file")
572 except (TypeError, ValueError), err:
573 logging.info("Can't parse pid file contents", exc_info=True)
579 def MatchNameComponent(key, name_list, case_sensitive=True):
580 """Try to match a name against a list.
582 This function will try to match a name like test1 against a list
583 like C{['test1.example.com', 'test2.example.com', ...]}. Against
584 this list, I{'test1'} as well as I{'test1.example'} will match, but
585 not I{'test1.ex'}. A multiple match will be considered as no match
586 at all (e.g. I{'test1'} against C{['test1.example.com',
587 'test1.example.org']}), except when the key fully matches an entry
588 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
591 @param key: the name to be searched
592 @type name_list: list
593 @param name_list: the list of strings against which to search the key
594 @type case_sensitive: boolean
595 @param case_sensitive: whether to provide a case-sensitive match
598 @return: None if there is no match I{or} if there are multiple matches,
599 otherwise the element from the list which matches
606 if not case_sensitive:
607 re_flags |= re.IGNORECASE
609 mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
612 for name in name_list:
613 if mo.match(name) is not None:
614 names_filtered.append(name)
615 if not case_sensitive and key == name.upper():
616 string_matches.append(name)
618 if len(string_matches) == 1:
619 return string_matches[0]
620 if len(names_filtered) == 1:
621 return names_filtered[0]
626 """Class implementing resolver and hostname functionality
629 _VALID_NAME_RE = re.compile("^[a-z0-9._-]{1,255}$")
631 def __init__(self, name=None):
632 """Initialize the host name object.
634 If the name argument is not passed, it will use this system's
639 name = self.SysName()
642 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
643 self.ip = self.ipaddrs[0]
646 """Returns the hostname without domain.
649 return self.name.split('.')[0]
653 """Return the current system's name.
655 This is simply a wrapper over C{socket.gethostname()}.
658 return socket.gethostname()
661 def LookupHostname(hostname):
665 @param hostname: hostname to look up
668 @return: a tuple (name, aliases, ipaddrs) as returned by
669 C{socket.gethostbyname_ex}
670 @raise errors.ResolverError: in case of errors in resolving
674 result = socket.gethostbyname_ex(hostname)
675 except socket.gaierror, err:
676 # hostname not found in DNS
677 raise errors.ResolverError(hostname, err.args[0], err.args[1])
682 def NormalizeName(cls, hostname):
683 """Validate and normalize the given hostname.
685 @attention: the validation is a bit more relaxed than the standards
686 require; most importantly, we allow underscores in names
687 @raise errors.OpPrereqError: when the name is not valid
690 hostname = hostname.lower()
691 if (not cls._VALID_NAME_RE.match(hostname) or
692 # double-dots, meaning empty label
694 # empty initial label
695 hostname.startswith(".")):
696 raise errors.OpPrereqError("Invalid hostname '%s'" % hostname,
698 if hostname.endswith("."):
699 hostname = hostname.rstrip(".")
703 def GetHostInfo(name=None):
704 """Lookup host name and raise an OpPrereqError for failures"""
707 return HostInfo(name)
708 except errors.ResolverError, err:
709 raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
710 (err[0], err[2]), errors.ECODE_RESOLVER)
713 def ListVolumeGroups():
714 """List volume groups and their size
718 Dictionary with keys volume name and values
719 the size of the volume
722 command = "vgs --noheadings --units m --nosuffix -o name,size"
723 result = RunCmd(command)
728 for line in result.stdout.splitlines():
730 name, size = line.split()
731 size = int(float(size))
732 except (IndexError, ValueError), err:
733 logging.error("Invalid output from vgs (%s): %s", err, line)
741 def BridgeExists(bridge):
742 """Check whether the given bridge exists in the system
745 @param bridge: the bridge name to check
747 @return: True if it does
750 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
753 def NiceSort(name_list):
754 """Sort a list of strings based on digit and non-digit groupings.
756 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
757 will sort the list in the logical order C{['a1', 'a2', 'a10',
760 The sort algorithm breaks each name in groups of either only-digits
761 or no-digits. Only the first eight such groups are considered, and
762 after that we just use what's left of the string.
764 @type name_list: list
765 @param name_list: the names to be sorted
767 @return: a copy of the name list sorted with our algorithm
770 _SORTER_BASE = "(\D+|\d+)"
771 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
772 _SORTER_BASE, _SORTER_BASE,
773 _SORTER_BASE, _SORTER_BASE,
774 _SORTER_BASE, _SORTER_BASE)
775 _SORTER_RE = re.compile(_SORTER_FULL)
776 _SORTER_NODIGIT = re.compile("^\D*$")
778 """Attempts to convert a variable to integer."""
779 if val is None or _SORTER_NODIGIT.match(val):
784 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
785 for name in name_list]
787 return [tup[1] for tup in to_sort]
790 def TryConvert(fn, val):
791 """Try to convert a value ignoring errors.
793 This function tries to apply function I{fn} to I{val}. If no
794 C{ValueError} or C{TypeError} exceptions are raised, it will return
795 the result, else it will return the original value. Any other
796 exceptions are propagated to the caller.
799 @param fn: function to apply to the value
800 @param val: the value to be converted
801 @return: The converted value if the conversion was successful,
802 otherwise the original value.
807 except (ValueError, TypeError):
813 """Verifies the syntax of an IPv4 address.
815 This function checks if the IPv4 address passes is valid or not based
816 on syntax (not IP range, class calculations, etc.).
819 @param ip: the address to be checked
820 @rtype: a regular expression match object
821 @return: a regular expression match object, or None if the
825 unit = "(0|[1-9]\d{0,2})"
826 #TODO: convert and return only boolean
827 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
830 def IsValidShellParam(word):
831 """Verifies is the given word is safe from the shell's p.o.v.
833 This means that we can pass this to a command via the shell and be
834 sure that it doesn't alter the command line and is passed as such to
837 Note that we are overly restrictive here, in order to be on the safe
841 @param word: the word to check
843 @return: True if the word is 'safe'
846 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
849 def BuildShellCmd(template, *args):
850 """Build a safe shell command line from the given arguments.
852 This function will check all arguments in the args list so that they
853 are valid shell parameters (i.e. they don't contain shell
854 metacharacters). If everything is ok, it will return the result of
858 @param template: the string holding the template for the
861 @return: the expanded command line
865 if not IsValidShellParam(word):
866 raise errors.ProgrammerError("Shell argument '%s' contains"
867 " invalid characters" % word)
868 return template % args
871 def FormatUnit(value, units):
872 """Formats an incoming number of MiB with the appropriate unit.
875 @param value: integer representing the value in MiB (1048576)
877 @param units: the type of formatting we should do:
878 - 'h' for automatic scaling
883 @return: the formatted value (with suffix)
886 if units not in ('m', 'g', 't', 'h'):
887 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
891 if units == 'm' or (units == 'h' and value < 1024):
894 return "%d%s" % (round(value, 0), suffix)
896 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
899 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
904 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
907 def ParseUnit(input_string):
908 """Tries to extract number and scale from the given string.
910 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
911 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
912 is always an int in MiB.
915 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
917 raise errors.UnitParseError("Invalid format")
919 value = float(m.groups()[0])
923 lcunit = unit.lower()
927 if lcunit in ('m', 'mb', 'mib'):
928 # Value already in MiB
931 elif lcunit in ('g', 'gb', 'gib'):
934 elif lcunit in ('t', 'tb', 'tib'):
938 raise errors.UnitParseError("Unknown unit: %s" % unit)
940 # Make sure we round up
941 if int(value) < value:
944 # Round up to the next multiple of 4
947 value += 4 - value % 4
952 def AddAuthorizedKey(file_name, key):
953 """Adds an SSH public key to an authorized_keys file.
956 @param file_name: path to authorized_keys file
958 @param key: string containing key
961 key_fields = key.split()
963 f = open(file_name, 'a+')
967 # Ignore whitespace changes
968 if line.split() == key_fields:
970 nl = line.endswith('\n')
974 f.write(key.rstrip('\r\n'))
981 def RemoveAuthorizedKey(file_name, key):
982 """Removes an SSH public key from an authorized_keys file.
985 @param file_name: path to authorized_keys file
987 @param key: string containing key
990 key_fields = key.split()
992 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
994 out = os.fdopen(fd, 'w')
996 f = open(file_name, 'r')
999 # Ignore whitespace changes while comparing lines
1000 if line.split() != key_fields:
1004 os.rename(tmpname, file_name)
1014 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1015 """Sets the name of an IP address and hostname in /etc/hosts.
1017 @type file_name: str
1018 @param file_name: path to the file to modify (usually C{/etc/hosts})
1020 @param ip: the IP address
1022 @param hostname: the hostname to be added
1024 @param aliases: the list of aliases to add for the hostname
1027 # FIXME: use WriteFile + fn rather than duplicating its efforts
1028 # Ensure aliases are unique
1029 aliases = UniqueSequence([hostname] + aliases)[1:]
1031 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1033 out = os.fdopen(fd, 'w')
1035 f = open(file_name, 'r')
1038 fields = line.split()
1039 if fields and not fields[0].startswith('#') and ip == fields[0]:
1043 out.write("%s\t%s" % (ip, hostname))
1045 out.write(" %s" % ' '.join(aliases))
1050 os.chmod(tmpname, 0644)
1051 os.rename(tmpname, file_name)
1061 def AddHostToEtcHosts(hostname):
1062 """Wrapper around SetEtcHostsEntry.
1065 @param hostname: a hostname that will be resolved and added to
1066 L{constants.ETC_HOSTS}
1069 hi = HostInfo(name=hostname)
1070 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1073 def RemoveEtcHostsEntry(file_name, hostname):
1074 """Removes a hostname from /etc/hosts.
1076 IP addresses without names are removed from the file.
1078 @type file_name: str
1079 @param file_name: path to the file to modify (usually C{/etc/hosts})
1081 @param hostname: the hostname to be removed
1084 # FIXME: use WriteFile + fn rather than duplicating its efforts
1085 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1087 out = os.fdopen(fd, 'w')
1089 f = open(file_name, 'r')
1092 fields = line.split()
1093 if len(fields) > 1 and not fields[0].startswith('#'):
1095 if hostname in names:
1096 while hostname in names:
1097 names.remove(hostname)
1099 out.write("%s %s\n" % (fields[0], ' '.join(names)))
1106 os.chmod(tmpname, 0644)
1107 os.rename(tmpname, file_name)
1117 def RemoveHostFromEtcHosts(hostname):
1118 """Wrapper around RemoveEtcHostsEntry.
1121 @param hostname: hostname that will be resolved and its
1122 full and shot name will be removed from
1123 L{constants.ETC_HOSTS}
1126 hi = HostInfo(name=hostname)
1127 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1128 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1131 def TimestampForFilename():
1132 """Returns the current time formatted for filenames.
1134 The format doesn't contain colons as some shells and applications them as
1138 return time.strftime("%Y-%m-%d_%H_%M_%S")
1141 def CreateBackup(file_name):
1142 """Creates a backup of a file.
1144 @type file_name: str
1145 @param file_name: file to be backed up
1147 @return: the path to the newly created backup
1148 @raise errors.ProgrammerError: for invalid file names
1151 if not os.path.isfile(file_name):
1152 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1155 prefix = ("%s.backup-%s." %
1156 (os.path.basename(file_name), TimestampForFilename()))
1157 dir_name = os.path.dirname(file_name)
1159 fsrc = open(file_name, 'rb')
1161 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1162 fdst = os.fdopen(fd, 'wb')
1164 logging.debug("Backing up %s at %s", file_name, backup_name)
1165 shutil.copyfileobj(fsrc, fdst)
1174 def ShellQuote(value):
1175 """Quotes shell argument according to POSIX.
1178 @param value: the argument to be quoted
1180 @return: the quoted value
1183 if _re_shell_unquoted.match(value):
1186 return "'%s'" % value.replace("'", "'\\''")
1189 def ShellQuoteArgs(args):
1190 """Quotes a list of shell arguments.
1193 @param args: list of arguments to be quoted
1195 @return: the quoted arguments concatenated with spaces
1198 return ' '.join([ShellQuote(i) for i in args])
1201 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1202 """Simple ping implementation using TCP connect(2).
1204 Check if the given IP is reachable by doing attempting a TCP connect
1208 @param target: the IP or hostname to ping
1210 @param port: the port to connect to
1212 @param timeout: the timeout on the connection attempt
1213 @type live_port_needed: boolean
1214 @param live_port_needed: whether a closed port will cause the
1215 function to return failure, as if there was a timeout
1216 @type source: str or None
1217 @param source: if specified, will cause the connect to be made
1218 from this specific source address; failures to bind other
1219 than C{EADDRNOTAVAIL} will be ignored
1222 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1226 if source is not None:
1228 sock.bind((source, 0))
1229 except socket.error, (errcode, _):
1230 if errcode == errno.EADDRNOTAVAIL:
1233 sock.settimeout(timeout)
1236 sock.connect((target, port))
1239 except socket.timeout:
1241 except socket.error, (errcode, _):
1242 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1247 def OwnIpAddress(address):
1248 """Check if the current host has the the given IP address.
1250 Currently this is done by TCP-pinging the address from the loopback
1253 @type address: string
1254 @param address: the address to check
1256 @return: True if we own the address
1259 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1260 source=constants.LOCALHOST_IP_ADDRESS)
1263 def ListVisibleFiles(path):
1264 """Returns a list of visible files in a directory.
1267 @param path: the directory to enumerate
1269 @return: the list of all files not starting with a dot
1270 @raise ProgrammerError: if L{path} is not an absolue and normalized path
1273 if not IsNormAbsPath(path):
1274 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
1275 " absolute/normalized: '%s'" % path)
1276 files = [i for i in os.listdir(path) if not i.startswith(".")]
1281 def GetHomeDir(user, default=None):
1282 """Try to get the homedir of the given user.
1284 The user can be passed either as a string (denoting the name) or as
1285 an integer (denoting the user id). If the user is not found, the
1286 'default' argument is returned, which defaults to None.
1290 if isinstance(user, basestring):
1291 result = pwd.getpwnam(user)
1292 elif isinstance(user, (int, long)):
1293 result = pwd.getpwuid(user)
1295 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1299 return result.pw_dir
1303 """Returns a random UUID.
1305 @note: This is a Linux-specific method as it uses the /proc
1310 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1313 def GenerateSecret(numbytes=20):
1314 """Generates a random secret.
1316 This will generate a pseudo-random secret returning an hex string
1317 (so that it can be used where an ASCII string is needed).
1319 @param numbytes: the number of bytes which will be represented by the returned
1320 string (defaulting to 20, the length of a SHA1 hash)
1322 @return: an hex representation of the pseudo-random sequence
1325 return os.urandom(numbytes).encode('hex')
1328 def EnsureDirs(dirs):
1329 """Make required directories, if they don't exist.
1331 @param dirs: list of tuples (dir_name, dir_mode)
1332 @type dirs: list of (string, integer)
1335 for dir_name, dir_mode in dirs:
1337 os.mkdir(dir_name, dir_mode)
1338 except EnvironmentError, err:
1339 if err.errno != errno.EEXIST:
1340 raise errors.GenericError("Cannot create needed directory"
1341 " '%s': %s" % (dir_name, err))
1342 if not os.path.isdir(dir_name):
1343 raise errors.GenericError("%s is not a directory" % dir_name)
1346 def ReadFile(file_name, size=-1):
1350 @param size: Read at most size bytes (if negative, entire file)
1352 @return: the (possibly partial) content of the file
1355 f = open(file_name, "r")
1362 def WriteFile(file_name, fn=None, data=None,
1363 mode=None, uid=-1, gid=-1,
1364 atime=None, mtime=None, close=True,
1365 dry_run=False, backup=False,
1366 prewrite=None, postwrite=None):
1367 """(Over)write a file atomically.
1369 The file_name and either fn (a function taking one argument, the
1370 file descriptor, and which should write the data to it) or data (the
1371 contents of the file) must be passed. The other arguments are
1372 optional and allow setting the file mode, owner and group, and the
1373 mtime/atime of the file.
1375 If the function doesn't raise an exception, it has succeeded and the
1376 target file has the new contents. If the function has raised an
1377 exception, an existing target file should be unmodified and the
1378 temporary file should be removed.
1380 @type file_name: str
1381 @param file_name: the target filename
1383 @param fn: content writing function, called with
1384 file descriptor as parameter
1386 @param data: contents of the file
1388 @param mode: file mode
1390 @param uid: the owner of the file
1392 @param gid: the group of the file
1394 @param atime: a custom access time to be set on the file
1396 @param mtime: a custom modification time to be set on the file
1397 @type close: boolean
1398 @param close: whether to close file after writing it
1399 @type prewrite: callable
1400 @param prewrite: function to be called before writing content
1401 @type postwrite: callable
1402 @param postwrite: function to be called after writing content
1405 @return: None if the 'close' parameter evaluates to True,
1406 otherwise the file descriptor
1408 @raise errors.ProgrammerError: if any of the arguments are not valid
1411 if not os.path.isabs(file_name):
1412 raise errors.ProgrammerError("Path passed to WriteFile is not"
1413 " absolute: '%s'" % file_name)
1415 if [fn, data].count(None) != 1:
1416 raise errors.ProgrammerError("fn or data required")
1418 if [atime, mtime].count(None) == 1:
1419 raise errors.ProgrammerError("Both atime and mtime must be either"
1422 if backup and not dry_run and os.path.isfile(file_name):
1423 CreateBackup(file_name)
1425 dir_name, base_name = os.path.split(file_name)
1426 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1428 # here we need to make sure we remove the temp file, if any error
1429 # leaves it in place
1431 if uid != -1 or gid != -1:
1432 os.chown(new_name, uid, gid)
1434 os.chmod(new_name, mode)
1435 if callable(prewrite):
1437 if data is not None:
1441 if callable(postwrite):
1444 if atime is not None and mtime is not None:
1445 os.utime(new_name, (atime, mtime))
1447 os.rename(new_name, file_name)
1456 RemoveFile(new_name)
1461 def FirstFree(seq, base=0):
1462 """Returns the first non-existing integer from seq.
1464 The seq argument should be a sorted list of positive integers. The
1465 first time the index of an element is smaller than the element
1466 value, the index will be returned.
1468 The base argument is used to start at a different offset,
1469 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1471 Example: C{[0, 1, 3]} will return I{2}.
1474 @param seq: the sequence to be analyzed.
1476 @param base: use this value as the base index of the sequence
1478 @return: the first non-used index in the sequence
1481 for idx, elem in enumerate(seq):
1482 assert elem >= base, "Passed element is higher than base offset"
1483 if elem > idx + base:
1489 def all(seq, pred=bool): # pylint: disable-msg=W0622
1490 "Returns True if pred(x) is True for every element in the iterable"
1491 for _ in itertools.ifilterfalse(pred, seq):
1496 def any(seq, pred=bool): # pylint: disable-msg=W0622
1497 "Returns True if pred(x) is True for at least one element in the iterable"
1498 for _ in itertools.ifilter(pred, seq):
1503 def SingleWaitForFdCondition(fdobj, event, timeout):
1504 """Waits for a condition to occur on the socket.
1506 Immediately returns at the first interruption.
1508 @type fdobj: integer or object supporting a fileno() method
1509 @param fdobj: entity to wait for events on
1510 @type event: integer
1511 @param event: ORed condition (see select module)
1512 @type timeout: float or None
1513 @param timeout: Timeout in seconds
1515 @return: None for timeout, otherwise occured conditions
1518 check = (event | select.POLLPRI |
1519 select.POLLNVAL | select.POLLHUP | select.POLLERR)
1521 if timeout is not None:
1522 # Poller object expects milliseconds
1525 poller = select.poll()
1526 poller.register(fdobj, event)
1528 # TODO: If the main thread receives a signal and we have no timeout, we
1529 # could wait forever. This should check a global "quit" flag or something
1531 io_events = poller.poll(timeout)
1532 except select.error, err:
1533 if err[0] != errno.EINTR:
1536 if io_events and io_events[0][1] & check:
1537 return io_events[0][1]
1542 class FdConditionWaiterHelper(object):
1543 """Retry helper for WaitForFdCondition.
1545 This class contains the retried and wait functions that make sure
1546 WaitForFdCondition can continue waiting until the timeout is actually
1551 def __init__(self, timeout):
1552 self.timeout = timeout
1554 def Poll(self, fdobj, event):
1555 result = SingleWaitForFdCondition(fdobj, event, self.timeout)
1561 def UpdateTimeout(self, timeout):
1562 self.timeout = timeout
1565 def WaitForFdCondition(fdobj, event, timeout):
1566 """Waits for a condition to occur on the socket.
1568 Retries until the timeout is expired, even if interrupted.
1570 @type fdobj: integer or object supporting a fileno() method
1571 @param fdobj: entity to wait for events on
1572 @type event: integer
1573 @param event: ORed condition (see select module)
1574 @type timeout: float or None
1575 @param timeout: Timeout in seconds
1577 @return: None for timeout, otherwise occured conditions
1580 if timeout is not None:
1581 retrywaiter = FdConditionWaiterHelper(timeout)
1582 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
1583 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
1586 while result is None:
1587 result = SingleWaitForFdCondition(fdobj, event, timeout)
1591 def partition(seq, pred=bool): # # pylint: disable-msg=W0622
1592 "Partition a list in two, based on the given predicate"
1593 return (list(itertools.ifilter(pred, seq)),
1594 list(itertools.ifilterfalse(pred, seq)))
1597 def UniqueSequence(seq):
1598 """Returns a list with unique elements.
1600 Element order is preserved.
1603 @param seq: the sequence with the source elements
1605 @return: list of unique elements from seq
1609 return [i for i in seq if i not in seen and not seen.add(i)]
1612 def NormalizeAndValidateMac(mac):
1613 """Normalizes and check if a MAC address is valid.
1615 Checks whether the supplied MAC address is formally correct, only
1616 accepts colon separated format. Normalize it to all lower.
1619 @param mac: the MAC to be validated
1621 @return: returns the normalized and validated MAC.
1623 @raise errors.OpPrereqError: If the MAC isn't valid
1626 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1627 if not mac_check.match(mac):
1628 raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1629 mac, errors.ECODE_INVAL)
1634 def TestDelay(duration):
1635 """Sleep for a fixed amount of time.
1637 @type duration: float
1638 @param duration: the sleep duration
1640 @return: False for negative value, True otherwise
1644 return False, "Invalid sleep duration"
1645 time.sleep(duration)
1649 def _CloseFDNoErr(fd, retries=5):
1650 """Close a file descriptor ignoring errors.
1653 @param fd: the file descriptor
1655 @param retries: how many retries to make, in case we get any
1656 other error than EBADF
1661 except OSError, err:
1662 if err.errno != errno.EBADF:
1664 _CloseFDNoErr(fd, retries - 1)
1665 # else either it's closed already or we're out of retries, so we
1666 # ignore this and go on
1669 def CloseFDs(noclose_fds=None):
1670 """Close file descriptors.
1672 This closes all file descriptors above 2 (i.e. except
1675 @type noclose_fds: list or None
1676 @param noclose_fds: if given, it denotes a list of file descriptor
1677 that should not be closed
1680 # Default maximum for the number of available file descriptors.
1681 if 'SC_OPEN_MAX' in os.sysconf_names:
1683 MAXFD = os.sysconf('SC_OPEN_MAX')
1690 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1691 if (maxfd == resource.RLIM_INFINITY):
1694 # Iterate through and close all file descriptors (except the standard ones)
1695 for fd in range(3, maxfd):
1696 if noclose_fds and fd in noclose_fds:
1701 def Daemonize(logfile):
1702 """Daemonize the current process.
1704 This detaches the current process from the controlling terminal and
1705 runs it in the background as a daemon.
1708 @param logfile: the logfile to which we should redirect stdout/stderr
1710 @return: the value zero
1713 # pylint: disable-msg=W0212
1714 # yes, we really want os._exit
1720 if (pid == 0): # The first child.
1723 pid = os.fork() # Fork a second child.
1724 if (pid == 0): # The second child.
1728 # exit() or _exit()? See below.
1729 os._exit(0) # Exit parent (the first child) of the second child.
1731 os._exit(0) # Exit parent of the first child.
1735 i = os.open("/dev/null", os.O_RDONLY) # stdin
1736 assert i == 0, "Can't close/reopen stdin"
1737 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1738 assert i == 1, "Can't close/reopen stdout"
1739 # Duplicate standard output to standard error.
1744 def DaemonPidFileName(name):
1745 """Compute a ganeti pid file absolute path
1748 @param name: the daemon name
1750 @return: the full path to the pidfile corresponding to the given
1754 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
1757 def EnsureDaemon(name):
1758 """Check for and start daemon if not alive.
1761 result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
1763 logging.error("Can't start daemon '%s', failure %s, output: %s",
1764 name, result.fail_reason, result.output)
1770 def WritePidFile(name):
1771 """Write the current process pidfile.
1773 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1776 @param name: the daemon name to use
1777 @raise errors.GenericError: if the pid file already exists and
1778 points to a live process
1782 pidfilename = DaemonPidFileName(name)
1783 if IsProcessAlive(ReadPidFile(pidfilename)):
1784 raise errors.GenericError("%s contains a live process" % pidfilename)
1786 WriteFile(pidfilename, data="%d\n" % pid)
1789 def RemovePidFile(name):
1790 """Remove the current process pidfile.
1792 Any errors are ignored.
1795 @param name: the daemon name used to derive the pidfile name
1798 pidfilename = DaemonPidFileName(name)
1799 # TODO: we could check here that the file contains our pid
1801 RemoveFile(pidfilename)
1802 except: # pylint: disable-msg=W0702
1806 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1808 """Kill a process given by its pid.
1811 @param pid: The PID to terminate.
1813 @param signal_: The signal to send, by default SIGTERM
1815 @param timeout: The timeout after which, if the process is still alive,
1816 a SIGKILL will be sent. If not positive, no such checking
1818 @type waitpid: boolean
1819 @param waitpid: If true, we should waitpid on this process after
1820 sending signals, since it's our own child and otherwise it
1821 would remain as zombie
1824 def _helper(pid, signal_, wait):
1825 """Simple helper to encapsulate the kill/waitpid sequence"""
1826 os.kill(pid, signal_)
1829 os.waitpid(pid, os.WNOHANG)
1834 # kill with pid=0 == suicide
1835 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1837 if not IsProcessAlive(pid):
1840 _helper(pid, signal_, waitpid)
1845 def _CheckProcess():
1846 if not IsProcessAlive(pid):
1850 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1860 # Wait up to $timeout seconds
1861 Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1862 except RetryTimeout:
1865 if IsProcessAlive(pid):
1866 # Kill process if it's still alive
1867 _helper(pid, signal.SIGKILL, waitpid)
1870 def FindFile(name, search_path, test=os.path.exists):
1871 """Look for a filesystem object in a given path.
1873 This is an abstract method to search for filesystem object (files,
1874 dirs) under a given search path.
1877 @param name: the name to look for
1878 @type search_path: str
1879 @param search_path: location to start at
1880 @type test: callable
1881 @param test: a function taking one argument that should return True
1882 if the a given object is valid; the default value is
1883 os.path.exists, causing only existing files to be returned
1885 @return: full path to the object if found, None otherwise
1888 # validate the filename mask
1889 if constants.EXT_PLUGIN_MASK.match(name) is None:
1890 logging.critical("Invalid value passed for external script name: '%s'",
1894 for dir_name in search_path:
1895 # FIXME: investigate switch to PathJoin
1896 item_name = os.path.sep.join([dir_name, name])
1897 # check the user test and that we're indeed resolving to the given
1899 if test(item_name) and os.path.basename(item_name) == name:
1904 def CheckVolumeGroupSize(vglist, vgname, minsize):
1905 """Checks if the volume group list is valid.
1907 The function will check if a given volume group is in the list of
1908 volume groups and has a minimum size.
1911 @param vglist: dictionary of volume group names and their size
1913 @param vgname: the volume group we should check
1915 @param minsize: the minimum size we accept
1917 @return: None for success, otherwise the error message
1920 vgsize = vglist.get(vgname, None)
1922 return "volume group '%s' missing" % vgname
1923 elif vgsize < minsize:
1924 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1925 (vgname, minsize, vgsize))
1929 def SplitTime(value):
1930 """Splits time as floating point number into a tuple.
1932 @param value: Time in seconds
1933 @type value: int or float
1934 @return: Tuple containing (seconds, microseconds)
1937 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1939 assert 0 <= seconds, \
1940 "Seconds must be larger than or equal to 0, but are %s" % seconds
1941 assert 0 <= microseconds <= 999999, \
1942 "Microseconds must be 0-999999, but are %s" % microseconds
1944 return (int(seconds), int(microseconds))
1947 def MergeTime(timetuple):
1948 """Merges a tuple into time as a floating point number.
1950 @param timetuple: Time as tuple, (seconds, microseconds)
1951 @type timetuple: tuple
1952 @return: Time as a floating point number expressed in seconds
1955 (seconds, microseconds) = timetuple
1957 assert 0 <= seconds, \
1958 "Seconds must be larger than or equal to 0, but are %s" % seconds
1959 assert 0 <= microseconds <= 999999, \
1960 "Microseconds must be 0-999999, but are %s" % microseconds
1962 return float(seconds) + (float(microseconds) * 0.000001)
1965 def GetDaemonPort(daemon_name):
1966 """Get the daemon port for this cluster.
1968 Note that this routine does not read a ganeti-specific file, but
1969 instead uses C{socket.getservbyname} to allow pre-customization of
1970 this parameter outside of Ganeti.
1972 @type daemon_name: string
1973 @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1977 if daemon_name not in constants.DAEMONS_PORTS:
1978 raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1980 (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1982 port = socket.getservbyname(daemon_name, proto)
1983 except socket.error:
1989 def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
1990 multithreaded=False, syslog=constants.SYSLOG_USAGE):
1991 """Configures the logging module.
1994 @param logfile: the filename to which we should log
1995 @type debug: integer
1996 @param debug: if greater than zero, enable debug messages, otherwise
1997 only those at C{INFO} and above level
1998 @type stderr_logging: boolean
1999 @param stderr_logging: whether we should also log to the standard error
2001 @param program: the name under which we should log messages
2002 @type multithreaded: boolean
2003 @param multithreaded: if True, will add the thread name to the log file
2004 @type syslog: string
2005 @param syslog: one of 'no', 'yes', 'only':
2006 - if no, syslog is not used
2007 - if yes, syslog is used (in addition to file-logging)
2008 - if only, only syslog is used
2009 @raise EnvironmentError: if we can't open the log file and
2010 syslog/stderr logging is disabled
2013 fmt = "%(asctime)s: " + program + " pid=%(process)d"
2014 sft = program + "[%(process)d]:"
2016 fmt += "/%(threadName)s"
2017 sft += " (%(threadName)s)"
2019 fmt += " %(module)s:%(lineno)s"
2020 # no debug info for syslog loggers
2021 fmt += " %(levelname)s %(message)s"
2022 # yes, we do want the textual level, as remote syslog will probably
2023 # lose the error level, and it's easier to grep for it
2024 sft += " %(levelname)s %(message)s"
2025 formatter = logging.Formatter(fmt)
2026 sys_fmt = logging.Formatter(sft)
2028 root_logger = logging.getLogger("")
2029 root_logger.setLevel(logging.NOTSET)
2031 # Remove all previously setup handlers
2032 for handler in root_logger.handlers:
2034 root_logger.removeHandler(handler)
2037 stderr_handler = logging.StreamHandler()
2038 stderr_handler.setFormatter(formatter)
2040 stderr_handler.setLevel(logging.NOTSET)
2042 stderr_handler.setLevel(logging.CRITICAL)
2043 root_logger.addHandler(stderr_handler)
2045 if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
2046 facility = logging.handlers.SysLogHandler.LOG_DAEMON
2047 syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
2049 syslog_handler.setFormatter(sys_fmt)
2050 # Never enable debug over syslog
2051 syslog_handler.setLevel(logging.INFO)
2052 root_logger.addHandler(syslog_handler)
2054 if syslog != constants.SYSLOG_ONLY:
2055 # this can fail, if the logging directories are not setup or we have
2056 # a permisssion problem; in this case, it's best to log but ignore
2057 # the error if stderr_logging is True, and if false we re-raise the
2058 # exception since otherwise we could run but without any logs at all
2060 logfile_handler = logging.FileHandler(logfile)
2061 logfile_handler.setFormatter(formatter)
2063 logfile_handler.setLevel(logging.DEBUG)
2065 logfile_handler.setLevel(logging.INFO)
2066 root_logger.addHandler(logfile_handler)
2067 except EnvironmentError:
2068 if stderr_logging or syslog == constants.SYSLOG_YES:
2069 logging.exception("Failed to enable logging to file '%s'", logfile)
2071 # we need to re-raise the exception
2075 def IsNormAbsPath(path):
2076 """Check whether a path is absolute and also normalized
2078 This avoids things like /dir/../../other/path to be valid.
2081 return os.path.normpath(path) == path and os.path.isabs(path)
2084 def PathJoin(*args):
2085 """Safe-join a list of path components.
2088 - the first argument must be an absolute path
2089 - no component in the path must have backtracking (e.g. /../),
2090 since we check for normalization at the end
2092 @param args: the path components to be joined
2093 @raise ValueError: for invalid paths
2096 # ensure we're having at least one path passed in
2098 # ensure the first component is an absolute and normalized path name
2100 if not IsNormAbsPath(root):
2101 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
2102 result = os.path.join(*args)
2103 # ensure that the whole path is normalized
2104 if not IsNormAbsPath(result):
2105 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
2106 # check that we're still under the original prefix
2107 prefix = os.path.commonprefix([root, result])
2109 raise ValueError("Error: path joining resulted in different prefix"
2110 " (%s != %s)" % (prefix, root))
2114 def TailFile(fname, lines=20):
2115 """Return the last lines from a file.
2117 @note: this function will only read and parse the last 4KB of
2118 the file; if the lines are very long, it could be that less
2119 than the requested number of lines are returned
2121 @param fname: the file name
2123 @param lines: the (maximum) number of lines to return
2126 fd = open(fname, "r")
2130 pos = max(0, pos-4096)
2132 raw_data = fd.read()
2136 rows = raw_data.splitlines()
2137 return rows[-lines:]
2140 def _ParseAsn1Generalizedtime(value):
2141 """Parses an ASN1 GENERALIZEDTIME timestamp as used by pyOpenSSL.
2144 @param value: ASN1 GENERALIZEDTIME timestamp
2147 m = re.match(r"^(\d+)([-+]\d\d)(\d\d)$", value)
2150 asn1time = m.group(1)
2151 hours = int(m.group(2))
2152 minutes = int(m.group(3))
2153 utcoffset = (60 * hours) + minutes
2155 if not value.endswith("Z"):
2156 raise ValueError("Missing timezone")
2157 asn1time = value[:-1]
2160 parsed = time.strptime(asn1time, "%Y%m%d%H%M%S")
2162 tt = datetime.datetime(*(parsed[:7])) - datetime.timedelta(minutes=utcoffset)
2164 return calendar.timegm(tt.utctimetuple())
2167 def GetX509CertValidity(cert):
2168 """Returns the validity period of the certificate.
2170 @type cert: OpenSSL.crypto.X509
2171 @param cert: X509 certificate object
2174 # The get_notBefore and get_notAfter functions are only supported in
2175 # pyOpenSSL 0.7 and above.
2177 get_notbefore_fn = cert.get_notBefore
2178 except AttributeError:
2181 not_before_asn1 = get_notbefore_fn()
2183 if not_before_asn1 is None:
2186 not_before = _ParseAsn1Generalizedtime(not_before_asn1)
2189 get_notafter_fn = cert.get_notAfter
2190 except AttributeError:
2193 not_after_asn1 = get_notafter_fn()
2195 if not_after_asn1 is None:
2198 not_after = _ParseAsn1Generalizedtime(not_after_asn1)
2200 return (not_before, not_after)
2203 def SafeEncode(text):
2204 """Return a 'safe' version of a source string.
2206 This function mangles the input string and returns a version that
2207 should be safe to display/encode as ASCII. To this end, we first
2208 convert it to ASCII using the 'backslashreplace' encoding which
2209 should get rid of any non-ASCII chars, and then we process it
2210 through a loop copied from the string repr sources in the python; we
2211 don't use string_escape anymore since that escape single quotes and
2212 backslashes too, and that is too much; and that escaping is not
2213 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2215 @type text: str or unicode
2216 @param text: input data
2218 @return: a safe version of text
2221 if isinstance(text, unicode):
2222 # only if unicode; if str already, we handle it below
2223 text = text.encode('ascii', 'backslashreplace')
2233 elif c < 32 or c >= 127: # non-printable
2234 resu += "\\x%02x" % (c & 0xff)
2240 def UnescapeAndSplit(text, sep=","):
2241 """Split and unescape a string based on a given separator.
2243 This function splits a string based on a separator where the
2244 separator itself can be escape in order to be an element of the
2245 elements. The escaping rules are (assuming coma being the
2247 - a plain , separates the elements
2248 - a sequence \\\\, (double backslash plus comma) is handled as a
2249 backslash plus a separator comma
2250 - a sequence \, (backslash plus comma) is handled as a
2254 @param text: the string to split
2256 @param text: the separator
2258 @return: a list of strings
2261 # we split the list by sep (with no escaping at this stage)
2262 slist = text.split(sep)
2263 # next, we revisit the elements and if any of them ended with an odd
2264 # number of backslashes, then we join it with the next
2268 if e1.endswith("\\"):
2269 num_b = len(e1) - len(e1.rstrip("\\"))
2272 # here the backslashes remain (all), and will be reduced in
2274 rlist.append(e1 + sep + e2)
2277 # finally, replace backslash-something with something
2278 rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2282 def CommaJoin(names):
2283 """Nicely join a set of identifiers.
2285 @param names: set, list or tuple
2286 @return: a string with the formatted results
2289 return ", ".join([str(val) for val in names])
2292 def BytesToMebibyte(value):
2293 """Converts bytes to mebibytes.
2296 @param value: Value in bytes
2298 @return: Value in mebibytes
2301 return int(round(value / (1024.0 * 1024.0), 0))
2304 def CalculateDirectorySize(path):
2305 """Calculates the size of a directory recursively.
2308 @param path: Path to directory
2310 @return: Size in mebibytes
2315 for (curpath, _, files) in os.walk(path):
2316 for filename in files:
2317 st = os.lstat(PathJoin(curpath, filename))
2320 return BytesToMebibyte(size)
2323 def GetFilesystemStats(path):
2324 """Returns the total and free space on a filesystem.
2327 @param path: Path on filesystem to be examined
2329 @return: tuple of (Total space, Free space) in mebibytes
2332 st = os.statvfs(path)
2334 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2335 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2336 return (tsize, fsize)
2339 def RunInSeparateProcess(fn, *args):
2340 """Runs a function in a separate process.
2342 Note: Only boolean return values are supported.
2345 @param fn: Function to be called
2347 @return: Function's result
2354 # In case the function uses temporary files
2355 ResetTempfileModule()
2358 result = int(bool(fn(*args)))
2359 assert result in (0, 1)
2360 except: # pylint: disable-msg=W0702
2361 logging.exception("Error while calling function in separate process")
2362 # 0 and 1 are reserved for the return value
2365 os._exit(result) # pylint: disable-msg=W0212
2369 # Avoid zombies and check exit code
2370 (_, status) = os.waitpid(pid, 0)
2372 if os.WIFSIGNALED(status):
2374 signum = os.WTERMSIG(status)
2376 exitcode = os.WEXITSTATUS(status)
2379 if not (exitcode in (0, 1) and signum is None):
2380 raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2383 return bool(exitcode)
2386 def LockedMethod(fn):
2387 """Synchronized object access decorator.
2389 This decorator is intended to protect access to an object using the
2390 object's own lock which is hardcoded to '_lock'.
2393 def _LockDebug(*args, **kwargs):
2395 logging.debug(*args, **kwargs)
2397 def wrapper(self, *args, **kwargs):
2398 # pylint: disable-msg=W0212
2399 assert hasattr(self, '_lock')
2401 _LockDebug("Waiting for %s", lock)
2404 _LockDebug("Acquired %s", lock)
2405 result = fn(self, *args, **kwargs)
2407 _LockDebug("Releasing %s", lock)
2409 _LockDebug("Released %s", lock)
2415 """Locks a file using POSIX locks.
2418 @param fd: the file descriptor we need to lock
2422 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2423 except IOError, err:
2424 if err.errno == errno.EAGAIN:
2425 raise errors.LockError("File already locked")
2429 def FormatTime(val):
2430 """Formats a time value.
2432 @type val: float or None
2433 @param val: the timestamp as returned by time.time()
2434 @return: a string value or N/A if we don't have a valid timestamp
2437 if val is None or not isinstance(val, (int, float)):
2439 # these two codes works on Linux, but they are not guaranteed on all
2441 return time.strftime("%F %T", time.localtime(val))
2444 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2445 """Reads the watcher pause file.
2447 @type filename: string
2448 @param filename: Path to watcher pause file
2449 @type now: None, float or int
2450 @param now: Current time as Unix timestamp
2451 @type remove_after: int
2452 @param remove_after: Remove watcher pause file after specified amount of
2453 seconds past the pause end time
2460 value = ReadFile(filename)
2461 except IOError, err:
2462 if err.errno != errno.ENOENT:
2466 if value is not None:
2470 logging.warning(("Watcher pause file (%s) contains invalid value,"
2471 " removing it"), filename)
2472 RemoveFile(filename)
2475 if value is not None:
2476 # Remove file if it's outdated
2477 if now > (value + remove_after):
2478 RemoveFile(filename)
2487 class RetryTimeout(Exception):
2488 """Retry loop timed out.
2493 class RetryAgain(Exception):
2499 class _RetryDelayCalculator(object):
2500 """Calculator for increasing delays.
2510 def __init__(self, start, factor, limit):
2511 """Initializes this class.
2514 @param start: Initial delay
2516 @param factor: Factor for delay increase
2517 @type limit: float or None
2518 @param limit: Upper limit for delay or None for no limit
2522 assert factor >= 1.0
2523 assert limit is None or limit >= 0.0
2526 self._factor = factor
2532 """Returns current delay and calculates the next one.
2535 current = self._next
2537 # Update for next run
2538 if self._limit is None or self._next < self._limit:
2539 self._next = min(self._limit, self._next * self._factor)
2544 #: Special delay to specify whole remaining timeout
2545 RETRY_REMAINING_TIME = object()
2548 def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2549 _time_fn=time.time):
2550 """Call a function repeatedly until it succeeds.
2552 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2553 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2554 total of C{timeout} seconds, this function throws L{RetryTimeout}.
2556 C{delay} can be one of the following:
2557 - callable returning the delay length as a float
2558 - Tuple of (start, factor, limit)
2559 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2560 useful when overriding L{wait_fn} to wait for an external event)
2561 - A static delay as a number (int or float)
2564 @param fn: Function to be called
2565 @param delay: Either a callable (returning the delay), a tuple of (start,
2566 factor, limit) (see L{_RetryDelayCalculator}),
2567 L{RETRY_REMAINING_TIME} or a number (int or float)
2568 @type timeout: float
2569 @param timeout: Total timeout
2570 @type wait_fn: callable
2571 @param wait_fn: Waiting function
2572 @return: Return value of function
2576 assert callable(wait_fn)
2577 assert callable(_time_fn)
2582 end_time = _time_fn() + timeout
2585 # External function to calculate delay
2588 elif isinstance(delay, (tuple, list)):
2589 # Increasing delay with optional upper boundary
2590 (start, factor, limit) = delay
2591 calc_delay = _RetryDelayCalculator(start, factor, limit)
2593 elif delay is RETRY_REMAINING_TIME:
2594 # Always use the remaining time
2599 calc_delay = lambda: delay
2601 assert calc_delay is None or callable(calc_delay)
2605 # pylint: disable-msg=W0142
2610 remaining_time = end_time - _time_fn()
2612 if remaining_time < 0.0:
2613 raise RetryTimeout()
2615 assert remaining_time >= 0.0
2617 if calc_delay is None:
2618 wait_fn(remaining_time)
2620 current_delay = calc_delay()
2621 if current_delay > 0.0:
2622 wait_fn(current_delay)
2625 class FileLock(object):
2626 """Utility class for file locks.
2629 def __init__(self, fd, filename):
2630 """Constructor for FileLock.
2633 @param fd: File object
2635 @param filename: Path of the file opened at I{fd}
2639 self.filename = filename
2642 def Open(cls, filename):
2643 """Creates and opens a file to be used as a file-based lock.
2645 @type filename: string
2646 @param filename: path to the file to be locked
2649 # Using "os.open" is necessary to allow both opening existing file
2650 # read/write and creating if not existing. Vanilla "open" will truncate an
2651 # existing file -or- allow creating if not existing.
2652 return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
2659 """Close the file and release the lock.
2662 if hasattr(self, "fd") and self.fd:
2666 def _flock(self, flag, blocking, timeout, errmsg):
2667 """Wrapper for fcntl.flock.
2670 @param flag: operation flag
2671 @type blocking: bool
2672 @param blocking: whether the operation should be done in blocking mode.
2673 @type timeout: None or float
2674 @param timeout: for how long the operation should be retried (implies
2676 @type errmsg: string
2677 @param errmsg: error message in case operation fails.
2680 assert self.fd, "Lock was closed"
2681 assert timeout is None or timeout >= 0, \
2682 "If specified, timeout must be positive"
2683 assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
2685 # When a timeout is used, LOCK_NB must always be set
2686 if not (timeout is None and blocking):
2687 flag |= fcntl.LOCK_NB
2690 self._Lock(self.fd, flag, timeout)
2693 Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
2694 args=(self.fd, flag, timeout))
2695 except RetryTimeout:
2696 raise errors.LockError(errmsg)
2699 def _Lock(fd, flag, timeout):
2701 fcntl.flock(fd, flag)
2702 except IOError, err:
2703 if timeout is not None and err.errno == errno.EAGAIN:
2706 logging.exception("fcntl.flock failed")
2709 def Exclusive(self, blocking=False, timeout=None):
2710 """Locks the file in exclusive mode.
2712 @type blocking: boolean
2713 @param blocking: whether to block and wait until we
2714 can lock the file or return immediately
2715 @type timeout: int or None
2716 @param timeout: if not None, the duration to wait for the lock
2720 self._flock(fcntl.LOCK_EX, blocking, timeout,
2721 "Failed to lock %s in exclusive mode" % self.filename)
2723 def Shared(self, blocking=False, timeout=None):
2724 """Locks the file in shared mode.
2726 @type blocking: boolean
2727 @param blocking: whether to block and wait until we
2728 can lock the file or return immediately
2729 @type timeout: int or None
2730 @param timeout: if not None, the duration to wait for the lock
2734 self._flock(fcntl.LOCK_SH, blocking, timeout,
2735 "Failed to lock %s in shared mode" % self.filename)
2737 def Unlock(self, blocking=True, timeout=None):
2738 """Unlocks the file.
2740 According to C{flock(2)}, unlocking can also be a nonblocking
2743 To make a non-blocking request, include LOCK_NB with any of the above
2746 @type blocking: boolean
2747 @param blocking: whether to block and wait until we
2748 can lock the file or return immediately
2749 @type timeout: int or None
2750 @param timeout: if not None, the duration to wait for the lock
2754 self._flock(fcntl.LOCK_UN, blocking, timeout,
2755 "Failed to unlock %s" % self.filename)
2758 def SignalHandled(signums):
2759 """Signal Handled decoration.
2761 This special decorator installs a signal handler and then calls the target
2762 function. The function must accept a 'signal_handlers' keyword argument,
2763 which will contain a dict indexed by signal number, with SignalHandler
2766 The decorator can be safely stacked with iself, to handle multiple signals
2767 with different handlers.
2770 @param signums: signals to intercept
2774 def sig_function(*args, **kwargs):
2775 assert 'signal_handlers' not in kwargs or \
2776 kwargs['signal_handlers'] is None or \
2777 isinstance(kwargs['signal_handlers'], dict), \
2778 "Wrong signal_handlers parameter in original function call"
2779 if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2780 signal_handlers = kwargs['signal_handlers']
2782 signal_handlers = {}
2783 kwargs['signal_handlers'] = signal_handlers
2784 sighandler = SignalHandler(signums)
2787 signal_handlers[sig] = sighandler
2788 return fn(*args, **kwargs)
2795 class SignalHandler(object):
2796 """Generic signal handler class.
2798 It automatically restores the original handler when deconstructed or
2799 when L{Reset} is called. You can either pass your own handler
2800 function in or query the L{called} attribute to detect whether the
2804 @ivar signum: the signals we handle
2805 @type called: boolean
2806 @ivar called: tracks whether any of the signals have been raised
2809 def __init__(self, signum):
2810 """Constructs a new SignalHandler instance.
2812 @type signum: int or list of ints
2813 @param signum: Single signal number or set of signal numbers
2816 self.signum = set(signum)
2821 for signum in self.signum:
2823 prev_handler = signal.signal(signum, self._HandleSignal)
2825 self._previous[signum] = prev_handler
2827 # Restore previous handler
2828 signal.signal(signum, prev_handler)
2831 # Reset all handlers
2833 # Here we have a race condition: a handler may have already been called,
2834 # but there's not much we can do about it at this point.
2841 """Restore previous handler.
2843 This will reset all the signals to their previous handlers.
2846 for signum, prev_handler in self._previous.items():
2847 signal.signal(signum, prev_handler)
2848 # If successful, remove from dict
2849 del self._previous[signum]
2852 """Unsets the L{called} flag.
2854 This function can be used in case a signal may arrive several times.
2859 # we don't care about arguments, but we leave them named for the future
2860 def _HandleSignal(self, signum, frame): # pylint: disable-msg=W0613
2861 """Actual signal handling function.
2864 # This is not nice and not absolutely atomic, but it appears to be the only
2865 # solution in Python -- there are no atomic types.
2869 class FieldSet(object):
2870 """A simple field set.
2872 Among the features are:
2873 - checking if a string is among a list of static string or regex objects
2874 - checking if a whole list of string matches
2875 - returning the matching groups from a regex match
2877 Internally, all fields are held as regular expression objects.
2880 def __init__(self, *items):
2881 self.items = [re.compile("^%s$" % value) for value in items]
2883 def Extend(self, other_set):
2884 """Extend the field set with the items from another one"""
2885 self.items.extend(other_set.items)
2887 def Matches(self, field):
2888 """Checks if a field matches the current set
2891 @param field: the string to match
2892 @return: either None or a regular expression match object
2895 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2899 def NonMatching(self, items):
2900 """Returns the list of fields not matching the current set
2903 @param items: the list of fields to check
2905 @return: list of non-matching fields
2908 return [val for val in items if not self.Matches(val)]