4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
44 import logging.handlers
50 from cStringIO import StringIO
53 from hashlib import sha1
58 from ganeti import errors
59 from ganeti import constants
63 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
67 #: when set to True, L{RunCmd} is disabled
70 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
73 class RunResult(object):
74 """Holds the result of running external programs.
77 @ivar exit_code: the exit code of the program, or None (if the program
79 @type signal: int or None
80 @ivar signal: the signal that caused the program to finish, or None
81 (if the program wasn't terminated by a signal)
83 @ivar stdout: the standard output of the program
85 @ivar stderr: the standard error of the program
87 @ivar failed: True in case the program was
88 terminated by a signal or exited with a non-zero exit code
89 @ivar fail_reason: a string detailing the termination reason
92 __slots__ = ["exit_code", "signal", "stdout", "stderr",
93 "failed", "fail_reason", "cmd"]
96 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
98 self.exit_code = exit_code
102 self.failed = (signal_ is not None or exit_code != 0)
104 if self.signal is not None:
105 self.fail_reason = "terminated by signal %s" % self.signal
106 elif self.exit_code is not None:
107 self.fail_reason = "exited with exit code %s" % self.exit_code
109 self.fail_reason = "unable to determine termination reason"
112 logging.debug("Command '%s' failed (%s); output: %s",
113 self.cmd, self.fail_reason, self.output)
115 def _GetOutput(self):
116 """Returns the combined stdout and stderr for easier usage.
119 return self.stdout + self.stderr
121 output = property(_GetOutput, None, None, "Return full output")
124 def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
125 """Execute a (shell) command.
127 The command should not read from its standard input, as it will be
130 @type cmd: string or list
131 @param cmd: Command to run
133 @param env: Additional environment
135 @param output: if desired, the output of the command can be
136 saved in a file instead of the RunResult instance; this
137 parameter denotes the file name (if not None)
139 @param cwd: if specified, will be used as the working
140 directory for the command; the default will be /
141 @type reset_env: boolean
142 @param reset_env: whether to reset or keep the default os environment
144 @return: RunResult instance
145 @raise errors.ProgrammerError: if we call this when forks are disabled
149 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
151 if isinstance(cmd, list):
152 cmd = [str(val) for val in cmd]
153 strcmd = " ".join(cmd)
158 logging.debug("RunCmd '%s'", strcmd)
161 cmd_env = os.environ.copy()
162 cmd_env["LC_ALL"] = "C"
171 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
173 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
176 if err.errno == errno.ENOENT:
177 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
189 return RunResult(exitcode, signal_, out, err, strcmd)
192 def _RunCmdPipe(cmd, env, via_shell, cwd):
193 """Run a command and return its output.
195 @type cmd: string or list
196 @param cmd: Command to run
198 @param env: The environment to use
199 @type via_shell: bool
200 @param via_shell: if we should run via the shell
202 @param cwd: the working directory for the program
204 @return: (out, err, status)
207 poller = select.poll()
208 child = subprocess.Popen(cmd, shell=via_shell,
209 stderr=subprocess.PIPE,
210 stdout=subprocess.PIPE,
211 stdin=subprocess.PIPE,
212 close_fds=True, env=env,
216 poller.register(child.stdout, select.POLLIN)
217 poller.register(child.stderr, select.POLLIN)
221 child.stdout.fileno(): (out, child.stdout),
222 child.stderr.fileno(): (err, child.stderr),
225 status = fcntl.fcntl(fd, fcntl.F_GETFL)
226 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
230 pollresult = poller.poll()
231 except EnvironmentError, eerr:
232 if eerr.errno == errno.EINTR:
235 except select.error, serr:
236 if serr[0] == errno.EINTR:
240 for fd, event in pollresult:
241 if event & select.POLLIN or event & select.POLLPRI:
242 data = fdmap[fd][1].read()
243 # no data from read signifies EOF (the same as POLLHUP)
245 poller.unregister(fd)
248 fdmap[fd][0].write(data)
249 if (event & select.POLLNVAL or event & select.POLLHUP or
250 event & select.POLLERR):
251 poller.unregister(fd)
257 status = child.wait()
258 return out, err, status
261 def _RunCmdFile(cmd, env, via_shell, output, cwd):
262 """Run a command and save its output to a file.
264 @type cmd: string or list
265 @param cmd: Command to run
267 @param env: The environment to use
268 @type via_shell: bool
269 @param via_shell: if we should run via the shell
271 @param output: the filename in which to save the output
273 @param cwd: the working directory for the program
275 @return: the exit status
278 fh = open(output, "a")
280 child = subprocess.Popen(cmd, shell=via_shell,
281 stderr=subprocess.STDOUT,
283 stdin=subprocess.PIPE,
284 close_fds=True, env=env,
288 status = child.wait()
294 def RunParts(dir_name, env=None, reset_env=False):
295 """Run Scripts or programs in a directory
297 @type dir_name: string
298 @param dir_name: absolute path to a directory
300 @param env: The environment to use
301 @type reset_env: boolean
302 @param reset_env: whether to reset or keep the default os environment
303 @rtype: list of tuples
304 @return: list of (name, (one of RUNDIR_STATUS), RunResult)
310 dir_contents = ListVisibleFiles(dir_name)
312 logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
315 for relname in sorted(dir_contents):
316 fname = PathJoin(dir_name, relname)
317 if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
318 constants.EXT_PLUGIN_MASK.match(relname) is not None):
319 rr.append((relname, constants.RUNPARTS_SKIP, None))
322 result = RunCmd([fname], env=env, reset_env=reset_env)
323 except Exception, err: # pylint: disable-msg=W0703
324 rr.append((relname, constants.RUNPARTS_ERR, str(err)))
326 rr.append((relname, constants.RUNPARTS_RUN, result))
331 def RemoveFile(filename):
332 """Remove a file ignoring some errors.
334 Remove a file, ignoring non-existing ones or directories. Other
338 @param filename: the file to be removed
344 if err.errno not in (errno.ENOENT, errno.EISDIR):
348 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
352 @param old: Original path
356 @param mkdir: Whether to create target directory if it doesn't exist
357 @type mkdir_mode: int
358 @param mkdir_mode: Mode for newly created directories
362 return os.rename(old, new)
364 # In at least one use case of this function, the job queue, directory
365 # creation is very rare. Checking for the directory before renaming is not
367 if mkdir and err.errno == errno.ENOENT:
368 # Create directory and try again
369 Makedirs(os.path.dirname(new), mode=mkdir_mode)
371 return os.rename(old, new)
376 def Makedirs(path, mode=0750):
377 """Super-mkdir; create a leaf directory and all intermediate ones.
379 This is a wrapper around C{os.makedirs} adding error handling not implemented
384 os.makedirs(path, mode)
386 # Ignore EEXIST. This is only handled in os.makedirs as included in
387 # Python 2.5 and above.
388 if err.errno != errno.EEXIST or not os.path.exists(path):
392 def ResetTempfileModule():
393 """Resets the random name generator of the tempfile module.
395 This function should be called after C{os.fork} in the child process to
396 ensure it creates a newly seeded random generator. Otherwise it would
397 generate the same random parts as the parent process. If several processes
398 race for the creation of a temporary file, this could lead to one not getting
402 # pylint: disable-msg=W0212
403 if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
404 tempfile._once_lock.acquire()
406 # Reset random name generator
407 tempfile._name_sequence = None
409 tempfile._once_lock.release()
411 logging.critical("The tempfile module misses at least one of the"
412 " '_once_lock' and '_name_sequence' attributes")
415 def _FingerprintFile(filename):
416 """Compute the fingerprint of a file.
418 If the file does not exist, a None will be returned
422 @param filename: the filename to checksum
424 @return: the hex digest of the sha checksum of the contents
428 if not (os.path.exists(filename) and os.path.isfile(filename)):
441 return fp.hexdigest()
444 def FingerprintFiles(files):
445 """Compute fingerprints for a list of files.
448 @param files: the list of filename to fingerprint
450 @return: a dictionary filename: fingerprint, holding only
456 for filename in files:
457 cksum = _FingerprintFile(filename)
459 ret[filename] = cksum
464 def ForceDictType(target, key_types, allowed_values=None):
465 """Force the values of a dict to have certain types.
468 @param target: the dict to update
469 @type key_types: dict
470 @param key_types: dict mapping target dict keys to types
471 in constants.ENFORCEABLE_TYPES
472 @type allowed_values: list
473 @keyword allowed_values: list of specially allowed values
476 if allowed_values is None:
479 if not isinstance(target, dict):
480 msg = "Expected dictionary, got '%s'" % target
481 raise errors.TypeEnforcementError(msg)
484 if key not in key_types:
485 msg = "Unknown key '%s'" % key
486 raise errors.TypeEnforcementError(msg)
488 if target[key] in allowed_values:
491 ktype = key_types[key]
492 if ktype not in constants.ENFORCEABLE_TYPES:
493 msg = "'%s' has non-enforceable type %s" % (key, ktype)
494 raise errors.ProgrammerError(msg)
496 if ktype == constants.VTYPE_STRING:
497 if not isinstance(target[key], basestring):
498 if isinstance(target[key], bool) and not target[key]:
501 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
502 raise errors.TypeEnforcementError(msg)
503 elif ktype == constants.VTYPE_BOOL:
504 if isinstance(target[key], basestring) and target[key]:
505 if target[key].lower() == constants.VALUE_FALSE:
507 elif target[key].lower() == constants.VALUE_TRUE:
510 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
511 raise errors.TypeEnforcementError(msg)
516 elif ktype == constants.VTYPE_SIZE:
518 target[key] = ParseUnit(target[key])
519 except errors.UnitParseError, err:
520 msg = "'%s' (value %s) is not a valid size. error: %s" % \
521 (key, target[key], err)
522 raise errors.TypeEnforcementError(msg)
523 elif ktype == constants.VTYPE_INT:
525 target[key] = int(target[key])
526 except (ValueError, TypeError):
527 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
528 raise errors.TypeEnforcementError(msg)
531 def IsProcessAlive(pid):
532 """Check if a given pid exists on the system.
534 @note: zombie status is not handled, so zombie processes
535 will be returned as alive
537 @param pid: the process ID to check
539 @return: True if the process exists
546 os.stat("/proc/%d/status" % pid)
548 except EnvironmentError, err:
549 if err.errno in (errno.ENOENT, errno.ENOTDIR):
554 def ReadPidFile(pidfile):
555 """Read a pid from a file.
557 @type pidfile: string
558 @param pidfile: path to the file containing the pid
560 @return: The process id, if the file exists and contains a valid PID,
565 raw_data = ReadFile(pidfile)
566 except EnvironmentError, err:
567 if err.errno != errno.ENOENT:
568 logging.exception("Can't read pid file")
573 except (TypeError, ValueError), err:
574 logging.info("Can't parse pid file contents", exc_info=True)
580 def MatchNameComponent(key, name_list, case_sensitive=True):
581 """Try to match a name against a list.
583 This function will try to match a name like test1 against a list
584 like C{['test1.example.com', 'test2.example.com', ...]}. Against
585 this list, I{'test1'} as well as I{'test1.example'} will match, but
586 not I{'test1.ex'}. A multiple match will be considered as no match
587 at all (e.g. I{'test1'} against C{['test1.example.com',
588 'test1.example.org']}), except when the key fully matches an entry
589 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
592 @param key: the name to be searched
593 @type name_list: list
594 @param name_list: the list of strings against which to search the key
595 @type case_sensitive: boolean
596 @param case_sensitive: whether to provide a case-sensitive match
599 @return: None if there is no match I{or} if there are multiple matches,
600 otherwise the element from the list which matches
607 if not case_sensitive:
608 re_flags |= re.IGNORECASE
610 mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
613 for name in name_list:
614 if mo.match(name) is not None:
615 names_filtered.append(name)
616 if not case_sensitive and key == name.upper():
617 string_matches.append(name)
619 if len(string_matches) == 1:
620 return string_matches[0]
621 if len(names_filtered) == 1:
622 return names_filtered[0]
627 """Class implementing resolver and hostname functionality
630 _VALID_NAME_RE = re.compile("^[a-z0-9._-]{1,255}$")
632 def __init__(self, name=None):
633 """Initialize the host name object.
635 If the name argument is not passed, it will use this system's
640 name = self.SysName()
643 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
644 self.ip = self.ipaddrs[0]
647 """Returns the hostname without domain.
650 return self.name.split('.')[0]
654 """Return the current system's name.
656 This is simply a wrapper over C{socket.gethostname()}.
659 return socket.gethostname()
662 def LookupHostname(hostname):
666 @param hostname: hostname to look up
669 @return: a tuple (name, aliases, ipaddrs) as returned by
670 C{socket.gethostbyname_ex}
671 @raise errors.ResolverError: in case of errors in resolving
675 result = socket.gethostbyname_ex(hostname)
676 except socket.gaierror, err:
677 # hostname not found in DNS
678 raise errors.ResolverError(hostname, err.args[0], err.args[1])
683 def NormalizeName(cls, hostname):
684 """Validate and normalize the given hostname.
686 @attention: the validation is a bit more relaxed than the standards
687 require; most importantly, we allow underscores in names
688 @raise errors.OpPrereqError: when the name is not valid
691 hostname = hostname.lower()
692 if (not cls._VALID_NAME_RE.match(hostname) or
693 # double-dots, meaning empty label
695 # empty initial label
696 hostname.startswith(".")):
697 raise errors.OpPrereqError("Invalid hostname '%s'" % hostname,
699 if hostname.endswith("."):
700 hostname = hostname.rstrip(".")
704 def GetHostInfo(name=None):
705 """Lookup host name and raise an OpPrereqError for failures"""
708 return HostInfo(name)
709 except errors.ResolverError, err:
710 raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
711 (err[0], err[2]), errors.ECODE_RESOLVER)
714 def ListVolumeGroups():
715 """List volume groups and their size
719 Dictionary with keys volume name and values
720 the size of the volume
723 command = "vgs --noheadings --units m --nosuffix -o name,size"
724 result = RunCmd(command)
729 for line in result.stdout.splitlines():
731 name, size = line.split()
732 size = int(float(size))
733 except (IndexError, ValueError), err:
734 logging.error("Invalid output from vgs (%s): %s", err, line)
742 def BridgeExists(bridge):
743 """Check whether the given bridge exists in the system
746 @param bridge: the bridge name to check
748 @return: True if it does
751 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
754 def NiceSort(name_list):
755 """Sort a list of strings based on digit and non-digit groupings.
757 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
758 will sort the list in the logical order C{['a1', 'a2', 'a10',
761 The sort algorithm breaks each name in groups of either only-digits
762 or no-digits. Only the first eight such groups are considered, and
763 after that we just use what's left of the string.
765 @type name_list: list
766 @param name_list: the names to be sorted
768 @return: a copy of the name list sorted with our algorithm
771 _SORTER_BASE = "(\D+|\d+)"
772 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
773 _SORTER_BASE, _SORTER_BASE,
774 _SORTER_BASE, _SORTER_BASE,
775 _SORTER_BASE, _SORTER_BASE)
776 _SORTER_RE = re.compile(_SORTER_FULL)
777 _SORTER_NODIGIT = re.compile("^\D*$")
779 """Attempts to convert a variable to integer."""
780 if val is None or _SORTER_NODIGIT.match(val):
785 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
786 for name in name_list]
788 return [tup[1] for tup in to_sort]
791 def TryConvert(fn, val):
792 """Try to convert a value ignoring errors.
794 This function tries to apply function I{fn} to I{val}. If no
795 C{ValueError} or C{TypeError} exceptions are raised, it will return
796 the result, else it will return the original value. Any other
797 exceptions are propagated to the caller.
800 @param fn: function to apply to the value
801 @param val: the value to be converted
802 @return: The converted value if the conversion was successful,
803 otherwise the original value.
808 except (ValueError, TypeError):
814 """Verifies the syntax of an IPv4 address.
816 This function checks if the IPv4 address passes is valid or not based
817 on syntax (not IP range, class calculations, etc.).
820 @param ip: the address to be checked
821 @rtype: a regular expression match object
822 @return: a regular expression match object, or None if the
826 unit = "(0|[1-9]\d{0,2})"
827 #TODO: convert and return only boolean
828 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
831 def IsValidShellParam(word):
832 """Verifies is the given word is safe from the shell's p.o.v.
834 This means that we can pass this to a command via the shell and be
835 sure that it doesn't alter the command line and is passed as such to
838 Note that we are overly restrictive here, in order to be on the safe
842 @param word: the word to check
844 @return: True if the word is 'safe'
847 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
850 def BuildShellCmd(template, *args):
851 """Build a safe shell command line from the given arguments.
853 This function will check all arguments in the args list so that they
854 are valid shell parameters (i.e. they don't contain shell
855 metacharacters). If everything is ok, it will return the result of
859 @param template: the string holding the template for the
862 @return: the expanded command line
866 if not IsValidShellParam(word):
867 raise errors.ProgrammerError("Shell argument '%s' contains"
868 " invalid characters" % word)
869 return template % args
872 def FormatUnit(value, units):
873 """Formats an incoming number of MiB with the appropriate unit.
876 @param value: integer representing the value in MiB (1048576)
878 @param units: the type of formatting we should do:
879 - 'h' for automatic scaling
884 @return: the formatted value (with suffix)
887 if units not in ('m', 'g', 't', 'h'):
888 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
892 if units == 'm' or (units == 'h' and value < 1024):
895 return "%d%s" % (round(value, 0), suffix)
897 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
900 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
905 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
908 def ParseUnit(input_string):
909 """Tries to extract number and scale from the given string.
911 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
912 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
913 is always an int in MiB.
916 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
918 raise errors.UnitParseError("Invalid format")
920 value = float(m.groups()[0])
924 lcunit = unit.lower()
928 if lcunit in ('m', 'mb', 'mib'):
929 # Value already in MiB
932 elif lcunit in ('g', 'gb', 'gib'):
935 elif lcunit in ('t', 'tb', 'tib'):
939 raise errors.UnitParseError("Unknown unit: %s" % unit)
941 # Make sure we round up
942 if int(value) < value:
945 # Round up to the next multiple of 4
948 value += 4 - value % 4
953 def AddAuthorizedKey(file_name, key):
954 """Adds an SSH public key to an authorized_keys file.
957 @param file_name: path to authorized_keys file
959 @param key: string containing key
962 key_fields = key.split()
964 f = open(file_name, 'a+')
968 # Ignore whitespace changes
969 if line.split() == key_fields:
971 nl = line.endswith('\n')
975 f.write(key.rstrip('\r\n'))
982 def RemoveAuthorizedKey(file_name, key):
983 """Removes an SSH public key from an authorized_keys file.
986 @param file_name: path to authorized_keys file
988 @param key: string containing key
991 key_fields = key.split()
993 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
995 out = os.fdopen(fd, 'w')
997 f = open(file_name, 'r')
1000 # Ignore whitespace changes while comparing lines
1001 if line.split() != key_fields:
1005 os.rename(tmpname, file_name)
1015 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1016 """Sets the name of an IP address and hostname in /etc/hosts.
1018 @type file_name: str
1019 @param file_name: path to the file to modify (usually C{/etc/hosts})
1021 @param ip: the IP address
1023 @param hostname: the hostname to be added
1025 @param aliases: the list of aliases to add for the hostname
1028 # FIXME: use WriteFile + fn rather than duplicating its efforts
1029 # Ensure aliases are unique
1030 aliases = UniqueSequence([hostname] + aliases)[1:]
1032 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1034 out = os.fdopen(fd, 'w')
1036 f = open(file_name, 'r')
1039 fields = line.split()
1040 if fields and not fields[0].startswith('#') and ip == fields[0]:
1044 out.write("%s\t%s" % (ip, hostname))
1046 out.write(" %s" % ' '.join(aliases))
1051 os.chmod(tmpname, 0644)
1052 os.rename(tmpname, file_name)
1062 def AddHostToEtcHosts(hostname):
1063 """Wrapper around SetEtcHostsEntry.
1066 @param hostname: a hostname that will be resolved and added to
1067 L{constants.ETC_HOSTS}
1070 hi = HostInfo(name=hostname)
1071 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1074 def RemoveEtcHostsEntry(file_name, hostname):
1075 """Removes a hostname from /etc/hosts.
1077 IP addresses without names are removed from the file.
1079 @type file_name: str
1080 @param file_name: path to the file to modify (usually C{/etc/hosts})
1082 @param hostname: the hostname to be removed
1085 # FIXME: use WriteFile + fn rather than duplicating its efforts
1086 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1088 out = os.fdopen(fd, 'w')
1090 f = open(file_name, 'r')
1093 fields = line.split()
1094 if len(fields) > 1 and not fields[0].startswith('#'):
1096 if hostname in names:
1097 while hostname in names:
1098 names.remove(hostname)
1100 out.write("%s %s\n" % (fields[0], ' '.join(names)))
1107 os.chmod(tmpname, 0644)
1108 os.rename(tmpname, file_name)
1118 def RemoveHostFromEtcHosts(hostname):
1119 """Wrapper around RemoveEtcHostsEntry.
1122 @param hostname: hostname that will be resolved and its
1123 full and shot name will be removed from
1124 L{constants.ETC_HOSTS}
1127 hi = HostInfo(name=hostname)
1128 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1129 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1132 def TimestampForFilename():
1133 """Returns the current time formatted for filenames.
1135 The format doesn't contain colons as some shells and applications them as
1139 return time.strftime("%Y-%m-%d_%H_%M_%S")
1142 def CreateBackup(file_name):
1143 """Creates a backup of a file.
1145 @type file_name: str
1146 @param file_name: file to be backed up
1148 @return: the path to the newly created backup
1149 @raise errors.ProgrammerError: for invalid file names
1152 if not os.path.isfile(file_name):
1153 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1156 prefix = ("%s.backup-%s." %
1157 (os.path.basename(file_name), TimestampForFilename()))
1158 dir_name = os.path.dirname(file_name)
1160 fsrc = open(file_name, 'rb')
1162 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1163 fdst = os.fdopen(fd, 'wb')
1165 logging.debug("Backing up %s at %s", file_name, backup_name)
1166 shutil.copyfileobj(fsrc, fdst)
1175 def ShellQuote(value):
1176 """Quotes shell argument according to POSIX.
1179 @param value: the argument to be quoted
1181 @return: the quoted value
1184 if _re_shell_unquoted.match(value):
1187 return "'%s'" % value.replace("'", "'\\''")
1190 def ShellQuoteArgs(args):
1191 """Quotes a list of shell arguments.
1194 @param args: list of arguments to be quoted
1196 @return: the quoted arguments concatenated with spaces
1199 return ' '.join([ShellQuote(i) for i in args])
1202 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1203 """Simple ping implementation using TCP connect(2).
1205 Check if the given IP is reachable by doing attempting a TCP connect
1209 @param target: the IP or hostname to ping
1211 @param port: the port to connect to
1213 @param timeout: the timeout on the connection attempt
1214 @type live_port_needed: boolean
1215 @param live_port_needed: whether a closed port will cause the
1216 function to return failure, as if there was a timeout
1217 @type source: str or None
1218 @param source: if specified, will cause the connect to be made
1219 from this specific source address; failures to bind other
1220 than C{EADDRNOTAVAIL} will be ignored
1223 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1227 if source is not None:
1229 sock.bind((source, 0))
1230 except socket.error, (errcode, _):
1231 if errcode == errno.EADDRNOTAVAIL:
1234 sock.settimeout(timeout)
1237 sock.connect((target, port))
1240 except socket.timeout:
1242 except socket.error, (errcode, _):
1243 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1248 def OwnIpAddress(address):
1249 """Check if the current host has the the given IP address.
1251 Currently this is done by TCP-pinging the address from the loopback
1254 @type address: string
1255 @param address: the address to check
1257 @return: True if we own the address
1260 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1261 source=constants.LOCALHOST_IP_ADDRESS)
1264 def ListVisibleFiles(path):
1265 """Returns a list of visible files in a directory.
1268 @param path: the directory to enumerate
1270 @return: the list of all files not starting with a dot
1271 @raise ProgrammerError: if L{path} is not an absolue and normalized path
1274 if not IsNormAbsPath(path):
1275 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
1276 " absolute/normalized: '%s'" % path)
1277 files = [i for i in os.listdir(path) if not i.startswith(".")]
1282 def GetHomeDir(user, default=None):
1283 """Try to get the homedir of the given user.
1285 The user can be passed either as a string (denoting the name) or as
1286 an integer (denoting the user id). If the user is not found, the
1287 'default' argument is returned, which defaults to None.
1291 if isinstance(user, basestring):
1292 result = pwd.getpwnam(user)
1293 elif isinstance(user, (int, long)):
1294 result = pwd.getpwuid(user)
1296 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1300 return result.pw_dir
1304 """Returns a random UUID.
1306 @note: This is a Linux-specific method as it uses the /proc
1311 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1314 def GenerateSecret(numbytes=20):
1315 """Generates a random secret.
1317 This will generate a pseudo-random secret returning an hex string
1318 (so that it can be used where an ASCII string is needed).
1320 @param numbytes: the number of bytes which will be represented by the returned
1321 string (defaulting to 20, the length of a SHA1 hash)
1323 @return: an hex representation of the pseudo-random sequence
1326 return os.urandom(numbytes).encode('hex')
1329 def EnsureDirs(dirs):
1330 """Make required directories, if they don't exist.
1332 @param dirs: list of tuples (dir_name, dir_mode)
1333 @type dirs: list of (string, integer)
1336 for dir_name, dir_mode in dirs:
1338 os.mkdir(dir_name, dir_mode)
1339 except EnvironmentError, err:
1340 if err.errno != errno.EEXIST:
1341 raise errors.GenericError("Cannot create needed directory"
1342 " '%s': %s" % (dir_name, err))
1343 if not os.path.isdir(dir_name):
1344 raise errors.GenericError("%s is not a directory" % dir_name)
1347 def ReadFile(file_name, size=-1):
1351 @param size: Read at most size bytes (if negative, entire file)
1353 @return: the (possibly partial) content of the file
1356 f = open(file_name, "r")
1363 def WriteFile(file_name, fn=None, data=None,
1364 mode=None, uid=-1, gid=-1,
1365 atime=None, mtime=None, close=True,
1366 dry_run=False, backup=False,
1367 prewrite=None, postwrite=None):
1368 """(Over)write a file atomically.
1370 The file_name and either fn (a function taking one argument, the
1371 file descriptor, and which should write the data to it) or data (the
1372 contents of the file) must be passed. The other arguments are
1373 optional and allow setting the file mode, owner and group, and the
1374 mtime/atime of the file.
1376 If the function doesn't raise an exception, it has succeeded and the
1377 target file has the new contents. If the function has raised an
1378 exception, an existing target file should be unmodified and the
1379 temporary file should be removed.
1381 @type file_name: str
1382 @param file_name: the target filename
1384 @param fn: content writing function, called with
1385 file descriptor as parameter
1387 @param data: contents of the file
1389 @param mode: file mode
1391 @param uid: the owner of the file
1393 @param gid: the group of the file
1395 @param atime: a custom access time to be set on the file
1397 @param mtime: a custom modification time to be set on the file
1398 @type close: boolean
1399 @param close: whether to close file after writing it
1400 @type prewrite: callable
1401 @param prewrite: function to be called before writing content
1402 @type postwrite: callable
1403 @param postwrite: function to be called after writing content
1406 @return: None if the 'close' parameter evaluates to True,
1407 otherwise the file descriptor
1409 @raise errors.ProgrammerError: if any of the arguments are not valid
1412 if not os.path.isabs(file_name):
1413 raise errors.ProgrammerError("Path passed to WriteFile is not"
1414 " absolute: '%s'" % file_name)
1416 if [fn, data].count(None) != 1:
1417 raise errors.ProgrammerError("fn or data required")
1419 if [atime, mtime].count(None) == 1:
1420 raise errors.ProgrammerError("Both atime and mtime must be either"
1423 if backup and not dry_run and os.path.isfile(file_name):
1424 CreateBackup(file_name)
1426 dir_name, base_name = os.path.split(file_name)
1427 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1429 # here we need to make sure we remove the temp file, if any error
1430 # leaves it in place
1432 if uid != -1 or gid != -1:
1433 os.chown(new_name, uid, gid)
1435 os.chmod(new_name, mode)
1436 if callable(prewrite):
1438 if data is not None:
1442 if callable(postwrite):
1445 if atime is not None and mtime is not None:
1446 os.utime(new_name, (atime, mtime))
1448 os.rename(new_name, file_name)
1457 RemoveFile(new_name)
1462 def FirstFree(seq, base=0):
1463 """Returns the first non-existing integer from seq.
1465 The seq argument should be a sorted list of positive integers. The
1466 first time the index of an element is smaller than the element
1467 value, the index will be returned.
1469 The base argument is used to start at a different offset,
1470 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1472 Example: C{[0, 1, 3]} will return I{2}.
1475 @param seq: the sequence to be analyzed.
1477 @param base: use this value as the base index of the sequence
1479 @return: the first non-used index in the sequence
1482 for idx, elem in enumerate(seq):
1483 assert elem >= base, "Passed element is higher than base offset"
1484 if elem > idx + base:
1490 def all(seq, pred=bool): # pylint: disable-msg=W0622
1491 "Returns True if pred(x) is True for every element in the iterable"
1492 for _ in itertools.ifilterfalse(pred, seq):
1497 def any(seq, pred=bool): # pylint: disable-msg=W0622
1498 "Returns True if pred(x) is True for at least one element in the iterable"
1499 for _ in itertools.ifilter(pred, seq):
1504 def SingleWaitForFdCondition(fdobj, event, timeout):
1505 """Waits for a condition to occur on the socket.
1507 Immediately returns at the first interruption.
1509 @type fdobj: integer or object supporting a fileno() method
1510 @param fdobj: entity to wait for events on
1511 @type event: integer
1512 @param event: ORed condition (see select module)
1513 @type timeout: float or None
1514 @param timeout: Timeout in seconds
1516 @return: None for timeout, otherwise occured conditions
1519 check = (event | select.POLLPRI |
1520 select.POLLNVAL | select.POLLHUP | select.POLLERR)
1522 if timeout is not None:
1523 # Poller object expects milliseconds
1526 poller = select.poll()
1527 poller.register(fdobj, event)
1529 # TODO: If the main thread receives a signal and we have no timeout, we
1530 # could wait forever. This should check a global "quit" flag or something
1532 io_events = poller.poll(timeout)
1533 except select.error, err:
1534 if err[0] != errno.EINTR:
1537 if io_events and io_events[0][1] & check:
1538 return io_events[0][1]
1543 class FdConditionWaiterHelper(object):
1544 """Retry helper for WaitForFdCondition.
1546 This class contains the retried and wait functions that make sure
1547 WaitForFdCondition can continue waiting until the timeout is actually
1552 def __init__(self, timeout):
1553 self.timeout = timeout
1555 def Poll(self, fdobj, event):
1556 result = SingleWaitForFdCondition(fdobj, event, self.timeout)
1562 def UpdateTimeout(self, timeout):
1563 self.timeout = timeout
1566 def WaitForFdCondition(fdobj, event, timeout):
1567 """Waits for a condition to occur on the socket.
1569 Retries until the timeout is expired, even if interrupted.
1571 @type fdobj: integer or object supporting a fileno() method
1572 @param fdobj: entity to wait for events on
1573 @type event: integer
1574 @param event: ORed condition (see select module)
1575 @type timeout: float or None
1576 @param timeout: Timeout in seconds
1578 @return: None for timeout, otherwise occured conditions
1581 if timeout is not None:
1582 retrywaiter = FdConditionWaiterHelper(timeout)
1584 result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
1585 args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
1586 except RetryTimeout:
1590 while result is None:
1591 result = SingleWaitForFdCondition(fdobj, event, timeout)
1595 def partition(seq, pred=bool): # # pylint: disable-msg=W0622
1596 "Partition a list in two, based on the given predicate"
1597 return (list(itertools.ifilter(pred, seq)),
1598 list(itertools.ifilterfalse(pred, seq)))
1601 def UniqueSequence(seq):
1602 """Returns a list with unique elements.
1604 Element order is preserved.
1607 @param seq: the sequence with the source elements
1609 @return: list of unique elements from seq
1613 return [i for i in seq if i not in seen and not seen.add(i)]
1616 def NormalizeAndValidateMac(mac):
1617 """Normalizes and check if a MAC address is valid.
1619 Checks whether the supplied MAC address is formally correct, only
1620 accepts colon separated format. Normalize it to all lower.
1623 @param mac: the MAC to be validated
1625 @return: returns the normalized and validated MAC.
1627 @raise errors.OpPrereqError: If the MAC isn't valid
1630 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1631 if not mac_check.match(mac):
1632 raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1633 mac, errors.ECODE_INVAL)
1638 def TestDelay(duration):
1639 """Sleep for a fixed amount of time.
1641 @type duration: float
1642 @param duration: the sleep duration
1644 @return: False for negative value, True otherwise
1648 return False, "Invalid sleep duration"
1649 time.sleep(duration)
1653 def _CloseFDNoErr(fd, retries=5):
1654 """Close a file descriptor ignoring errors.
1657 @param fd: the file descriptor
1659 @param retries: how many retries to make, in case we get any
1660 other error than EBADF
1665 except OSError, err:
1666 if err.errno != errno.EBADF:
1668 _CloseFDNoErr(fd, retries - 1)
1669 # else either it's closed already or we're out of retries, so we
1670 # ignore this and go on
1673 def CloseFDs(noclose_fds=None):
1674 """Close file descriptors.
1676 This closes all file descriptors above 2 (i.e. except
1679 @type noclose_fds: list or None
1680 @param noclose_fds: if given, it denotes a list of file descriptor
1681 that should not be closed
1684 # Default maximum for the number of available file descriptors.
1685 if 'SC_OPEN_MAX' in os.sysconf_names:
1687 MAXFD = os.sysconf('SC_OPEN_MAX')
1694 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1695 if (maxfd == resource.RLIM_INFINITY):
1698 # Iterate through and close all file descriptors (except the standard ones)
1699 for fd in range(3, maxfd):
1700 if noclose_fds and fd in noclose_fds:
1705 def Daemonize(logfile):
1706 """Daemonize the current process.
1708 This detaches the current process from the controlling terminal and
1709 runs it in the background as a daemon.
1712 @param logfile: the logfile to which we should redirect stdout/stderr
1714 @return: the value zero
1717 # pylint: disable-msg=W0212
1718 # yes, we really want os._exit
1724 if (pid == 0): # The first child.
1727 pid = os.fork() # Fork a second child.
1728 if (pid == 0): # The second child.
1732 # exit() or _exit()? See below.
1733 os._exit(0) # Exit parent (the first child) of the second child.
1735 os._exit(0) # Exit parent of the first child.
1739 i = os.open("/dev/null", os.O_RDONLY) # stdin
1740 assert i == 0, "Can't close/reopen stdin"
1741 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1742 assert i == 1, "Can't close/reopen stdout"
1743 # Duplicate standard output to standard error.
1748 def DaemonPidFileName(name):
1749 """Compute a ganeti pid file absolute path
1752 @param name: the daemon name
1754 @return: the full path to the pidfile corresponding to the given
1758 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
1761 def EnsureDaemon(name):
1762 """Check for and start daemon if not alive.
1765 result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
1767 logging.error("Can't start daemon '%s', failure %s, output: %s",
1768 name, result.fail_reason, result.output)
1774 def WritePidFile(name):
1775 """Write the current process pidfile.
1777 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1780 @param name: the daemon name to use
1781 @raise errors.GenericError: if the pid file already exists and
1782 points to a live process
1786 pidfilename = DaemonPidFileName(name)
1787 if IsProcessAlive(ReadPidFile(pidfilename)):
1788 raise errors.GenericError("%s contains a live process" % pidfilename)
1790 WriteFile(pidfilename, data="%d\n" % pid)
1793 def RemovePidFile(name):
1794 """Remove the current process pidfile.
1796 Any errors are ignored.
1799 @param name: the daemon name used to derive the pidfile name
1802 pidfilename = DaemonPidFileName(name)
1803 # TODO: we could check here that the file contains our pid
1805 RemoveFile(pidfilename)
1806 except: # pylint: disable-msg=W0702
1810 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1812 """Kill a process given by its pid.
1815 @param pid: The PID to terminate.
1817 @param signal_: The signal to send, by default SIGTERM
1819 @param timeout: The timeout after which, if the process is still alive,
1820 a SIGKILL will be sent. If not positive, no such checking
1822 @type waitpid: boolean
1823 @param waitpid: If true, we should waitpid on this process after
1824 sending signals, since it's our own child and otherwise it
1825 would remain as zombie
1828 def _helper(pid, signal_, wait):
1829 """Simple helper to encapsulate the kill/waitpid sequence"""
1830 os.kill(pid, signal_)
1833 os.waitpid(pid, os.WNOHANG)
1838 # kill with pid=0 == suicide
1839 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1841 if not IsProcessAlive(pid):
1844 _helper(pid, signal_, waitpid)
1849 def _CheckProcess():
1850 if not IsProcessAlive(pid):
1854 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1864 # Wait up to $timeout seconds
1865 Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1866 except RetryTimeout:
1869 if IsProcessAlive(pid):
1870 # Kill process if it's still alive
1871 _helper(pid, signal.SIGKILL, waitpid)
1874 def FindFile(name, search_path, test=os.path.exists):
1875 """Look for a filesystem object in a given path.
1877 This is an abstract method to search for filesystem object (files,
1878 dirs) under a given search path.
1881 @param name: the name to look for
1882 @type search_path: str
1883 @param search_path: location to start at
1884 @type test: callable
1885 @param test: a function taking one argument that should return True
1886 if the a given object is valid; the default value is
1887 os.path.exists, causing only existing files to be returned
1889 @return: full path to the object if found, None otherwise
1892 # validate the filename mask
1893 if constants.EXT_PLUGIN_MASK.match(name) is None:
1894 logging.critical("Invalid value passed for external script name: '%s'",
1898 for dir_name in search_path:
1899 # FIXME: investigate switch to PathJoin
1900 item_name = os.path.sep.join([dir_name, name])
1901 # check the user test and that we're indeed resolving to the given
1903 if test(item_name) and os.path.basename(item_name) == name:
1908 def CheckVolumeGroupSize(vglist, vgname, minsize):
1909 """Checks if the volume group list is valid.
1911 The function will check if a given volume group is in the list of
1912 volume groups and has a minimum size.
1915 @param vglist: dictionary of volume group names and their size
1917 @param vgname: the volume group we should check
1919 @param minsize: the minimum size we accept
1921 @return: None for success, otherwise the error message
1924 vgsize = vglist.get(vgname, None)
1926 return "volume group '%s' missing" % vgname
1927 elif vgsize < minsize:
1928 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1929 (vgname, minsize, vgsize))
1933 def SplitTime(value):
1934 """Splits time as floating point number into a tuple.
1936 @param value: Time in seconds
1937 @type value: int or float
1938 @return: Tuple containing (seconds, microseconds)
1941 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1943 assert 0 <= seconds, \
1944 "Seconds must be larger than or equal to 0, but are %s" % seconds
1945 assert 0 <= microseconds <= 999999, \
1946 "Microseconds must be 0-999999, but are %s" % microseconds
1948 return (int(seconds), int(microseconds))
1951 def MergeTime(timetuple):
1952 """Merges a tuple into time as a floating point number.
1954 @param timetuple: Time as tuple, (seconds, microseconds)
1955 @type timetuple: tuple
1956 @return: Time as a floating point number expressed in seconds
1959 (seconds, microseconds) = timetuple
1961 assert 0 <= seconds, \
1962 "Seconds must be larger than or equal to 0, but are %s" % seconds
1963 assert 0 <= microseconds <= 999999, \
1964 "Microseconds must be 0-999999, but are %s" % microseconds
1966 return float(seconds) + (float(microseconds) * 0.000001)
1969 def GetDaemonPort(daemon_name):
1970 """Get the daemon port for this cluster.
1972 Note that this routine does not read a ganeti-specific file, but
1973 instead uses C{socket.getservbyname} to allow pre-customization of
1974 this parameter outside of Ganeti.
1976 @type daemon_name: string
1977 @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1981 if daemon_name not in constants.DAEMONS_PORTS:
1982 raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1984 (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1986 port = socket.getservbyname(daemon_name, proto)
1987 except socket.error:
1993 def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
1994 multithreaded=False, syslog=constants.SYSLOG_USAGE):
1995 """Configures the logging module.
1998 @param logfile: the filename to which we should log
1999 @type debug: integer
2000 @param debug: if greater than zero, enable debug messages, otherwise
2001 only those at C{INFO} and above level
2002 @type stderr_logging: boolean
2003 @param stderr_logging: whether we should also log to the standard error
2005 @param program: the name under which we should log messages
2006 @type multithreaded: boolean
2007 @param multithreaded: if True, will add the thread name to the log file
2008 @type syslog: string
2009 @param syslog: one of 'no', 'yes', 'only':
2010 - if no, syslog is not used
2011 - if yes, syslog is used (in addition to file-logging)
2012 - if only, only syslog is used
2013 @raise EnvironmentError: if we can't open the log file and
2014 syslog/stderr logging is disabled
2017 fmt = "%(asctime)s: " + program + " pid=%(process)d"
2018 sft = program + "[%(process)d]:"
2020 fmt += "/%(threadName)s"
2021 sft += " (%(threadName)s)"
2023 fmt += " %(module)s:%(lineno)s"
2024 # no debug info for syslog loggers
2025 fmt += " %(levelname)s %(message)s"
2026 # yes, we do want the textual level, as remote syslog will probably
2027 # lose the error level, and it's easier to grep for it
2028 sft += " %(levelname)s %(message)s"
2029 formatter = logging.Formatter(fmt)
2030 sys_fmt = logging.Formatter(sft)
2032 root_logger = logging.getLogger("")
2033 root_logger.setLevel(logging.NOTSET)
2035 # Remove all previously setup handlers
2036 for handler in root_logger.handlers:
2038 root_logger.removeHandler(handler)
2041 stderr_handler = logging.StreamHandler()
2042 stderr_handler.setFormatter(formatter)
2044 stderr_handler.setLevel(logging.NOTSET)
2046 stderr_handler.setLevel(logging.CRITICAL)
2047 root_logger.addHandler(stderr_handler)
2049 if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
2050 facility = logging.handlers.SysLogHandler.LOG_DAEMON
2051 syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
2053 syslog_handler.setFormatter(sys_fmt)
2054 # Never enable debug over syslog
2055 syslog_handler.setLevel(logging.INFO)
2056 root_logger.addHandler(syslog_handler)
2058 if syslog != constants.SYSLOG_ONLY:
2059 # this can fail, if the logging directories are not setup or we have
2060 # a permisssion problem; in this case, it's best to log but ignore
2061 # the error if stderr_logging is True, and if false we re-raise the
2062 # exception since otherwise we could run but without any logs at all
2064 logfile_handler = logging.FileHandler(logfile)
2065 logfile_handler.setFormatter(formatter)
2067 logfile_handler.setLevel(logging.DEBUG)
2069 logfile_handler.setLevel(logging.INFO)
2070 root_logger.addHandler(logfile_handler)
2071 except EnvironmentError:
2072 if stderr_logging or syslog == constants.SYSLOG_YES:
2073 logging.exception("Failed to enable logging to file '%s'", logfile)
2075 # we need to re-raise the exception
2079 def IsNormAbsPath(path):
2080 """Check whether a path is absolute and also normalized
2082 This avoids things like /dir/../../other/path to be valid.
2085 return os.path.normpath(path) == path and os.path.isabs(path)
2088 def PathJoin(*args):
2089 """Safe-join a list of path components.
2092 - the first argument must be an absolute path
2093 - no component in the path must have backtracking (e.g. /../),
2094 since we check for normalization at the end
2096 @param args: the path components to be joined
2097 @raise ValueError: for invalid paths
2100 # ensure we're having at least one path passed in
2102 # ensure the first component is an absolute and normalized path name
2104 if not IsNormAbsPath(root):
2105 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
2106 result = os.path.join(*args)
2107 # ensure that the whole path is normalized
2108 if not IsNormAbsPath(result):
2109 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
2110 # check that we're still under the original prefix
2111 prefix = os.path.commonprefix([root, result])
2113 raise ValueError("Error: path joining resulted in different prefix"
2114 " (%s != %s)" % (prefix, root))
2118 def TailFile(fname, lines=20):
2119 """Return the last lines from a file.
2121 @note: this function will only read and parse the last 4KB of
2122 the file; if the lines are very long, it could be that less
2123 than the requested number of lines are returned
2125 @param fname: the file name
2127 @param lines: the (maximum) number of lines to return
2130 fd = open(fname, "r")
2134 pos = max(0, pos-4096)
2136 raw_data = fd.read()
2140 rows = raw_data.splitlines()
2141 return rows[-lines:]
2144 def _ParseAsn1Generalizedtime(value):
2145 """Parses an ASN1 GENERALIZEDTIME timestamp as used by pyOpenSSL.
2148 @param value: ASN1 GENERALIZEDTIME timestamp
2151 m = re.match(r"^(\d+)([-+]\d\d)(\d\d)$", value)
2154 asn1time = m.group(1)
2155 hours = int(m.group(2))
2156 minutes = int(m.group(3))
2157 utcoffset = (60 * hours) + minutes
2159 if not value.endswith("Z"):
2160 raise ValueError("Missing timezone")
2161 asn1time = value[:-1]
2164 parsed = time.strptime(asn1time, "%Y%m%d%H%M%S")
2166 tt = datetime.datetime(*(parsed[:7])) - datetime.timedelta(minutes=utcoffset)
2168 return calendar.timegm(tt.utctimetuple())
2171 def GetX509CertValidity(cert):
2172 """Returns the validity period of the certificate.
2174 @type cert: OpenSSL.crypto.X509
2175 @param cert: X509 certificate object
2178 # The get_notBefore and get_notAfter functions are only supported in
2179 # pyOpenSSL 0.7 and above.
2181 get_notbefore_fn = cert.get_notBefore
2182 except AttributeError:
2185 not_before_asn1 = get_notbefore_fn()
2187 if not_before_asn1 is None:
2190 not_before = _ParseAsn1Generalizedtime(not_before_asn1)
2193 get_notafter_fn = cert.get_notAfter
2194 except AttributeError:
2197 not_after_asn1 = get_notafter_fn()
2199 if not_after_asn1 is None:
2202 not_after = _ParseAsn1Generalizedtime(not_after_asn1)
2204 return (not_before, not_after)
2207 def SafeEncode(text):
2208 """Return a 'safe' version of a source string.
2210 This function mangles the input string and returns a version that
2211 should be safe to display/encode as ASCII. To this end, we first
2212 convert it to ASCII using the 'backslashreplace' encoding which
2213 should get rid of any non-ASCII chars, and then we process it
2214 through a loop copied from the string repr sources in the python; we
2215 don't use string_escape anymore since that escape single quotes and
2216 backslashes too, and that is too much; and that escaping is not
2217 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2219 @type text: str or unicode
2220 @param text: input data
2222 @return: a safe version of text
2225 if isinstance(text, unicode):
2226 # only if unicode; if str already, we handle it below
2227 text = text.encode('ascii', 'backslashreplace')
2237 elif c < 32 or c >= 127: # non-printable
2238 resu += "\\x%02x" % (c & 0xff)
2244 def UnescapeAndSplit(text, sep=","):
2245 """Split and unescape a string based on a given separator.
2247 This function splits a string based on a separator where the
2248 separator itself can be escape in order to be an element of the
2249 elements. The escaping rules are (assuming coma being the
2251 - a plain , separates the elements
2252 - a sequence \\\\, (double backslash plus comma) is handled as a
2253 backslash plus a separator comma
2254 - a sequence \, (backslash plus comma) is handled as a
2258 @param text: the string to split
2260 @param text: the separator
2262 @return: a list of strings
2265 # we split the list by sep (with no escaping at this stage)
2266 slist = text.split(sep)
2267 # next, we revisit the elements and if any of them ended with an odd
2268 # number of backslashes, then we join it with the next
2272 if e1.endswith("\\"):
2273 num_b = len(e1) - len(e1.rstrip("\\"))
2276 # here the backslashes remain (all), and will be reduced in
2278 rlist.append(e1 + sep + e2)
2281 # finally, replace backslash-something with something
2282 rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2286 def CommaJoin(names):
2287 """Nicely join a set of identifiers.
2289 @param names: set, list or tuple
2290 @return: a string with the formatted results
2293 return ", ".join([str(val) for val in names])
2296 def BytesToMebibyte(value):
2297 """Converts bytes to mebibytes.
2300 @param value: Value in bytes
2302 @return: Value in mebibytes
2305 return int(round(value / (1024.0 * 1024.0), 0))
2308 def CalculateDirectorySize(path):
2309 """Calculates the size of a directory recursively.
2312 @param path: Path to directory
2314 @return: Size in mebibytes
2319 for (curpath, _, files) in os.walk(path):
2320 for filename in files:
2321 st = os.lstat(PathJoin(curpath, filename))
2324 return BytesToMebibyte(size)
2327 def GetFilesystemStats(path):
2328 """Returns the total and free space on a filesystem.
2331 @param path: Path on filesystem to be examined
2333 @return: tuple of (Total space, Free space) in mebibytes
2336 st = os.statvfs(path)
2338 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2339 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2340 return (tsize, fsize)
2343 def RunInSeparateProcess(fn, *args):
2344 """Runs a function in a separate process.
2346 Note: Only boolean return values are supported.
2349 @param fn: Function to be called
2351 @return: Function's result
2358 # In case the function uses temporary files
2359 ResetTempfileModule()
2362 result = int(bool(fn(*args)))
2363 assert result in (0, 1)
2364 except: # pylint: disable-msg=W0702
2365 logging.exception("Error while calling function in separate process")
2366 # 0 and 1 are reserved for the return value
2369 os._exit(result) # pylint: disable-msg=W0212
2373 # Avoid zombies and check exit code
2374 (_, status) = os.waitpid(pid, 0)
2376 if os.WIFSIGNALED(status):
2378 signum = os.WTERMSIG(status)
2380 exitcode = os.WEXITSTATUS(status)
2383 if not (exitcode in (0, 1) and signum is None):
2384 raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2387 return bool(exitcode)
2390 def LockedMethod(fn):
2391 """Synchronized object access decorator.
2393 This decorator is intended to protect access to an object using the
2394 object's own lock which is hardcoded to '_lock'.
2397 def _LockDebug(*args, **kwargs):
2399 logging.debug(*args, **kwargs)
2401 def wrapper(self, *args, **kwargs):
2402 # pylint: disable-msg=W0212
2403 assert hasattr(self, '_lock')
2405 _LockDebug("Waiting for %s", lock)
2408 _LockDebug("Acquired %s", lock)
2409 result = fn(self, *args, **kwargs)
2411 _LockDebug("Releasing %s", lock)
2413 _LockDebug("Released %s", lock)
2419 """Locks a file using POSIX locks.
2422 @param fd: the file descriptor we need to lock
2426 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2427 except IOError, err:
2428 if err.errno == errno.EAGAIN:
2429 raise errors.LockError("File already locked")
2433 def FormatTime(val):
2434 """Formats a time value.
2436 @type val: float or None
2437 @param val: the timestamp as returned by time.time()
2438 @return: a string value or N/A if we don't have a valid timestamp
2441 if val is None or not isinstance(val, (int, float)):
2443 # these two codes works on Linux, but they are not guaranteed on all
2445 return time.strftime("%F %T", time.localtime(val))
2448 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2449 """Reads the watcher pause file.
2451 @type filename: string
2452 @param filename: Path to watcher pause file
2453 @type now: None, float or int
2454 @param now: Current time as Unix timestamp
2455 @type remove_after: int
2456 @param remove_after: Remove watcher pause file after specified amount of
2457 seconds past the pause end time
2464 value = ReadFile(filename)
2465 except IOError, err:
2466 if err.errno != errno.ENOENT:
2470 if value is not None:
2474 logging.warning(("Watcher pause file (%s) contains invalid value,"
2475 " removing it"), filename)
2476 RemoveFile(filename)
2479 if value is not None:
2480 # Remove file if it's outdated
2481 if now > (value + remove_after):
2482 RemoveFile(filename)
2491 class RetryTimeout(Exception):
2492 """Retry loop timed out.
2497 class RetryAgain(Exception):
2503 class _RetryDelayCalculator(object):
2504 """Calculator for increasing delays.
2514 def __init__(self, start, factor, limit):
2515 """Initializes this class.
2518 @param start: Initial delay
2520 @param factor: Factor for delay increase
2521 @type limit: float or None
2522 @param limit: Upper limit for delay or None for no limit
2526 assert factor >= 1.0
2527 assert limit is None or limit >= 0.0
2530 self._factor = factor
2536 """Returns current delay and calculates the next one.
2539 current = self._next
2541 # Update for next run
2542 if self._limit is None or self._next < self._limit:
2543 self._next = min(self._limit, self._next * self._factor)
2548 #: Special delay to specify whole remaining timeout
2549 RETRY_REMAINING_TIME = object()
2552 def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2553 _time_fn=time.time):
2554 """Call a function repeatedly until it succeeds.
2556 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2557 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2558 total of C{timeout} seconds, this function throws L{RetryTimeout}.
2560 C{delay} can be one of the following:
2561 - callable returning the delay length as a float
2562 - Tuple of (start, factor, limit)
2563 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2564 useful when overriding L{wait_fn} to wait for an external event)
2565 - A static delay as a number (int or float)
2568 @param fn: Function to be called
2569 @param delay: Either a callable (returning the delay), a tuple of (start,
2570 factor, limit) (see L{_RetryDelayCalculator}),
2571 L{RETRY_REMAINING_TIME} or a number (int or float)
2572 @type timeout: float
2573 @param timeout: Total timeout
2574 @type wait_fn: callable
2575 @param wait_fn: Waiting function
2576 @return: Return value of function
2580 assert callable(wait_fn)
2581 assert callable(_time_fn)
2586 end_time = _time_fn() + timeout
2589 # External function to calculate delay
2592 elif isinstance(delay, (tuple, list)):
2593 # Increasing delay with optional upper boundary
2594 (start, factor, limit) = delay
2595 calc_delay = _RetryDelayCalculator(start, factor, limit)
2597 elif delay is RETRY_REMAINING_TIME:
2598 # Always use the remaining time
2603 calc_delay = lambda: delay
2605 assert calc_delay is None or callable(calc_delay)
2609 # pylint: disable-msg=W0142
2613 except RetryTimeout:
2614 raise errors.ProgrammerError("Nested retry loop detected that didn't"
2615 " handle RetryTimeout")
2617 remaining_time = end_time - _time_fn()
2619 if remaining_time < 0.0:
2620 raise RetryTimeout()
2622 assert remaining_time >= 0.0
2624 if calc_delay is None:
2625 wait_fn(remaining_time)
2627 current_delay = calc_delay()
2628 if current_delay > 0.0:
2629 wait_fn(current_delay)
2632 class FileLock(object):
2633 """Utility class for file locks.
2636 def __init__(self, fd, filename):
2637 """Constructor for FileLock.
2640 @param fd: File object
2642 @param filename: Path of the file opened at I{fd}
2646 self.filename = filename
2649 def Open(cls, filename):
2650 """Creates and opens a file to be used as a file-based lock.
2652 @type filename: string
2653 @param filename: path to the file to be locked
2656 # Using "os.open" is necessary to allow both opening existing file
2657 # read/write and creating if not existing. Vanilla "open" will truncate an
2658 # existing file -or- allow creating if not existing.
2659 return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
2666 """Close the file and release the lock.
2669 if hasattr(self, "fd") and self.fd:
2673 def _flock(self, flag, blocking, timeout, errmsg):
2674 """Wrapper for fcntl.flock.
2677 @param flag: operation flag
2678 @type blocking: bool
2679 @param blocking: whether the operation should be done in blocking mode.
2680 @type timeout: None or float
2681 @param timeout: for how long the operation should be retried (implies
2683 @type errmsg: string
2684 @param errmsg: error message in case operation fails.
2687 assert self.fd, "Lock was closed"
2688 assert timeout is None or timeout >= 0, \
2689 "If specified, timeout must be positive"
2690 assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
2692 # When a timeout is used, LOCK_NB must always be set
2693 if not (timeout is None and blocking):
2694 flag |= fcntl.LOCK_NB
2697 self._Lock(self.fd, flag, timeout)
2700 Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
2701 args=(self.fd, flag, timeout))
2702 except RetryTimeout:
2703 raise errors.LockError(errmsg)
2706 def _Lock(fd, flag, timeout):
2708 fcntl.flock(fd, flag)
2709 except IOError, err:
2710 if timeout is not None and err.errno == errno.EAGAIN:
2713 logging.exception("fcntl.flock failed")
2716 def Exclusive(self, blocking=False, timeout=None):
2717 """Locks the file in exclusive mode.
2719 @type blocking: boolean
2720 @param blocking: whether to block and wait until we
2721 can lock the file or return immediately
2722 @type timeout: int or None
2723 @param timeout: if not None, the duration to wait for the lock
2727 self._flock(fcntl.LOCK_EX, blocking, timeout,
2728 "Failed to lock %s in exclusive mode" % self.filename)
2730 def Shared(self, blocking=False, timeout=None):
2731 """Locks the file in shared mode.
2733 @type blocking: boolean
2734 @param blocking: whether to block and wait until we
2735 can lock the file or return immediately
2736 @type timeout: int or None
2737 @param timeout: if not None, the duration to wait for the lock
2741 self._flock(fcntl.LOCK_SH, blocking, timeout,
2742 "Failed to lock %s in shared mode" % self.filename)
2744 def Unlock(self, blocking=True, timeout=None):
2745 """Unlocks the file.
2747 According to C{flock(2)}, unlocking can also be a nonblocking
2750 To make a non-blocking request, include LOCK_NB with any of the above
2753 @type blocking: boolean
2754 @param blocking: whether to block and wait until we
2755 can lock the file or return immediately
2756 @type timeout: int or None
2757 @param timeout: if not None, the duration to wait for the lock
2761 self._flock(fcntl.LOCK_UN, blocking, timeout,
2762 "Failed to unlock %s" % self.filename)
2766 """Splits data chunks into lines separated by newline.
2768 Instances provide a file-like interface.
2771 def __init__(self, line_fn, *args):
2772 """Initializes this class.
2774 @type line_fn: callable
2775 @param line_fn: Function called for each line, first parameter is line
2776 @param args: Extra arguments for L{line_fn}
2779 assert callable(line_fn)
2782 # Python 2.4 doesn't have functools.partial yet
2784 lambda line: line_fn(line, *args) # pylint: disable-msg=W0142
2786 self._line_fn = line_fn
2788 self._lines = collections.deque()
2791 def write(self, data):
2792 parts = (self._buffer + data).split("\n")
2793 self._buffer = parts.pop()
2794 self._lines.extend(parts)
2798 self._line_fn(self._lines.popleft().rstrip("\r\n"))
2803 self._line_fn(self._buffer)
2806 def SignalHandled(signums):
2807 """Signal Handled decoration.
2809 This special decorator installs a signal handler and then calls the target
2810 function. The function must accept a 'signal_handlers' keyword argument,
2811 which will contain a dict indexed by signal number, with SignalHandler
2814 The decorator can be safely stacked with iself, to handle multiple signals
2815 with different handlers.
2818 @param signums: signals to intercept
2822 def sig_function(*args, **kwargs):
2823 assert 'signal_handlers' not in kwargs or \
2824 kwargs['signal_handlers'] is None or \
2825 isinstance(kwargs['signal_handlers'], dict), \
2826 "Wrong signal_handlers parameter in original function call"
2827 if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2828 signal_handlers = kwargs['signal_handlers']
2830 signal_handlers = {}
2831 kwargs['signal_handlers'] = signal_handlers
2832 sighandler = SignalHandler(signums)
2835 signal_handlers[sig] = sighandler
2836 return fn(*args, **kwargs)
2843 class SignalHandler(object):
2844 """Generic signal handler class.
2846 It automatically restores the original handler when deconstructed or
2847 when L{Reset} is called. You can either pass your own handler
2848 function in or query the L{called} attribute to detect whether the
2852 @ivar signum: the signals we handle
2853 @type called: boolean
2854 @ivar called: tracks whether any of the signals have been raised
2857 def __init__(self, signum):
2858 """Constructs a new SignalHandler instance.
2860 @type signum: int or list of ints
2861 @param signum: Single signal number or set of signal numbers
2864 self.signum = set(signum)
2869 for signum in self.signum:
2871 prev_handler = signal.signal(signum, self._HandleSignal)
2873 self._previous[signum] = prev_handler
2875 # Restore previous handler
2876 signal.signal(signum, prev_handler)
2879 # Reset all handlers
2881 # Here we have a race condition: a handler may have already been called,
2882 # but there's not much we can do about it at this point.
2889 """Restore previous handler.
2891 This will reset all the signals to their previous handlers.
2894 for signum, prev_handler in self._previous.items():
2895 signal.signal(signum, prev_handler)
2896 # If successful, remove from dict
2897 del self._previous[signum]
2900 """Unsets the L{called} flag.
2902 This function can be used in case a signal may arrive several times.
2907 # we don't care about arguments, but we leave them named for the future
2908 def _HandleSignal(self, signum, frame): # pylint: disable-msg=W0613
2909 """Actual signal handling function.
2912 # This is not nice and not absolutely atomic, but it appears to be the only
2913 # solution in Python -- there are no atomic types.
2917 class FieldSet(object):
2918 """A simple field set.
2920 Among the features are:
2921 - checking if a string is among a list of static string or regex objects
2922 - checking if a whole list of string matches
2923 - returning the matching groups from a regex match
2925 Internally, all fields are held as regular expression objects.
2928 def __init__(self, *items):
2929 self.items = [re.compile("^%s$" % value) for value in items]
2931 def Extend(self, other_set):
2932 """Extend the field set with the items from another one"""
2933 self.items.extend(other_set.items)
2935 def Matches(self, field):
2936 """Checks if a field matches the current set
2939 @param field: the string to match
2940 @return: either None or a regular expression match object
2943 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2947 def NonMatching(self, items):
2948 """Returns the list of fields not matching the current set
2951 @param items: the list of fields to check
2953 @return: list of non-matching fields
2956 return [val for val in items if not self.Matches(val)]