4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
44 import logging.handlers
47 from cStringIO import StringIO
50 from hashlib import sha1
55 from ganeti import errors
56 from ganeti import constants
60 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
64 #: when set to True, L{RunCmd} is disabled
67 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
70 class RunResult(object):
71 """Holds the result of running external programs.
74 @ivar exit_code: the exit code of the program, or None (if the program
76 @type signal: int or None
77 @ivar signal: the signal that caused the program to finish, or None
78 (if the program wasn't terminated by a signal)
80 @ivar stdout: the standard output of the program
82 @ivar stderr: the standard error of the program
84 @ivar failed: True in case the program was
85 terminated by a signal or exited with a non-zero exit code
86 @ivar fail_reason: a string detailing the termination reason
89 __slots__ = ["exit_code", "signal", "stdout", "stderr",
90 "failed", "fail_reason", "cmd"]
93 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
95 self.exit_code = exit_code
99 self.failed = (signal_ is not None or exit_code != 0)
101 if self.signal is not None:
102 self.fail_reason = "terminated by signal %s" % self.signal
103 elif self.exit_code is not None:
104 self.fail_reason = "exited with exit code %s" % self.exit_code
106 self.fail_reason = "unable to determine termination reason"
109 logging.debug("Command '%s' failed (%s); output: %s",
110 self.cmd, self.fail_reason, self.output)
112 def _GetOutput(self):
113 """Returns the combined stdout and stderr for easier usage.
116 return self.stdout + self.stderr
118 output = property(_GetOutput, None, None, "Return full output")
121 def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
122 """Execute a (shell) command.
124 The command should not read from its standard input, as it will be
127 @type cmd: string or list
128 @param cmd: Command to run
130 @param env: Additional environment
132 @param output: if desired, the output of the command can be
133 saved in a file instead of the RunResult instance; this
134 parameter denotes the file name (if not None)
136 @param cwd: if specified, will be used as the working
137 directory for the command; the default will be /
138 @type reset_env: boolean
139 @param reset_env: whether to reset or keep the default os environment
141 @return: RunResult instance
142 @raise errors.ProgrammerError: if we call this when forks are disabled
146 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
148 if isinstance(cmd, list):
149 cmd = [str(val) for val in cmd]
150 strcmd = " ".join(cmd)
155 logging.debug("RunCmd '%s'", strcmd)
158 cmd_env = os.environ.copy()
159 cmd_env["LC_ALL"] = "C"
168 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
170 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
173 if err.errno == errno.ENOENT:
174 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
186 return RunResult(exitcode, signal_, out, err, strcmd)
189 def _RunCmdPipe(cmd, env, via_shell, cwd):
190 """Run a command and return its output.
192 @type cmd: string or list
193 @param cmd: Command to run
195 @param env: The environment to use
196 @type via_shell: bool
197 @param via_shell: if we should run via the shell
199 @param cwd: the working directory for the program
201 @return: (out, err, status)
204 poller = select.poll()
205 child = subprocess.Popen(cmd, shell=via_shell,
206 stderr=subprocess.PIPE,
207 stdout=subprocess.PIPE,
208 stdin=subprocess.PIPE,
209 close_fds=True, env=env,
213 poller.register(child.stdout, select.POLLIN)
214 poller.register(child.stderr, select.POLLIN)
218 child.stdout.fileno(): (out, child.stdout),
219 child.stderr.fileno(): (err, child.stderr),
222 status = fcntl.fcntl(fd, fcntl.F_GETFL)
223 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
227 pollresult = poller.poll()
228 except EnvironmentError, eerr:
229 if eerr.errno == errno.EINTR:
232 except select.error, serr:
233 if serr[0] == errno.EINTR:
237 for fd, event in pollresult:
238 if event & select.POLLIN or event & select.POLLPRI:
239 data = fdmap[fd][1].read()
240 # no data from read signifies EOF (the same as POLLHUP)
242 poller.unregister(fd)
245 fdmap[fd][0].write(data)
246 if (event & select.POLLNVAL or event & select.POLLHUP or
247 event & select.POLLERR):
248 poller.unregister(fd)
254 status = child.wait()
255 return out, err, status
258 def _RunCmdFile(cmd, env, via_shell, output, cwd):
259 """Run a command and save its output to a file.
261 @type cmd: string or list
262 @param cmd: Command to run
264 @param env: The environment to use
265 @type via_shell: bool
266 @param via_shell: if we should run via the shell
268 @param output: the filename in which to save the output
270 @param cwd: the working directory for the program
272 @return: the exit status
275 fh = open(output, "a")
277 child = subprocess.Popen(cmd, shell=via_shell,
278 stderr=subprocess.STDOUT,
280 stdin=subprocess.PIPE,
281 close_fds=True, env=env,
285 status = child.wait()
291 def RunParts(dir_name, env=None, reset_env=False):
292 """Run Scripts or programs in a directory
294 @type dir_name: string
295 @param dir_name: absolute path to a directory
297 @param env: The environment to use
298 @type reset_env: boolean
299 @param reset_env: whether to reset or keep the default os environment
300 @rtype: list of tuples
301 @return: list of (name, (one of RUNDIR_STATUS), RunResult)
307 dir_contents = ListVisibleFiles(dir_name)
309 logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
312 for relname in sorted(dir_contents):
313 fname = PathJoin(dir_name, relname)
314 if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
315 constants.EXT_PLUGIN_MASK.match(relname) is not None):
316 rr.append((relname, constants.RUNPARTS_SKIP, None))
319 result = RunCmd([fname], env=env, reset_env=reset_env)
320 except Exception, err: # pylint: disable-msg=W0703
321 rr.append((relname, constants.RUNPARTS_ERR, str(err)))
323 rr.append((relname, constants.RUNPARTS_RUN, result))
328 def RemoveFile(filename):
329 """Remove a file ignoring some errors.
331 Remove a file, ignoring non-existing ones or directories. Other
335 @param filename: the file to be removed
341 if err.errno not in (errno.ENOENT, errno.EISDIR):
345 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
349 @param old: Original path
353 @param mkdir: Whether to create target directory if it doesn't exist
354 @type mkdir_mode: int
355 @param mkdir_mode: Mode for newly created directories
359 return os.rename(old, new)
361 # In at least one use case of this function, the job queue, directory
362 # creation is very rare. Checking for the directory before renaming is not
364 if mkdir and err.errno == errno.ENOENT:
365 # Create directory and try again
366 dirname = os.path.dirname(new)
368 os.makedirs(dirname, mode=mkdir_mode)
370 # Ignore EEXIST. This is only handled in os.makedirs as included in
371 # Python 2.5 and above.
372 if err.errno != errno.EEXIST or not os.path.exists(dirname):
375 return os.rename(old, new)
380 def ResetTempfileModule():
381 """Resets the random name generator of the tempfile module.
383 This function should be called after C{os.fork} in the child process to
384 ensure it creates a newly seeded random generator. Otherwise it would
385 generate the same random parts as the parent process. If several processes
386 race for the creation of a temporary file, this could lead to one not getting
390 # pylint: disable-msg=W0212
391 if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
392 tempfile._once_lock.acquire()
394 # Reset random name generator
395 tempfile._name_sequence = None
397 tempfile._once_lock.release()
399 logging.critical("The tempfile module misses at least one of the"
400 " '_once_lock' and '_name_sequence' attributes")
403 def _FingerprintFile(filename):
404 """Compute the fingerprint of a file.
406 If the file does not exist, a None will be returned
410 @param filename: the filename to checksum
412 @return: the hex digest of the sha checksum of the contents
416 if not (os.path.exists(filename) and os.path.isfile(filename)):
429 return fp.hexdigest()
432 def FingerprintFiles(files):
433 """Compute fingerprints for a list of files.
436 @param files: the list of filename to fingerprint
438 @return: a dictionary filename: fingerprint, holding only
444 for filename in files:
445 cksum = _FingerprintFile(filename)
447 ret[filename] = cksum
452 def ForceDictType(target, key_types, allowed_values=None):
453 """Force the values of a dict to have certain types.
456 @param target: the dict to update
457 @type key_types: dict
458 @param key_types: dict mapping target dict keys to types
459 in constants.ENFORCEABLE_TYPES
460 @type allowed_values: list
461 @keyword allowed_values: list of specially allowed values
464 if allowed_values is None:
467 if not isinstance(target, dict):
468 msg = "Expected dictionary, got '%s'" % target
469 raise errors.TypeEnforcementError(msg)
472 if key not in key_types:
473 msg = "Unknown key '%s'" % key
474 raise errors.TypeEnforcementError(msg)
476 if target[key] in allowed_values:
479 ktype = key_types[key]
480 if ktype not in constants.ENFORCEABLE_TYPES:
481 msg = "'%s' has non-enforceable type %s" % (key, ktype)
482 raise errors.ProgrammerError(msg)
484 if ktype == constants.VTYPE_STRING:
485 if not isinstance(target[key], basestring):
486 if isinstance(target[key], bool) and not target[key]:
489 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
490 raise errors.TypeEnforcementError(msg)
491 elif ktype == constants.VTYPE_BOOL:
492 if isinstance(target[key], basestring) and target[key]:
493 if target[key].lower() == constants.VALUE_FALSE:
495 elif target[key].lower() == constants.VALUE_TRUE:
498 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
499 raise errors.TypeEnforcementError(msg)
504 elif ktype == constants.VTYPE_SIZE:
506 target[key] = ParseUnit(target[key])
507 except errors.UnitParseError, err:
508 msg = "'%s' (value %s) is not a valid size. error: %s" % \
509 (key, target[key], err)
510 raise errors.TypeEnforcementError(msg)
511 elif ktype == constants.VTYPE_INT:
513 target[key] = int(target[key])
514 except (ValueError, TypeError):
515 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
516 raise errors.TypeEnforcementError(msg)
519 def IsProcessAlive(pid):
520 """Check if a given pid exists on the system.
522 @note: zombie status is not handled, so zombie processes
523 will be returned as alive
525 @param pid: the process ID to check
527 @return: True if the process exists
534 os.stat("/proc/%d/status" % pid)
536 except EnvironmentError, err:
537 if err.errno in (errno.ENOENT, errno.ENOTDIR):
542 def ReadPidFile(pidfile):
543 """Read a pid from a file.
545 @type pidfile: string
546 @param pidfile: path to the file containing the pid
548 @return: The process id, if the file exists and contains a valid PID,
553 raw_data = ReadFile(pidfile)
554 except EnvironmentError, err:
555 if err.errno != errno.ENOENT:
556 logging.exception("Can't read pid file")
561 except (TypeError, ValueError), err:
562 logging.info("Can't parse pid file contents", exc_info=True)
568 def MatchNameComponent(key, name_list, case_sensitive=True):
569 """Try to match a name against a list.
571 This function will try to match a name like test1 against a list
572 like C{['test1.example.com', 'test2.example.com', ...]}. Against
573 this list, I{'test1'} as well as I{'test1.example'} will match, but
574 not I{'test1.ex'}. A multiple match will be considered as no match
575 at all (e.g. I{'test1'} against C{['test1.example.com',
576 'test1.example.org']}), except when the key fully matches an entry
577 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
580 @param key: the name to be searched
581 @type name_list: list
582 @param name_list: the list of strings against which to search the key
583 @type case_sensitive: boolean
584 @param case_sensitive: whether to provide a case-sensitive match
587 @return: None if there is no match I{or} if there are multiple matches,
588 otherwise the element from the list which matches
595 if not case_sensitive:
596 re_flags |= re.IGNORECASE
598 mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
601 for name in name_list:
602 if mo.match(name) is not None:
603 names_filtered.append(name)
604 if not case_sensitive and key == name.upper():
605 string_matches.append(name)
607 if len(string_matches) == 1:
608 return string_matches[0]
609 if len(names_filtered) == 1:
610 return names_filtered[0]
615 """Class implementing resolver and hostname functionality
618 _VALID_NAME_RE = re.compile("^[a-z0-9._-]{1,255}$")
620 def __init__(self, name=None):
621 """Initialize the host name object.
623 If the name argument is not passed, it will use this system's
628 name = self.SysName()
631 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
632 self.ip = self.ipaddrs[0]
635 """Returns the hostname without domain.
638 return self.name.split('.')[0]
642 """Return the current system's name.
644 This is simply a wrapper over C{socket.gethostname()}.
647 return socket.gethostname()
650 def LookupHostname(hostname):
654 @param hostname: hostname to look up
657 @return: a tuple (name, aliases, ipaddrs) as returned by
658 C{socket.gethostbyname_ex}
659 @raise errors.ResolverError: in case of errors in resolving
663 result = socket.gethostbyname_ex(hostname)
664 except socket.gaierror, err:
665 # hostname not found in DNS
666 raise errors.ResolverError(hostname, err.args[0], err.args[1])
671 def NormalizeName(cls, hostname):
672 """Validate and normalize the given hostname.
674 @attention: the validation is a bit more relaxed than the standards
675 require; most importantly, we allow underscores in names
676 @raise errors.OpPrereqError: when the name is not valid
679 hostname = hostname.lower()
680 if (not cls._VALID_NAME_RE.match(hostname) or
681 # double-dots, meaning empty label
683 # empty initial label
684 hostname.startswith(".")):
685 raise errors.OpPrereqError("Invalid hostname '%s'" % hostname,
687 if hostname.endswith("."):
688 hostname = hostname.rstrip(".")
692 def GetHostInfo(name=None):
693 """Lookup host name and raise an OpPrereqError for failures"""
696 return HostInfo(name)
697 except errors.ResolverError, err:
698 raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
699 (err[0], err[2]), errors.ECODE_RESOLVER)
702 def ListVolumeGroups():
703 """List volume groups and their size
707 Dictionary with keys volume name and values
708 the size of the volume
711 command = "vgs --noheadings --units m --nosuffix -o name,size"
712 result = RunCmd(command)
717 for line in result.stdout.splitlines():
719 name, size = line.split()
720 size = int(float(size))
721 except (IndexError, ValueError), err:
722 logging.error("Invalid output from vgs (%s): %s", err, line)
730 def BridgeExists(bridge):
731 """Check whether the given bridge exists in the system
734 @param bridge: the bridge name to check
736 @return: True if it does
739 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
742 def NiceSort(name_list):
743 """Sort a list of strings based on digit and non-digit groupings.
745 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
746 will sort the list in the logical order C{['a1', 'a2', 'a10',
749 The sort algorithm breaks each name in groups of either only-digits
750 or no-digits. Only the first eight such groups are considered, and
751 after that we just use what's left of the string.
753 @type name_list: list
754 @param name_list: the names to be sorted
756 @return: a copy of the name list sorted with our algorithm
759 _SORTER_BASE = "(\D+|\d+)"
760 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
761 _SORTER_BASE, _SORTER_BASE,
762 _SORTER_BASE, _SORTER_BASE,
763 _SORTER_BASE, _SORTER_BASE)
764 _SORTER_RE = re.compile(_SORTER_FULL)
765 _SORTER_NODIGIT = re.compile("^\D*$")
767 """Attempts to convert a variable to integer."""
768 if val is None or _SORTER_NODIGIT.match(val):
773 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
774 for name in name_list]
776 return [tup[1] for tup in to_sort]
779 def TryConvert(fn, val):
780 """Try to convert a value ignoring errors.
782 This function tries to apply function I{fn} to I{val}. If no
783 C{ValueError} or C{TypeError} exceptions are raised, it will return
784 the result, else it will return the original value. Any other
785 exceptions are propagated to the caller.
788 @param fn: function to apply to the value
789 @param val: the value to be converted
790 @return: The converted value if the conversion was successful,
791 otherwise the original value.
796 except (ValueError, TypeError):
802 """Verifies the syntax of an IPv4 address.
804 This function checks if the IPv4 address passes is valid or not based
805 on syntax (not IP range, class calculations, etc.).
808 @param ip: the address to be checked
809 @rtype: a regular expression match object
810 @return: a regular expression match object, or None if the
814 unit = "(0|[1-9]\d{0,2})"
815 #TODO: convert and return only boolean
816 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
819 def IsValidShellParam(word):
820 """Verifies is the given word is safe from the shell's p.o.v.
822 This means that we can pass this to a command via the shell and be
823 sure that it doesn't alter the command line and is passed as such to
826 Note that we are overly restrictive here, in order to be on the safe
830 @param word: the word to check
832 @return: True if the word is 'safe'
835 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
838 def BuildShellCmd(template, *args):
839 """Build a safe shell command line from the given arguments.
841 This function will check all arguments in the args list so that they
842 are valid shell parameters (i.e. they don't contain shell
843 metacharacters). If everything is ok, it will return the result of
847 @param template: the string holding the template for the
850 @return: the expanded command line
854 if not IsValidShellParam(word):
855 raise errors.ProgrammerError("Shell argument '%s' contains"
856 " invalid characters" % word)
857 return template % args
860 def FormatUnit(value, units):
861 """Formats an incoming number of MiB with the appropriate unit.
864 @param value: integer representing the value in MiB (1048576)
866 @param units: the type of formatting we should do:
867 - 'h' for automatic scaling
872 @return: the formatted value (with suffix)
875 if units not in ('m', 'g', 't', 'h'):
876 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
880 if units == 'm' or (units == 'h' and value < 1024):
883 return "%d%s" % (round(value, 0), suffix)
885 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
888 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
893 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
896 def ParseUnit(input_string):
897 """Tries to extract number and scale from the given string.
899 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
900 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
901 is always an int in MiB.
904 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
906 raise errors.UnitParseError("Invalid format")
908 value = float(m.groups()[0])
912 lcunit = unit.lower()
916 if lcunit in ('m', 'mb', 'mib'):
917 # Value already in MiB
920 elif lcunit in ('g', 'gb', 'gib'):
923 elif lcunit in ('t', 'tb', 'tib'):
927 raise errors.UnitParseError("Unknown unit: %s" % unit)
929 # Make sure we round up
930 if int(value) < value:
933 # Round up to the next multiple of 4
936 value += 4 - value % 4
941 def AddAuthorizedKey(file_name, key):
942 """Adds an SSH public key to an authorized_keys file.
945 @param file_name: path to authorized_keys file
947 @param key: string containing key
950 key_fields = key.split()
952 f = open(file_name, 'a+')
956 # Ignore whitespace changes
957 if line.split() == key_fields:
959 nl = line.endswith('\n')
963 f.write(key.rstrip('\r\n'))
970 def RemoveAuthorizedKey(file_name, key):
971 """Removes an SSH public key from an authorized_keys file.
974 @param file_name: path to authorized_keys file
976 @param key: string containing key
979 key_fields = key.split()
981 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
983 out = os.fdopen(fd, 'w')
985 f = open(file_name, 'r')
988 # Ignore whitespace changes while comparing lines
989 if line.split() != key_fields:
993 os.rename(tmpname, file_name)
1003 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1004 """Sets the name of an IP address and hostname in /etc/hosts.
1006 @type file_name: str
1007 @param file_name: path to the file to modify (usually C{/etc/hosts})
1009 @param ip: the IP address
1011 @param hostname: the hostname to be added
1013 @param aliases: the list of aliases to add for the hostname
1016 # FIXME: use WriteFile + fn rather than duplicating its efforts
1017 # Ensure aliases are unique
1018 aliases = UniqueSequence([hostname] + aliases)[1:]
1020 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1022 out = os.fdopen(fd, 'w')
1024 f = open(file_name, 'r')
1027 fields = line.split()
1028 if fields and not fields[0].startswith('#') and ip == fields[0]:
1032 out.write("%s\t%s" % (ip, hostname))
1034 out.write(" %s" % ' '.join(aliases))
1039 os.chmod(tmpname, 0644)
1040 os.rename(tmpname, file_name)
1050 def AddHostToEtcHosts(hostname):
1051 """Wrapper around SetEtcHostsEntry.
1054 @param hostname: a hostname that will be resolved and added to
1055 L{constants.ETC_HOSTS}
1058 hi = HostInfo(name=hostname)
1059 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1062 def RemoveEtcHostsEntry(file_name, hostname):
1063 """Removes a hostname from /etc/hosts.
1065 IP addresses without names are removed from the file.
1067 @type file_name: str
1068 @param file_name: path to the file to modify (usually C{/etc/hosts})
1070 @param hostname: the hostname to be removed
1073 # FIXME: use WriteFile + fn rather than duplicating its efforts
1074 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1076 out = os.fdopen(fd, 'w')
1078 f = open(file_name, 'r')
1081 fields = line.split()
1082 if len(fields) > 1 and not fields[0].startswith('#'):
1084 if hostname in names:
1085 while hostname in names:
1086 names.remove(hostname)
1088 out.write("%s %s\n" % (fields[0], ' '.join(names)))
1095 os.chmod(tmpname, 0644)
1096 os.rename(tmpname, file_name)
1106 def RemoveHostFromEtcHosts(hostname):
1107 """Wrapper around RemoveEtcHostsEntry.
1110 @param hostname: hostname that will be resolved and its
1111 full and shot name will be removed from
1112 L{constants.ETC_HOSTS}
1115 hi = HostInfo(name=hostname)
1116 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1117 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1120 def CreateBackup(file_name):
1121 """Creates a backup of a file.
1123 @type file_name: str
1124 @param file_name: file to be backed up
1126 @return: the path to the newly created backup
1127 @raise errors.ProgrammerError: for invalid file names
1130 if not os.path.isfile(file_name):
1131 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1134 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1135 dir_name = os.path.dirname(file_name)
1137 fsrc = open(file_name, 'rb')
1139 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1140 fdst = os.fdopen(fd, 'wb')
1142 shutil.copyfileobj(fsrc, fdst)
1151 def ShellQuote(value):
1152 """Quotes shell argument according to POSIX.
1155 @param value: the argument to be quoted
1157 @return: the quoted value
1160 if _re_shell_unquoted.match(value):
1163 return "'%s'" % value.replace("'", "'\\''")
1166 def ShellQuoteArgs(args):
1167 """Quotes a list of shell arguments.
1170 @param args: list of arguments to be quoted
1172 @return: the quoted arguments concatenated with spaces
1175 return ' '.join([ShellQuote(i) for i in args])
1178 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1179 """Simple ping implementation using TCP connect(2).
1181 Check if the given IP is reachable by doing attempting a TCP connect
1185 @param target: the IP or hostname to ping
1187 @param port: the port to connect to
1189 @param timeout: the timeout on the connection attempt
1190 @type live_port_needed: boolean
1191 @param live_port_needed: whether a closed port will cause the
1192 function to return failure, as if there was a timeout
1193 @type source: str or None
1194 @param source: if specified, will cause the connect to be made
1195 from this specific source address; failures to bind other
1196 than C{EADDRNOTAVAIL} will be ignored
1199 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1203 if source is not None:
1205 sock.bind((source, 0))
1206 except socket.error, (errcode, _):
1207 if errcode == errno.EADDRNOTAVAIL:
1210 sock.settimeout(timeout)
1213 sock.connect((target, port))
1216 except socket.timeout:
1218 except socket.error, (errcode, _):
1219 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1224 def OwnIpAddress(address):
1225 """Check if the current host has the the given IP address.
1227 Currently this is done by TCP-pinging the address from the loopback
1230 @type address: string
1231 @param address: the address to check
1233 @return: True if we own the address
1236 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1237 source=constants.LOCALHOST_IP_ADDRESS)
1240 def ListVisibleFiles(path):
1241 """Returns a list of visible files in a directory.
1244 @param path: the directory to enumerate
1246 @return: the list of all files not starting with a dot
1247 @raise ProgrammerError: if L{path} is not an absolue and normalized path
1250 if not IsNormAbsPath(path):
1251 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
1252 " absolute/normalized: '%s'" % path)
1253 files = [i for i in os.listdir(path) if not i.startswith(".")]
1258 def GetHomeDir(user, default=None):
1259 """Try to get the homedir of the given user.
1261 The user can be passed either as a string (denoting the name) or as
1262 an integer (denoting the user id). If the user is not found, the
1263 'default' argument is returned, which defaults to None.
1267 if isinstance(user, basestring):
1268 result = pwd.getpwnam(user)
1269 elif isinstance(user, (int, long)):
1270 result = pwd.getpwuid(user)
1272 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1276 return result.pw_dir
1280 """Returns a random UUID.
1282 @note: This is a Linux-specific method as it uses the /proc
1287 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1290 def GenerateSecret(numbytes=20):
1291 """Generates a random secret.
1293 This will generate a pseudo-random secret returning an hex string
1294 (so that it can be used where an ASCII string is needed).
1296 @param numbytes: the number of bytes which will be represented by the returned
1297 string (defaulting to 20, the length of a SHA1 hash)
1299 @return: an hex representation of the pseudo-random sequence
1302 return os.urandom(numbytes).encode('hex')
1305 def EnsureDirs(dirs):
1306 """Make required directories, if they don't exist.
1308 @param dirs: list of tuples (dir_name, dir_mode)
1309 @type dirs: list of (string, integer)
1312 for dir_name, dir_mode in dirs:
1314 os.mkdir(dir_name, dir_mode)
1315 except EnvironmentError, err:
1316 if err.errno != errno.EEXIST:
1317 raise errors.GenericError("Cannot create needed directory"
1318 " '%s': %s" % (dir_name, err))
1319 if not os.path.isdir(dir_name):
1320 raise errors.GenericError("%s is not a directory" % dir_name)
1323 def ReadFile(file_name, size=-1):
1327 @param size: Read at most size bytes (if negative, entire file)
1329 @return: the (possibly partial) content of the file
1332 f = open(file_name, "r")
1339 def WriteFile(file_name, fn=None, data=None,
1340 mode=None, uid=-1, gid=-1,
1341 atime=None, mtime=None, close=True,
1342 dry_run=False, backup=False,
1343 prewrite=None, postwrite=None):
1344 """(Over)write a file atomically.
1346 The file_name and either fn (a function taking one argument, the
1347 file descriptor, and which should write the data to it) or data (the
1348 contents of the file) must be passed. The other arguments are
1349 optional and allow setting the file mode, owner and group, and the
1350 mtime/atime of the file.
1352 If the function doesn't raise an exception, it has succeeded and the
1353 target file has the new contents. If the function has raised an
1354 exception, an existing target file should be unmodified and the
1355 temporary file should be removed.
1357 @type file_name: str
1358 @param file_name: the target filename
1360 @param fn: content writing function, called with
1361 file descriptor as parameter
1363 @param data: contents of the file
1365 @param mode: file mode
1367 @param uid: the owner of the file
1369 @param gid: the group of the file
1371 @param atime: a custom access time to be set on the file
1373 @param mtime: a custom modification time to be set on the file
1374 @type close: boolean
1375 @param close: whether to close file after writing it
1376 @type prewrite: callable
1377 @param prewrite: function to be called before writing content
1378 @type postwrite: callable
1379 @param postwrite: function to be called after writing content
1382 @return: None if the 'close' parameter evaluates to True,
1383 otherwise the file descriptor
1385 @raise errors.ProgrammerError: if any of the arguments are not valid
1388 if not os.path.isabs(file_name):
1389 raise errors.ProgrammerError("Path passed to WriteFile is not"
1390 " absolute: '%s'" % file_name)
1392 if [fn, data].count(None) != 1:
1393 raise errors.ProgrammerError("fn or data required")
1395 if [atime, mtime].count(None) == 1:
1396 raise errors.ProgrammerError("Both atime and mtime must be either"
1399 if backup and not dry_run and os.path.isfile(file_name):
1400 CreateBackup(file_name)
1402 dir_name, base_name = os.path.split(file_name)
1403 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1405 # here we need to make sure we remove the temp file, if any error
1406 # leaves it in place
1408 if uid != -1 or gid != -1:
1409 os.chown(new_name, uid, gid)
1411 os.chmod(new_name, mode)
1412 if callable(prewrite):
1414 if data is not None:
1418 if callable(postwrite):
1421 if atime is not None and mtime is not None:
1422 os.utime(new_name, (atime, mtime))
1424 os.rename(new_name, file_name)
1433 RemoveFile(new_name)
1438 def FirstFree(seq, base=0):
1439 """Returns the first non-existing integer from seq.
1441 The seq argument should be a sorted list of positive integers. The
1442 first time the index of an element is smaller than the element
1443 value, the index will be returned.
1445 The base argument is used to start at a different offset,
1446 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1448 Example: C{[0, 1, 3]} will return I{2}.
1451 @param seq: the sequence to be analyzed.
1453 @param base: use this value as the base index of the sequence
1455 @return: the first non-used index in the sequence
1458 for idx, elem in enumerate(seq):
1459 assert elem >= base, "Passed element is higher than base offset"
1460 if elem > idx + base:
1466 def all(seq, pred=bool): # pylint: disable-msg=W0622
1467 "Returns True if pred(x) is True for every element in the iterable"
1468 for _ in itertools.ifilterfalse(pred, seq):
1473 def any(seq, pred=bool): # pylint: disable-msg=W0622
1474 "Returns True if pred(x) is True for at least one element in the iterable"
1475 for _ in itertools.ifilter(pred, seq):
1480 def UniqueSequence(seq):
1481 """Returns a list with unique elements.
1483 Element order is preserved.
1486 @param seq: the sequence with the source elements
1488 @return: list of unique elements from seq
1492 return [i for i in seq if i not in seen and not seen.add(i)]
1495 def NormalizeAndValidateMac(mac):
1496 """Normalizes and check if a MAC address is valid.
1498 Checks whether the supplied MAC address is formally correct, only
1499 accepts colon separated format. Normalize it to all lower.
1502 @param mac: the MAC to be validated
1504 @return: returns the normalized and validated MAC.
1506 @raise errors.OpPrereqError: If the MAC isn't valid
1509 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1510 if not mac_check.match(mac):
1511 raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1512 mac, errors.ECODE_INVAL)
1517 def TestDelay(duration):
1518 """Sleep for a fixed amount of time.
1520 @type duration: float
1521 @param duration: the sleep duration
1523 @return: False for negative value, True otherwise
1527 return False, "Invalid sleep duration"
1528 time.sleep(duration)
1532 def _CloseFDNoErr(fd, retries=5):
1533 """Close a file descriptor ignoring errors.
1536 @param fd: the file descriptor
1538 @param retries: how many retries to make, in case we get any
1539 other error than EBADF
1544 except OSError, err:
1545 if err.errno != errno.EBADF:
1547 _CloseFDNoErr(fd, retries - 1)
1548 # else either it's closed already or we're out of retries, so we
1549 # ignore this and go on
1552 def CloseFDs(noclose_fds=None):
1553 """Close file descriptors.
1555 This closes all file descriptors above 2 (i.e. except
1558 @type noclose_fds: list or None
1559 @param noclose_fds: if given, it denotes a list of file descriptor
1560 that should not be closed
1563 # Default maximum for the number of available file descriptors.
1564 if 'SC_OPEN_MAX' in os.sysconf_names:
1566 MAXFD = os.sysconf('SC_OPEN_MAX')
1573 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1574 if (maxfd == resource.RLIM_INFINITY):
1577 # Iterate through and close all file descriptors (except the standard ones)
1578 for fd in range(3, maxfd):
1579 if noclose_fds and fd in noclose_fds:
1584 def Daemonize(logfile):
1585 """Daemonize the current process.
1587 This detaches the current process from the controlling terminal and
1588 runs it in the background as a daemon.
1591 @param logfile: the logfile to which we should redirect stdout/stderr
1593 @return: the value zero
1596 # pylint: disable-msg=W0212
1597 # yes, we really want os._exit
1603 if (pid == 0): # The first child.
1606 pid = os.fork() # Fork a second child.
1607 if (pid == 0): # The second child.
1611 # exit() or _exit()? See below.
1612 os._exit(0) # Exit parent (the first child) of the second child.
1614 os._exit(0) # Exit parent of the first child.
1618 i = os.open("/dev/null", os.O_RDONLY) # stdin
1619 assert i == 0, "Can't close/reopen stdin"
1620 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1621 assert i == 1, "Can't close/reopen stdout"
1622 # Duplicate standard output to standard error.
1627 def DaemonPidFileName(name):
1628 """Compute a ganeti pid file absolute path
1631 @param name: the daemon name
1633 @return: the full path to the pidfile corresponding to the given
1637 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
1640 def EnsureDaemon(name):
1641 """Check for and start daemon if not alive.
1644 result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
1646 logging.error("Can't start daemon '%s', failure %s, output: %s",
1647 name, result.fail_reason, result.output)
1653 def WritePidFile(name):
1654 """Write the current process pidfile.
1656 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1659 @param name: the daemon name to use
1660 @raise errors.GenericError: if the pid file already exists and
1661 points to a live process
1665 pidfilename = DaemonPidFileName(name)
1666 if IsProcessAlive(ReadPidFile(pidfilename)):
1667 raise errors.GenericError("%s contains a live process" % pidfilename)
1669 WriteFile(pidfilename, data="%d\n" % pid)
1672 def RemovePidFile(name):
1673 """Remove the current process pidfile.
1675 Any errors are ignored.
1678 @param name: the daemon name used to derive the pidfile name
1681 pidfilename = DaemonPidFileName(name)
1682 # TODO: we could check here that the file contains our pid
1684 RemoveFile(pidfilename)
1685 except: # pylint: disable-msg=W0702
1689 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1691 """Kill a process given by its pid.
1694 @param pid: The PID to terminate.
1696 @param signal_: The signal to send, by default SIGTERM
1698 @param timeout: The timeout after which, if the process is still alive,
1699 a SIGKILL will be sent. If not positive, no such checking
1701 @type waitpid: boolean
1702 @param waitpid: If true, we should waitpid on this process after
1703 sending signals, since it's our own child and otherwise it
1704 would remain as zombie
1707 def _helper(pid, signal_, wait):
1708 """Simple helper to encapsulate the kill/waitpid sequence"""
1709 os.kill(pid, signal_)
1712 os.waitpid(pid, os.WNOHANG)
1717 # kill with pid=0 == suicide
1718 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1720 if not IsProcessAlive(pid):
1723 _helper(pid, signal_, waitpid)
1728 def _CheckProcess():
1729 if not IsProcessAlive(pid):
1733 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1743 # Wait up to $timeout seconds
1744 Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1745 except RetryTimeout:
1748 if IsProcessAlive(pid):
1749 # Kill process if it's still alive
1750 _helper(pid, signal.SIGKILL, waitpid)
1753 def FindFile(name, search_path, test=os.path.exists):
1754 """Look for a filesystem object in a given path.
1756 This is an abstract method to search for filesystem object (files,
1757 dirs) under a given search path.
1760 @param name: the name to look for
1761 @type search_path: str
1762 @param search_path: location to start at
1763 @type test: callable
1764 @param test: a function taking one argument that should return True
1765 if the a given object is valid; the default value is
1766 os.path.exists, causing only existing files to be returned
1768 @return: full path to the object if found, None otherwise
1771 # validate the filename mask
1772 if constants.EXT_PLUGIN_MASK.match(name) is None:
1773 logging.critical("Invalid value passed for external script name: '%s'",
1777 for dir_name in search_path:
1778 # FIXME: investigate switch to PathJoin
1779 item_name = os.path.sep.join([dir_name, name])
1780 # check the user test and that we're indeed resolving to the given
1782 if test(item_name) and os.path.basename(item_name) == name:
1787 def CheckVolumeGroupSize(vglist, vgname, minsize):
1788 """Checks if the volume group list is valid.
1790 The function will check if a given volume group is in the list of
1791 volume groups and has a minimum size.
1794 @param vglist: dictionary of volume group names and their size
1796 @param vgname: the volume group we should check
1798 @param minsize: the minimum size we accept
1800 @return: None for success, otherwise the error message
1803 vgsize = vglist.get(vgname, None)
1805 return "volume group '%s' missing" % vgname
1806 elif vgsize < minsize:
1807 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1808 (vgname, minsize, vgsize))
1812 def SplitTime(value):
1813 """Splits time as floating point number into a tuple.
1815 @param value: Time in seconds
1816 @type value: int or float
1817 @return: Tuple containing (seconds, microseconds)
1820 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1822 assert 0 <= seconds, \
1823 "Seconds must be larger than or equal to 0, but are %s" % seconds
1824 assert 0 <= microseconds <= 999999, \
1825 "Microseconds must be 0-999999, but are %s" % microseconds
1827 return (int(seconds), int(microseconds))
1830 def MergeTime(timetuple):
1831 """Merges a tuple into time as a floating point number.
1833 @param timetuple: Time as tuple, (seconds, microseconds)
1834 @type timetuple: tuple
1835 @return: Time as a floating point number expressed in seconds
1838 (seconds, microseconds) = timetuple
1840 assert 0 <= seconds, \
1841 "Seconds must be larger than or equal to 0, but are %s" % seconds
1842 assert 0 <= microseconds <= 999999, \
1843 "Microseconds must be 0-999999, but are %s" % microseconds
1845 return float(seconds) + (float(microseconds) * 0.000001)
1848 def GetDaemonPort(daemon_name):
1849 """Get the daemon port for this cluster.
1851 Note that this routine does not read a ganeti-specific file, but
1852 instead uses C{socket.getservbyname} to allow pre-customization of
1853 this parameter outside of Ganeti.
1855 @type daemon_name: string
1856 @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1860 if daemon_name not in constants.DAEMONS_PORTS:
1861 raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1863 (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1865 port = socket.getservbyname(daemon_name, proto)
1866 except socket.error:
1872 def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
1873 multithreaded=False, syslog=constants.SYSLOG_USAGE):
1874 """Configures the logging module.
1877 @param logfile: the filename to which we should log
1878 @type debug: integer
1879 @param debug: if greater than zero, enable debug messages, otherwise
1880 only those at C{INFO} and above level
1881 @type stderr_logging: boolean
1882 @param stderr_logging: whether we should also log to the standard error
1884 @param program: the name under which we should log messages
1885 @type multithreaded: boolean
1886 @param multithreaded: if True, will add the thread name to the log file
1887 @type syslog: string
1888 @param syslog: one of 'no', 'yes', 'only':
1889 - if no, syslog is not used
1890 - if yes, syslog is used (in addition to file-logging)
1891 - if only, only syslog is used
1892 @raise EnvironmentError: if we can't open the log file and
1893 syslog/stderr logging is disabled
1896 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1897 sft = program + "[%(process)d]:"
1899 fmt += "/%(threadName)s"
1900 sft += " (%(threadName)s)"
1902 fmt += " %(module)s:%(lineno)s"
1903 # no debug info for syslog loggers
1904 fmt += " %(levelname)s %(message)s"
1905 # yes, we do want the textual level, as remote syslog will probably
1906 # lose the error level, and it's easier to grep for it
1907 sft += " %(levelname)s %(message)s"
1908 formatter = logging.Formatter(fmt)
1909 sys_fmt = logging.Formatter(sft)
1911 root_logger = logging.getLogger("")
1912 root_logger.setLevel(logging.NOTSET)
1914 # Remove all previously setup handlers
1915 for handler in root_logger.handlers:
1917 root_logger.removeHandler(handler)
1920 stderr_handler = logging.StreamHandler()
1921 stderr_handler.setFormatter(formatter)
1923 stderr_handler.setLevel(logging.NOTSET)
1925 stderr_handler.setLevel(logging.CRITICAL)
1926 root_logger.addHandler(stderr_handler)
1928 if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
1929 facility = logging.handlers.SysLogHandler.LOG_DAEMON
1930 syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
1932 syslog_handler.setFormatter(sys_fmt)
1933 # Never enable debug over syslog
1934 syslog_handler.setLevel(logging.INFO)
1935 root_logger.addHandler(syslog_handler)
1937 if syslog != constants.SYSLOG_ONLY:
1938 # this can fail, if the logging directories are not setup or we have
1939 # a permisssion problem; in this case, it's best to log but ignore
1940 # the error if stderr_logging is True, and if false we re-raise the
1941 # exception since otherwise we could run but without any logs at all
1943 logfile_handler = logging.FileHandler(logfile)
1944 logfile_handler.setFormatter(formatter)
1946 logfile_handler.setLevel(logging.DEBUG)
1948 logfile_handler.setLevel(logging.INFO)
1949 root_logger.addHandler(logfile_handler)
1950 except EnvironmentError:
1951 if stderr_logging or syslog == constants.SYSLOG_YES:
1952 logging.exception("Failed to enable logging to file '%s'", logfile)
1954 # we need to re-raise the exception
1958 def IsNormAbsPath(path):
1959 """Check whether a path is absolute and also normalized
1961 This avoids things like /dir/../../other/path to be valid.
1964 return os.path.normpath(path) == path and os.path.isabs(path)
1967 def PathJoin(*args):
1968 """Safe-join a list of path components.
1971 - the first argument must be an absolute path
1972 - no component in the path must have backtracking (e.g. /../),
1973 since we check for normalization at the end
1975 @param args: the path components to be joined
1976 @raise ValueError: for invalid paths
1979 # ensure we're having at least one path passed in
1981 # ensure the first component is an absolute and normalized path name
1983 if not IsNormAbsPath(root):
1984 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
1985 result = os.path.join(*args)
1986 # ensure that the whole path is normalized
1987 if not IsNormAbsPath(result):
1988 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
1989 # check that we're still under the original prefix
1990 prefix = os.path.commonprefix([root, result])
1992 raise ValueError("Error: path joining resulted in different prefix"
1993 " (%s != %s)" % (prefix, root))
1997 def TailFile(fname, lines=20):
1998 """Return the last lines from a file.
2000 @note: this function will only read and parse the last 4KB of
2001 the file; if the lines are very long, it could be that less
2002 than the requested number of lines are returned
2004 @param fname: the file name
2006 @param lines: the (maximum) number of lines to return
2009 fd = open(fname, "r")
2013 pos = max(0, pos-4096)
2015 raw_data = fd.read()
2019 rows = raw_data.splitlines()
2020 return rows[-lines:]
2023 def SafeEncode(text):
2024 """Return a 'safe' version of a source string.
2026 This function mangles the input string and returns a version that
2027 should be safe to display/encode as ASCII. To this end, we first
2028 convert it to ASCII using the 'backslashreplace' encoding which
2029 should get rid of any non-ASCII chars, and then we process it
2030 through a loop copied from the string repr sources in the python; we
2031 don't use string_escape anymore since that escape single quotes and
2032 backslashes too, and that is too much; and that escaping is not
2033 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2035 @type text: str or unicode
2036 @param text: input data
2038 @return: a safe version of text
2041 if isinstance(text, unicode):
2042 # only if unicode; if str already, we handle it below
2043 text = text.encode('ascii', 'backslashreplace')
2053 elif c < 32 or c >= 127: # non-printable
2054 resu += "\\x%02x" % (c & 0xff)
2060 def UnescapeAndSplit(text, sep=","):
2061 """Split and unescape a string based on a given separator.
2063 This function splits a string based on a separator where the
2064 separator itself can be escape in order to be an element of the
2065 elements. The escaping rules are (assuming coma being the
2067 - a plain , separates the elements
2068 - a sequence \\\\, (double backslash plus comma) is handled as a
2069 backslash plus a separator comma
2070 - a sequence \, (backslash plus comma) is handled as a
2074 @param text: the string to split
2076 @param text: the separator
2078 @return: a list of strings
2081 # we split the list by sep (with no escaping at this stage)
2082 slist = text.split(sep)
2083 # next, we revisit the elements and if any of them ended with an odd
2084 # number of backslashes, then we join it with the next
2088 if e1.endswith("\\"):
2089 num_b = len(e1) - len(e1.rstrip("\\"))
2092 # here the backslashes remain (all), and will be reduced in
2094 rlist.append(e1 + sep + e2)
2097 # finally, replace backslash-something with something
2098 rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2102 def CommaJoin(names):
2103 """Nicely join a set of identifiers.
2105 @param names: set, list or tuple
2106 @return: a string with the formatted results
2109 return ", ".join([str(val) for val in names])
2112 def BytesToMebibyte(value):
2113 """Converts bytes to mebibytes.
2116 @param value: Value in bytes
2118 @return: Value in mebibytes
2121 return int(round(value / (1024.0 * 1024.0), 0))
2124 def CalculateDirectorySize(path):
2125 """Calculates the size of a directory recursively.
2128 @param path: Path to directory
2130 @return: Size in mebibytes
2135 for (curpath, _, files) in os.walk(path):
2136 for filename in files:
2137 st = os.lstat(PathJoin(curpath, filename))
2140 return BytesToMebibyte(size)
2143 def GetFilesystemStats(path):
2144 """Returns the total and free space on a filesystem.
2147 @param path: Path on filesystem to be examined
2149 @return: tuple of (Total space, Free space) in mebibytes
2152 st = os.statvfs(path)
2154 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2155 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2156 return (tsize, fsize)
2159 def RunInSeparateProcess(fn, *args):
2160 """Runs a function in a separate process.
2162 Note: Only boolean return values are supported.
2165 @param fn: Function to be called
2167 @return: Function's result
2174 # In case the function uses temporary files
2175 ResetTempfileModule()
2178 result = int(bool(fn(*args)))
2179 assert result in (0, 1)
2180 except: # pylint: disable-msg=W0702
2181 logging.exception("Error while calling function in separate process")
2182 # 0 and 1 are reserved for the return value
2185 os._exit(result) # pylint: disable-msg=W0212
2189 # Avoid zombies and check exit code
2190 (_, status) = os.waitpid(pid, 0)
2192 if os.WIFSIGNALED(status):
2194 signum = os.WTERMSIG(status)
2196 exitcode = os.WEXITSTATUS(status)
2199 if not (exitcode in (0, 1) and signum is None):
2200 raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2203 return bool(exitcode)
2206 def LockedMethod(fn):
2207 """Synchronized object access decorator.
2209 This decorator is intended to protect access to an object using the
2210 object's own lock which is hardcoded to '_lock'.
2213 def _LockDebug(*args, **kwargs):
2215 logging.debug(*args, **kwargs)
2217 def wrapper(self, *args, **kwargs):
2218 # pylint: disable-msg=W0212
2219 assert hasattr(self, '_lock')
2221 _LockDebug("Waiting for %s", lock)
2224 _LockDebug("Acquired %s", lock)
2225 result = fn(self, *args, **kwargs)
2227 _LockDebug("Releasing %s", lock)
2229 _LockDebug("Released %s", lock)
2235 """Locks a file using POSIX locks.
2238 @param fd: the file descriptor we need to lock
2242 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2243 except IOError, err:
2244 if err.errno == errno.EAGAIN:
2245 raise errors.LockError("File already locked")
2249 def FormatTime(val):
2250 """Formats a time value.
2252 @type val: float or None
2253 @param val: the timestamp as returned by time.time()
2254 @return: a string value or N/A if we don't have a valid timestamp
2257 if val is None or not isinstance(val, (int, float)):
2259 # these two codes works on Linux, but they are not guaranteed on all
2261 return time.strftime("%F %T", time.localtime(val))
2264 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2265 """Reads the watcher pause file.
2267 @type filename: string
2268 @param filename: Path to watcher pause file
2269 @type now: None, float or int
2270 @param now: Current time as Unix timestamp
2271 @type remove_after: int
2272 @param remove_after: Remove watcher pause file after specified amount of
2273 seconds past the pause end time
2280 value = ReadFile(filename)
2281 except IOError, err:
2282 if err.errno != errno.ENOENT:
2286 if value is not None:
2290 logging.warning(("Watcher pause file (%s) contains invalid value,"
2291 " removing it"), filename)
2292 RemoveFile(filename)
2295 if value is not None:
2296 # Remove file if it's outdated
2297 if now > (value + remove_after):
2298 RemoveFile(filename)
2307 class RetryTimeout(Exception):
2308 """Retry loop timed out.
2313 class RetryAgain(Exception):
2319 class _RetryDelayCalculator(object):
2320 """Calculator for increasing delays.
2330 def __init__(self, start, factor, limit):
2331 """Initializes this class.
2334 @param start: Initial delay
2336 @param factor: Factor for delay increase
2337 @type limit: float or None
2338 @param limit: Upper limit for delay or None for no limit
2342 assert factor >= 1.0
2343 assert limit is None or limit >= 0.0
2346 self._factor = factor
2352 """Returns current delay and calculates the next one.
2355 current = self._next
2357 # Update for next run
2358 if self._limit is None or self._next < self._limit:
2359 self._next = min(self._limit, self._next * self._factor)
2364 #: Special delay to specify whole remaining timeout
2365 RETRY_REMAINING_TIME = object()
2368 def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2369 _time_fn=time.time):
2370 """Call a function repeatedly until it succeeds.
2372 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2373 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2374 total of C{timeout} seconds, this function throws L{RetryTimeout}.
2376 C{delay} can be one of the following:
2377 - callable returning the delay length as a float
2378 - Tuple of (start, factor, limit)
2379 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2380 useful when overriding L{wait_fn} to wait for an external event)
2381 - A static delay as a number (int or float)
2384 @param fn: Function to be called
2385 @param delay: Either a callable (returning the delay), a tuple of (start,
2386 factor, limit) (see L{_RetryDelayCalculator}),
2387 L{RETRY_REMAINING_TIME} or a number (int or float)
2388 @type timeout: float
2389 @param timeout: Total timeout
2390 @type wait_fn: callable
2391 @param wait_fn: Waiting function
2392 @return: Return value of function
2396 assert callable(wait_fn)
2397 assert callable(_time_fn)
2402 end_time = _time_fn() + timeout
2405 # External function to calculate delay
2408 elif isinstance(delay, (tuple, list)):
2409 # Increasing delay with optional upper boundary
2410 (start, factor, limit) = delay
2411 calc_delay = _RetryDelayCalculator(start, factor, limit)
2413 elif delay is RETRY_REMAINING_TIME:
2414 # Always use the remaining time
2419 calc_delay = lambda: delay
2421 assert calc_delay is None or callable(calc_delay)
2425 # pylint: disable-msg=W0142
2430 remaining_time = end_time - _time_fn()
2432 if remaining_time < 0.0:
2433 raise RetryTimeout()
2435 assert remaining_time >= 0.0
2437 if calc_delay is None:
2438 wait_fn(remaining_time)
2440 current_delay = calc_delay()
2441 if current_delay > 0.0:
2442 wait_fn(current_delay)
2445 class FileLock(object):
2446 """Utility class for file locks.
2449 def __init__(self, fd, filename):
2450 """Constructor for FileLock.
2453 @param fd: File object
2455 @param filename: Path of the file opened at I{fd}
2459 self.filename = filename
2462 def Open(cls, filename):
2463 """Creates and opens a file to be used as a file-based lock.
2465 @type filename: string
2466 @param filename: path to the file to be locked
2469 # Using "os.open" is necessary to allow both opening existing file
2470 # read/write and creating if not existing. Vanilla "open" will truncate an
2471 # existing file -or- allow creating if not existing.
2472 return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
2479 """Close the file and release the lock.
2482 if hasattr(self, "fd") and self.fd:
2486 def _flock(self, flag, blocking, timeout, errmsg):
2487 """Wrapper for fcntl.flock.
2490 @param flag: operation flag
2491 @type blocking: bool
2492 @param blocking: whether the operation should be done in blocking mode.
2493 @type timeout: None or float
2494 @param timeout: for how long the operation should be retried (implies
2496 @type errmsg: string
2497 @param errmsg: error message in case operation fails.
2500 assert self.fd, "Lock was closed"
2501 assert timeout is None or timeout >= 0, \
2502 "If specified, timeout must be positive"
2503 assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
2505 # When a timeout is used, LOCK_NB must always be set
2506 if not (timeout is None and blocking):
2507 flag |= fcntl.LOCK_NB
2510 self._Lock(self.fd, flag, timeout)
2513 Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
2514 args=(self.fd, flag, timeout))
2515 except RetryTimeout:
2516 raise errors.LockError(errmsg)
2519 def _Lock(fd, flag, timeout):
2521 fcntl.flock(fd, flag)
2522 except IOError, err:
2523 if timeout is not None and err.errno == errno.EAGAIN:
2526 logging.exception("fcntl.flock failed")
2529 def Exclusive(self, blocking=False, timeout=None):
2530 """Locks the file in exclusive mode.
2532 @type blocking: boolean
2533 @param blocking: whether to block and wait until we
2534 can lock the file or return immediately
2535 @type timeout: int or None
2536 @param timeout: if not None, the duration to wait for the lock
2540 self._flock(fcntl.LOCK_EX, blocking, timeout,
2541 "Failed to lock %s in exclusive mode" % self.filename)
2543 def Shared(self, blocking=False, timeout=None):
2544 """Locks the file in shared mode.
2546 @type blocking: boolean
2547 @param blocking: whether to block and wait until we
2548 can lock the file or return immediately
2549 @type timeout: int or None
2550 @param timeout: if not None, the duration to wait for the lock
2554 self._flock(fcntl.LOCK_SH, blocking, timeout,
2555 "Failed to lock %s in shared mode" % self.filename)
2557 def Unlock(self, blocking=True, timeout=None):
2558 """Unlocks the file.
2560 According to C{flock(2)}, unlocking can also be a nonblocking
2563 To make a non-blocking request, include LOCK_NB with any of the above
2566 @type blocking: boolean
2567 @param blocking: whether to block and wait until we
2568 can lock the file or return immediately
2569 @type timeout: int or None
2570 @param timeout: if not None, the duration to wait for the lock
2574 self._flock(fcntl.LOCK_UN, blocking, timeout,
2575 "Failed to unlock %s" % self.filename)
2578 def SignalHandled(signums):
2579 """Signal Handled decoration.
2581 This special decorator installs a signal handler and then calls the target
2582 function. The function must accept a 'signal_handlers' keyword argument,
2583 which will contain a dict indexed by signal number, with SignalHandler
2586 The decorator can be safely stacked with iself, to handle multiple signals
2587 with different handlers.
2590 @param signums: signals to intercept
2594 def sig_function(*args, **kwargs):
2595 assert 'signal_handlers' not in kwargs or \
2596 kwargs['signal_handlers'] is None or \
2597 isinstance(kwargs['signal_handlers'], dict), \
2598 "Wrong signal_handlers parameter in original function call"
2599 if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2600 signal_handlers = kwargs['signal_handlers']
2602 signal_handlers = {}
2603 kwargs['signal_handlers'] = signal_handlers
2604 sighandler = SignalHandler(signums)
2607 signal_handlers[sig] = sighandler
2608 return fn(*args, **kwargs)
2615 class SignalHandler(object):
2616 """Generic signal handler class.
2618 It automatically restores the original handler when deconstructed or
2619 when L{Reset} is called. You can either pass your own handler
2620 function in or query the L{called} attribute to detect whether the
2624 @ivar signum: the signals we handle
2625 @type called: boolean
2626 @ivar called: tracks whether any of the signals have been raised
2629 def __init__(self, signum):
2630 """Constructs a new SignalHandler instance.
2632 @type signum: int or list of ints
2633 @param signum: Single signal number or set of signal numbers
2636 self.signum = set(signum)
2641 for signum in self.signum:
2643 prev_handler = signal.signal(signum, self._HandleSignal)
2645 self._previous[signum] = prev_handler
2647 # Restore previous handler
2648 signal.signal(signum, prev_handler)
2651 # Reset all handlers
2653 # Here we have a race condition: a handler may have already been called,
2654 # but there's not much we can do about it at this point.
2661 """Restore previous handler.
2663 This will reset all the signals to their previous handlers.
2666 for signum, prev_handler in self._previous.items():
2667 signal.signal(signum, prev_handler)
2668 # If successful, remove from dict
2669 del self._previous[signum]
2672 """Unsets the L{called} flag.
2674 This function can be used in case a signal may arrive several times.
2679 # we don't care about arguments, but we leave them named for the future
2680 def _HandleSignal(self, signum, frame): # pylint: disable-msg=W0613
2681 """Actual signal handling function.
2684 # This is not nice and not absolutely atomic, but it appears to be the only
2685 # solution in Python -- there are no atomic types.
2689 class FieldSet(object):
2690 """A simple field set.
2692 Among the features are:
2693 - checking if a string is among a list of static string or regex objects
2694 - checking if a whole list of string matches
2695 - returning the matching groups from a regex match
2697 Internally, all fields are held as regular expression objects.
2700 def __init__(self, *items):
2701 self.items = [re.compile("^%s$" % value) for value in items]
2703 def Extend(self, other_set):
2704 """Extend the field set with the items from another one"""
2705 self.items.extend(other_set.items)
2707 def Matches(self, field):
2708 """Checks if a field matches the current set
2711 @param field: the string to match
2712 @return: either None or a regular expression match object
2715 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2719 def NonMatching(self, items):
2720 """Returns the list of fields not matching the current set
2723 @param items: the list of fields to check
2725 @return: list of non-matching fields
2728 return [val for val in items if not self.Matches(val)]