4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
44 import logging.handlers
47 from cStringIO import StringIO
50 from hashlib import sha1
55 from ganeti import errors
56 from ganeti import constants
60 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
64 #: when set to True, L{RunCmd} is disabled
67 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
70 class RunResult(object):
71 """Holds the result of running external programs.
74 @ivar exit_code: the exit code of the program, or None (if the program
76 @type signal: int or None
77 @ivar signal: the signal that caused the program to finish, or None
78 (if the program wasn't terminated by a signal)
80 @ivar stdout: the standard output of the program
82 @ivar stderr: the standard error of the program
84 @ivar failed: True in case the program was
85 terminated by a signal or exited with a non-zero exit code
86 @ivar fail_reason: a string detailing the termination reason
89 __slots__ = ["exit_code", "signal", "stdout", "stderr",
90 "failed", "fail_reason", "cmd"]
93 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
95 self.exit_code = exit_code
99 self.failed = (signal_ is not None or exit_code != 0)
101 if self.signal is not None:
102 self.fail_reason = "terminated by signal %s" % self.signal
103 elif self.exit_code is not None:
104 self.fail_reason = "exited with exit code %s" % self.exit_code
106 self.fail_reason = "unable to determine termination reason"
109 logging.debug("Command '%s' failed (%s); output: %s",
110 self.cmd, self.fail_reason, self.output)
112 def _GetOutput(self):
113 """Returns the combined stdout and stderr for easier usage.
116 return self.stdout + self.stderr
118 output = property(_GetOutput, None, None, "Return full output")
121 def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
122 """Execute a (shell) command.
124 The command should not read from its standard input, as it will be
127 @type cmd: string or list
128 @param cmd: Command to run
130 @param env: Additional environment
132 @param output: if desired, the output of the command can be
133 saved in a file instead of the RunResult instance; this
134 parameter denotes the file name (if not None)
136 @param cwd: if specified, will be used as the working
137 directory for the command; the default will be /
138 @type reset_env: boolean
139 @param reset_env: whether to reset or keep the default os environment
141 @return: RunResult instance
142 @raise errors.ProgrammerError: if we call this when forks are disabled
146 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
148 if isinstance(cmd, list):
149 cmd = [str(val) for val in cmd]
150 strcmd = " ".join(cmd)
155 logging.debug("RunCmd '%s'", strcmd)
158 cmd_env = os.environ.copy()
159 cmd_env["LC_ALL"] = "C"
168 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
170 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
173 if err.errno == errno.ENOENT:
174 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
186 return RunResult(exitcode, signal_, out, err, strcmd)
189 def _RunCmdPipe(cmd, env, via_shell, cwd):
190 """Run a command and return its output.
192 @type cmd: string or list
193 @param cmd: Command to run
195 @param env: The environment to use
196 @type via_shell: bool
197 @param via_shell: if we should run via the shell
199 @param cwd: the working directory for the program
201 @return: (out, err, status)
204 poller = select.poll()
205 child = subprocess.Popen(cmd, shell=via_shell,
206 stderr=subprocess.PIPE,
207 stdout=subprocess.PIPE,
208 stdin=subprocess.PIPE,
209 close_fds=True, env=env,
213 poller.register(child.stdout, select.POLLIN)
214 poller.register(child.stderr, select.POLLIN)
218 child.stdout.fileno(): (out, child.stdout),
219 child.stderr.fileno(): (err, child.stderr),
222 status = fcntl.fcntl(fd, fcntl.F_GETFL)
223 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
227 pollresult = poller.poll()
228 except EnvironmentError, eerr:
229 if eerr.errno == errno.EINTR:
232 except select.error, serr:
233 if serr[0] == errno.EINTR:
237 for fd, event in pollresult:
238 if event & select.POLLIN or event & select.POLLPRI:
239 data = fdmap[fd][1].read()
240 # no data from read signifies EOF (the same as POLLHUP)
242 poller.unregister(fd)
245 fdmap[fd][0].write(data)
246 if (event & select.POLLNVAL or event & select.POLLHUP or
247 event & select.POLLERR):
248 poller.unregister(fd)
254 status = child.wait()
255 return out, err, status
258 def _RunCmdFile(cmd, env, via_shell, output, cwd):
259 """Run a command and save its output to a file.
261 @type cmd: string or list
262 @param cmd: Command to run
264 @param env: The environment to use
265 @type via_shell: bool
266 @param via_shell: if we should run via the shell
268 @param output: the filename in which to save the output
270 @param cwd: the working directory for the program
272 @return: the exit status
275 fh = open(output, "a")
277 child = subprocess.Popen(cmd, shell=via_shell,
278 stderr=subprocess.STDOUT,
280 stdin=subprocess.PIPE,
281 close_fds=True, env=env,
285 status = child.wait()
291 def RunParts(dir_name, env=None, reset_env=False):
292 """Run Scripts or programs in a directory
294 @type dir_name: string
295 @param dir_name: absolute path to a directory
297 @param env: The environment to use
298 @type reset_env: boolean
299 @param reset_env: whether to reset or keep the default os environment
300 @rtype: list of tuples
301 @return: list of (name, (one of RUNDIR_STATUS), RunResult)
307 dir_contents = ListVisibleFiles(dir_name)
309 logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
312 for relname in sorted(dir_contents):
313 fname = PathJoin(dir_name, relname)
314 if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
315 constants.EXT_PLUGIN_MASK.match(relname) is not None):
316 rr.append((relname, constants.RUNPARTS_SKIP, None))
319 result = RunCmd([fname], env=env, reset_env=reset_env)
320 except Exception, err: # pylint: disable-msg=W0703
321 rr.append((relname, constants.RUNPARTS_ERR, str(err)))
323 rr.append((relname, constants.RUNPARTS_RUN, result))
328 def RemoveFile(filename):
329 """Remove a file ignoring some errors.
331 Remove a file, ignoring non-existing ones or directories. Other
335 @param filename: the file to be removed
341 if err.errno not in (errno.ENOENT, errno.EISDIR):
345 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
349 @param old: Original path
353 @param mkdir: Whether to create target directory if it doesn't exist
354 @type mkdir_mode: int
355 @param mkdir_mode: Mode for newly created directories
359 return os.rename(old, new)
361 # In at least one use case of this function, the job queue, directory
362 # creation is very rare. Checking for the directory before renaming is not
364 if mkdir and err.errno == errno.ENOENT:
365 # Create directory and try again
366 dirname = os.path.dirname(new)
368 os.makedirs(dirname, mode=mkdir_mode)
370 # Ignore EEXIST. This is only handled in os.makedirs as included in
371 # Python 2.5 and above.
372 if err.errno != errno.EEXIST or not os.path.exists(dirname):
375 return os.rename(old, new)
380 def ResetTempfileModule():
381 """Resets the random name generator of the tempfile module.
383 This function should be called after C{os.fork} in the child process to
384 ensure it creates a newly seeded random generator. Otherwise it would
385 generate the same random parts as the parent process. If several processes
386 race for the creation of a temporary file, this could lead to one not getting
390 # pylint: disable-msg=W0212
391 if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
392 tempfile._once_lock.acquire()
394 # Reset random name generator
395 tempfile._name_sequence = None
397 tempfile._once_lock.release()
399 logging.critical("The tempfile module misses at least one of the"
400 " '_once_lock' and '_name_sequence' attributes")
403 def _FingerprintFile(filename):
404 """Compute the fingerprint of a file.
406 If the file does not exist, a None will be returned
410 @param filename: the filename to checksum
412 @return: the hex digest of the sha checksum of the contents
416 if not (os.path.exists(filename) and os.path.isfile(filename)):
429 return fp.hexdigest()
432 def FingerprintFiles(files):
433 """Compute fingerprints for a list of files.
436 @param files: the list of filename to fingerprint
438 @return: a dictionary filename: fingerprint, holding only
444 for filename in files:
445 cksum = _FingerprintFile(filename)
447 ret[filename] = cksum
452 def ForceDictType(target, key_types, allowed_values=None):
453 """Force the values of a dict to have certain types.
456 @param target: the dict to update
457 @type key_types: dict
458 @param key_types: dict mapping target dict keys to types
459 in constants.ENFORCEABLE_TYPES
460 @type allowed_values: list
461 @keyword allowed_values: list of specially allowed values
464 if allowed_values is None:
467 if not isinstance(target, dict):
468 msg = "Expected dictionary, got '%s'" % target
469 raise errors.TypeEnforcementError(msg)
472 if key not in key_types:
473 msg = "Unknown key '%s'" % key
474 raise errors.TypeEnforcementError(msg)
476 if target[key] in allowed_values:
479 ktype = key_types[key]
480 if ktype not in constants.ENFORCEABLE_TYPES:
481 msg = "'%s' has non-enforceable type %s" % (key, ktype)
482 raise errors.ProgrammerError(msg)
484 if ktype == constants.VTYPE_STRING:
485 if not isinstance(target[key], basestring):
486 if isinstance(target[key], bool) and not target[key]:
489 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
490 raise errors.TypeEnforcementError(msg)
491 elif ktype == constants.VTYPE_BOOL:
492 if isinstance(target[key], basestring) and target[key]:
493 if target[key].lower() == constants.VALUE_FALSE:
495 elif target[key].lower() == constants.VALUE_TRUE:
498 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
499 raise errors.TypeEnforcementError(msg)
504 elif ktype == constants.VTYPE_SIZE:
506 target[key] = ParseUnit(target[key])
507 except errors.UnitParseError, err:
508 msg = "'%s' (value %s) is not a valid size. error: %s" % \
509 (key, target[key], err)
510 raise errors.TypeEnforcementError(msg)
511 elif ktype == constants.VTYPE_INT:
513 target[key] = int(target[key])
514 except (ValueError, TypeError):
515 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
516 raise errors.TypeEnforcementError(msg)
519 def IsProcessAlive(pid):
520 """Check if a given pid exists on the system.
522 @note: zombie status is not handled, so zombie processes
523 will be returned as alive
525 @param pid: the process ID to check
527 @return: True if the process exists
534 os.stat("/proc/%d/status" % pid)
536 except EnvironmentError, err:
537 if err.errno in (errno.ENOENT, errno.ENOTDIR):
542 def ReadPidFile(pidfile):
543 """Read a pid from a file.
545 @type pidfile: string
546 @param pidfile: path to the file containing the pid
548 @return: The process id, if the file exists and contains a valid PID,
553 raw_data = ReadFile(pidfile)
554 except EnvironmentError, err:
555 if err.errno != errno.ENOENT:
556 logging.exception("Can't read pid file")
561 except (TypeError, ValueError), err:
562 logging.info("Can't parse pid file contents", exc_info=True)
568 def MatchNameComponent(key, name_list, case_sensitive=True):
569 """Try to match a name against a list.
571 This function will try to match a name like test1 against a list
572 like C{['test1.example.com', 'test2.example.com', ...]}. Against
573 this list, I{'test1'} as well as I{'test1.example'} will match, but
574 not I{'test1.ex'}. A multiple match will be considered as no match
575 at all (e.g. I{'test1'} against C{['test1.example.com',
576 'test1.example.org']}), except when the key fully matches an entry
577 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
580 @param key: the name to be searched
581 @type name_list: list
582 @param name_list: the list of strings against which to search the key
583 @type case_sensitive: boolean
584 @param case_sensitive: whether to provide a case-sensitive match
587 @return: None if there is no match I{or} if there are multiple matches,
588 otherwise the element from the list which matches
595 if not case_sensitive:
596 re_flags |= re.IGNORECASE
598 mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
601 for name in name_list:
602 if mo.match(name) is not None:
603 names_filtered.append(name)
604 if not case_sensitive and key == name.upper():
605 string_matches.append(name)
607 if len(string_matches) == 1:
608 return string_matches[0]
609 if len(names_filtered) == 1:
610 return names_filtered[0]
615 """Class implementing resolver and hostname functionality
618 def __init__(self, name=None):
619 """Initialize the host name object.
621 If the name argument is not passed, it will use this system's
626 name = self.SysName()
629 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
630 self.ip = self.ipaddrs[0]
633 """Returns the hostname without domain.
636 return self.name.split('.')[0]
640 """Return the current system's name.
642 This is simply a wrapper over C{socket.gethostname()}.
645 return socket.gethostname()
648 def LookupHostname(hostname):
652 @param hostname: hostname to look up
655 @return: a tuple (name, aliases, ipaddrs) as returned by
656 C{socket.gethostbyname_ex}
657 @raise errors.ResolverError: in case of errors in resolving
661 result = socket.gethostbyname_ex(hostname)
662 except socket.gaierror, err:
663 # hostname not found in DNS
664 raise errors.ResolverError(hostname, err.args[0], err.args[1])
669 def GetHostInfo(name=None):
670 """Lookup host name and raise an OpPrereqError for failures"""
673 return HostInfo(name)
674 except errors.ResolverError, err:
675 raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
676 (err[0], err[2]), errors.ECODE_RESOLVER)
679 def ListVolumeGroups():
680 """List volume groups and their size
684 Dictionary with keys volume name and values
685 the size of the volume
688 command = "vgs --noheadings --units m --nosuffix -o name,size"
689 result = RunCmd(command)
694 for line in result.stdout.splitlines():
696 name, size = line.split()
697 size = int(float(size))
698 except (IndexError, ValueError), err:
699 logging.error("Invalid output from vgs (%s): %s", err, line)
707 def BridgeExists(bridge):
708 """Check whether the given bridge exists in the system
711 @param bridge: the bridge name to check
713 @return: True if it does
716 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
719 def NiceSort(name_list):
720 """Sort a list of strings based on digit and non-digit groupings.
722 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
723 will sort the list in the logical order C{['a1', 'a2', 'a10',
726 The sort algorithm breaks each name in groups of either only-digits
727 or no-digits. Only the first eight such groups are considered, and
728 after that we just use what's left of the string.
730 @type name_list: list
731 @param name_list: the names to be sorted
733 @return: a copy of the name list sorted with our algorithm
736 _SORTER_BASE = "(\D+|\d+)"
737 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
738 _SORTER_BASE, _SORTER_BASE,
739 _SORTER_BASE, _SORTER_BASE,
740 _SORTER_BASE, _SORTER_BASE)
741 _SORTER_RE = re.compile(_SORTER_FULL)
742 _SORTER_NODIGIT = re.compile("^\D*$")
744 """Attempts to convert a variable to integer."""
745 if val is None or _SORTER_NODIGIT.match(val):
750 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
751 for name in name_list]
753 return [tup[1] for tup in to_sort]
756 def TryConvert(fn, val):
757 """Try to convert a value ignoring errors.
759 This function tries to apply function I{fn} to I{val}. If no
760 C{ValueError} or C{TypeError} exceptions are raised, it will return
761 the result, else it will return the original value. Any other
762 exceptions are propagated to the caller.
765 @param fn: function to apply to the value
766 @param val: the value to be converted
767 @return: The converted value if the conversion was successful,
768 otherwise the original value.
773 except (ValueError, TypeError):
779 """Verifies the syntax of an IPv4 address.
781 This function checks if the IPv4 address passes is valid or not based
782 on syntax (not IP range, class calculations, etc.).
785 @param ip: the address to be checked
786 @rtype: a regular expression match object
787 @return: a regular expression match object, or None if the
791 unit = "(0|[1-9]\d{0,2})"
792 #TODO: convert and return only boolean
793 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
796 def IsValidShellParam(word):
797 """Verifies is the given word is safe from the shell's p.o.v.
799 This means that we can pass this to a command via the shell and be
800 sure that it doesn't alter the command line and is passed as such to
803 Note that we are overly restrictive here, in order to be on the safe
807 @param word: the word to check
809 @return: True if the word is 'safe'
812 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
815 def BuildShellCmd(template, *args):
816 """Build a safe shell command line from the given arguments.
818 This function will check all arguments in the args list so that they
819 are valid shell parameters (i.e. they don't contain shell
820 metacharacters). If everything is ok, it will return the result of
824 @param template: the string holding the template for the
827 @return: the expanded command line
831 if not IsValidShellParam(word):
832 raise errors.ProgrammerError("Shell argument '%s' contains"
833 " invalid characters" % word)
834 return template % args
837 def FormatUnit(value, units):
838 """Formats an incoming number of MiB with the appropriate unit.
841 @param value: integer representing the value in MiB (1048576)
843 @param units: the type of formatting we should do:
844 - 'h' for automatic scaling
849 @return: the formatted value (with suffix)
852 if units not in ('m', 'g', 't', 'h'):
853 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
857 if units == 'm' or (units == 'h' and value < 1024):
860 return "%d%s" % (round(value, 0), suffix)
862 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
865 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
870 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
873 def ParseUnit(input_string):
874 """Tries to extract number and scale from the given string.
876 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
877 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
878 is always an int in MiB.
881 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
883 raise errors.UnitParseError("Invalid format")
885 value = float(m.groups()[0])
889 lcunit = unit.lower()
893 if lcunit in ('m', 'mb', 'mib'):
894 # Value already in MiB
897 elif lcunit in ('g', 'gb', 'gib'):
900 elif lcunit in ('t', 'tb', 'tib'):
904 raise errors.UnitParseError("Unknown unit: %s" % unit)
906 # Make sure we round up
907 if int(value) < value:
910 # Round up to the next multiple of 4
913 value += 4 - value % 4
918 def AddAuthorizedKey(file_name, key):
919 """Adds an SSH public key to an authorized_keys file.
922 @param file_name: path to authorized_keys file
924 @param key: string containing key
927 key_fields = key.split()
929 f = open(file_name, 'a+')
933 # Ignore whitespace changes
934 if line.split() == key_fields:
936 nl = line.endswith('\n')
940 f.write(key.rstrip('\r\n'))
947 def RemoveAuthorizedKey(file_name, key):
948 """Removes an SSH public key from an authorized_keys file.
951 @param file_name: path to authorized_keys file
953 @param key: string containing key
956 key_fields = key.split()
958 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
960 out = os.fdopen(fd, 'w')
962 f = open(file_name, 'r')
965 # Ignore whitespace changes while comparing lines
966 if line.split() != key_fields:
970 os.rename(tmpname, file_name)
980 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
981 """Sets the name of an IP address and hostname in /etc/hosts.
984 @param file_name: path to the file to modify (usually C{/etc/hosts})
986 @param ip: the IP address
988 @param hostname: the hostname to be added
990 @param aliases: the list of aliases to add for the hostname
993 # FIXME: use WriteFile + fn rather than duplicating its efforts
994 # Ensure aliases are unique
995 aliases = UniqueSequence([hostname] + aliases)[1:]
997 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
999 out = os.fdopen(fd, 'w')
1001 f = open(file_name, 'r')
1004 fields = line.split()
1005 if fields and not fields[0].startswith('#') and ip == fields[0]:
1009 out.write("%s\t%s" % (ip, hostname))
1011 out.write(" %s" % ' '.join(aliases))
1016 os.chmod(tmpname, 0644)
1017 os.rename(tmpname, file_name)
1027 def AddHostToEtcHosts(hostname):
1028 """Wrapper around SetEtcHostsEntry.
1031 @param hostname: a hostname that will be resolved and added to
1032 L{constants.ETC_HOSTS}
1035 hi = HostInfo(name=hostname)
1036 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1039 def RemoveEtcHostsEntry(file_name, hostname):
1040 """Removes a hostname from /etc/hosts.
1042 IP addresses without names are removed from the file.
1044 @type file_name: str
1045 @param file_name: path to the file to modify (usually C{/etc/hosts})
1047 @param hostname: the hostname to be removed
1050 # FIXME: use WriteFile + fn rather than duplicating its efforts
1051 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1053 out = os.fdopen(fd, 'w')
1055 f = open(file_name, 'r')
1058 fields = line.split()
1059 if len(fields) > 1 and not fields[0].startswith('#'):
1061 if hostname in names:
1062 while hostname in names:
1063 names.remove(hostname)
1065 out.write("%s %s\n" % (fields[0], ' '.join(names)))
1072 os.chmod(tmpname, 0644)
1073 os.rename(tmpname, file_name)
1083 def RemoveHostFromEtcHosts(hostname):
1084 """Wrapper around RemoveEtcHostsEntry.
1087 @param hostname: hostname that will be resolved and its
1088 full and shot name will be removed from
1089 L{constants.ETC_HOSTS}
1092 hi = HostInfo(name=hostname)
1093 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1094 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1097 def CreateBackup(file_name):
1098 """Creates a backup of a file.
1100 @type file_name: str
1101 @param file_name: file to be backed up
1103 @return: the path to the newly created backup
1104 @raise errors.ProgrammerError: for invalid file names
1107 if not os.path.isfile(file_name):
1108 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1111 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1112 dir_name = os.path.dirname(file_name)
1114 fsrc = open(file_name, 'rb')
1116 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1117 fdst = os.fdopen(fd, 'wb')
1119 shutil.copyfileobj(fsrc, fdst)
1128 def ShellQuote(value):
1129 """Quotes shell argument according to POSIX.
1132 @param value: the argument to be quoted
1134 @return: the quoted value
1137 if _re_shell_unquoted.match(value):
1140 return "'%s'" % value.replace("'", "'\\''")
1143 def ShellQuoteArgs(args):
1144 """Quotes a list of shell arguments.
1147 @param args: list of arguments to be quoted
1149 @return: the quoted arguments concatenated with spaces
1152 return ' '.join([ShellQuote(i) for i in args])
1155 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1156 """Simple ping implementation using TCP connect(2).
1158 Check if the given IP is reachable by doing attempting a TCP connect
1162 @param target: the IP or hostname to ping
1164 @param port: the port to connect to
1166 @param timeout: the timeout on the connection attempt
1167 @type live_port_needed: boolean
1168 @param live_port_needed: whether a closed port will cause the
1169 function to return failure, as if there was a timeout
1170 @type source: str or None
1171 @param source: if specified, will cause the connect to be made
1172 from this specific source address; failures to bind other
1173 than C{EADDRNOTAVAIL} will be ignored
1176 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1180 if source is not None:
1182 sock.bind((source, 0))
1183 except socket.error, (errcode, _):
1184 if errcode == errno.EADDRNOTAVAIL:
1187 sock.settimeout(timeout)
1190 sock.connect((target, port))
1193 except socket.timeout:
1195 except socket.error, (errcode, _):
1196 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1201 def OwnIpAddress(address):
1202 """Check if the current host has the the given IP address.
1204 Currently this is done by TCP-pinging the address from the loopback
1207 @type address: string
1208 @param address: the address to check
1210 @return: True if we own the address
1213 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1214 source=constants.LOCALHOST_IP_ADDRESS)
1217 def ListVisibleFiles(path):
1218 """Returns a list of visible files in a directory.
1221 @param path: the directory to enumerate
1223 @return: the list of all files not starting with a dot
1226 files = [i for i in os.listdir(path) if not i.startswith(".")]
1231 def GetHomeDir(user, default=None):
1232 """Try to get the homedir of the given user.
1234 The user can be passed either as a string (denoting the name) or as
1235 an integer (denoting the user id). If the user is not found, the
1236 'default' argument is returned, which defaults to None.
1240 if isinstance(user, basestring):
1241 result = pwd.getpwnam(user)
1242 elif isinstance(user, (int, long)):
1243 result = pwd.getpwuid(user)
1245 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1249 return result.pw_dir
1253 """Returns a random UUID.
1255 @note: This is a Linux-specific method as it uses the /proc
1260 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1263 def GenerateSecret(numbytes=20):
1264 """Generates a random secret.
1266 This will generate a pseudo-random secret returning an hex string
1267 (so that it can be used where an ASCII string is needed).
1269 @param numbytes: the number of bytes which will be represented by the returned
1270 string (defaulting to 20, the length of a SHA1 hash)
1272 @return: an hex representation of the pseudo-random sequence
1275 return os.urandom(numbytes).encode('hex')
1278 def EnsureDirs(dirs):
1279 """Make required directories, if they don't exist.
1281 @param dirs: list of tuples (dir_name, dir_mode)
1282 @type dirs: list of (string, integer)
1285 for dir_name, dir_mode in dirs:
1287 os.mkdir(dir_name, dir_mode)
1288 except EnvironmentError, err:
1289 if err.errno != errno.EEXIST:
1290 raise errors.GenericError("Cannot create needed directory"
1291 " '%s': %s" % (dir_name, err))
1292 if not os.path.isdir(dir_name):
1293 raise errors.GenericError("%s is not a directory" % dir_name)
1296 def ReadFile(file_name, size=-1):
1300 @param size: Read at most size bytes (if negative, entire file)
1302 @return: the (possibly partial) content of the file
1305 f = open(file_name, "r")
1312 def WriteFile(file_name, fn=None, data=None,
1313 mode=None, uid=-1, gid=-1,
1314 atime=None, mtime=None, close=True,
1315 dry_run=False, backup=False,
1316 prewrite=None, postwrite=None):
1317 """(Over)write a file atomically.
1319 The file_name and either fn (a function taking one argument, the
1320 file descriptor, and which should write the data to it) or data (the
1321 contents of the file) must be passed. The other arguments are
1322 optional and allow setting the file mode, owner and group, and the
1323 mtime/atime of the file.
1325 If the function doesn't raise an exception, it has succeeded and the
1326 target file has the new contents. If the function has raised an
1327 exception, an existing target file should be unmodified and the
1328 temporary file should be removed.
1330 @type file_name: str
1331 @param file_name: the target filename
1333 @param fn: content writing function, called with
1334 file descriptor as parameter
1336 @param data: contents of the file
1338 @param mode: file mode
1340 @param uid: the owner of the file
1342 @param gid: the group of the file
1344 @param atime: a custom access time to be set on the file
1346 @param mtime: a custom modification time to be set on the file
1347 @type close: boolean
1348 @param close: whether to close file after writing it
1349 @type prewrite: callable
1350 @param prewrite: function to be called before writing content
1351 @type postwrite: callable
1352 @param postwrite: function to be called after writing content
1355 @return: None if the 'close' parameter evaluates to True,
1356 otherwise the file descriptor
1358 @raise errors.ProgrammerError: if any of the arguments are not valid
1361 if not os.path.isabs(file_name):
1362 raise errors.ProgrammerError("Path passed to WriteFile is not"
1363 " absolute: '%s'" % file_name)
1365 if [fn, data].count(None) != 1:
1366 raise errors.ProgrammerError("fn or data required")
1368 if [atime, mtime].count(None) == 1:
1369 raise errors.ProgrammerError("Both atime and mtime must be either"
1372 if backup and not dry_run and os.path.isfile(file_name):
1373 CreateBackup(file_name)
1375 dir_name, base_name = os.path.split(file_name)
1376 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1378 # here we need to make sure we remove the temp file, if any error
1379 # leaves it in place
1381 if uid != -1 or gid != -1:
1382 os.chown(new_name, uid, gid)
1384 os.chmod(new_name, mode)
1385 if callable(prewrite):
1387 if data is not None:
1391 if callable(postwrite):
1394 if atime is not None and mtime is not None:
1395 os.utime(new_name, (atime, mtime))
1397 os.rename(new_name, file_name)
1406 RemoveFile(new_name)
1411 def FirstFree(seq, base=0):
1412 """Returns the first non-existing integer from seq.
1414 The seq argument should be a sorted list of positive integers. The
1415 first time the index of an element is smaller than the element
1416 value, the index will be returned.
1418 The base argument is used to start at a different offset,
1419 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1421 Example: C{[0, 1, 3]} will return I{2}.
1424 @param seq: the sequence to be analyzed.
1426 @param base: use this value as the base index of the sequence
1428 @return: the first non-used index in the sequence
1431 for idx, elem in enumerate(seq):
1432 assert elem >= base, "Passed element is higher than base offset"
1433 if elem > idx + base:
1439 def all(seq, pred=bool): # pylint: disable-msg=W0622
1440 "Returns True if pred(x) is True for every element in the iterable"
1441 for _ in itertools.ifilterfalse(pred, seq):
1446 def any(seq, pred=bool): # pylint: disable-msg=W0622
1447 "Returns True if pred(x) is True for at least one element in the iterable"
1448 for _ in itertools.ifilter(pred, seq):
1453 def UniqueSequence(seq):
1454 """Returns a list with unique elements.
1456 Element order is preserved.
1459 @param seq: the sequence with the source elements
1461 @return: list of unique elements from seq
1465 return [i for i in seq if i not in seen and not seen.add(i)]
1468 def NormalizeAndValidateMac(mac):
1469 """Normalizes and check if a MAC address is valid.
1471 Checks whether the supplied MAC address is formally correct, only
1472 accepts colon separated format. Normalize it to all lower.
1475 @param mac: the MAC to be validated
1477 @return: returns the normalized and validated MAC.
1479 @raise errors.OpPrereqError: If the MAC isn't valid
1482 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1483 if not mac_check.match(mac):
1484 raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1485 mac, errors.ECODE_INVAL)
1490 def TestDelay(duration):
1491 """Sleep for a fixed amount of time.
1493 @type duration: float
1494 @param duration: the sleep duration
1496 @return: False for negative value, True otherwise
1500 return False, "Invalid sleep duration"
1501 time.sleep(duration)
1505 def _CloseFDNoErr(fd, retries=5):
1506 """Close a file descriptor ignoring errors.
1509 @param fd: the file descriptor
1511 @param retries: how many retries to make, in case we get any
1512 other error than EBADF
1517 except OSError, err:
1518 if err.errno != errno.EBADF:
1520 _CloseFDNoErr(fd, retries - 1)
1521 # else either it's closed already or we're out of retries, so we
1522 # ignore this and go on
1525 def CloseFDs(noclose_fds=None):
1526 """Close file descriptors.
1528 This closes all file descriptors above 2 (i.e. except
1531 @type noclose_fds: list or None
1532 @param noclose_fds: if given, it denotes a list of file descriptor
1533 that should not be closed
1536 # Default maximum for the number of available file descriptors.
1537 if 'SC_OPEN_MAX' in os.sysconf_names:
1539 MAXFD = os.sysconf('SC_OPEN_MAX')
1546 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1547 if (maxfd == resource.RLIM_INFINITY):
1550 # Iterate through and close all file descriptors (except the standard ones)
1551 for fd in range(3, maxfd):
1552 if noclose_fds and fd in noclose_fds:
1557 def Daemonize(logfile):
1558 """Daemonize the current process.
1560 This detaches the current process from the controlling terminal and
1561 runs it in the background as a daemon.
1564 @param logfile: the logfile to which we should redirect stdout/stderr
1566 @return: the value zero
1569 # pylint: disable-msg=W0212
1570 # yes, we really want os._exit
1576 if (pid == 0): # The first child.
1579 pid = os.fork() # Fork a second child.
1580 if (pid == 0): # The second child.
1584 # exit() or _exit()? See below.
1585 os._exit(0) # Exit parent (the first child) of the second child.
1587 os._exit(0) # Exit parent of the first child.
1591 i = os.open("/dev/null", os.O_RDONLY) # stdin
1592 assert i == 0, "Can't close/reopen stdin"
1593 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1594 assert i == 1, "Can't close/reopen stdout"
1595 # Duplicate standard output to standard error.
1600 def DaemonPidFileName(name):
1601 """Compute a ganeti pid file absolute path
1604 @param name: the daemon name
1606 @return: the full path to the pidfile corresponding to the given
1610 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
1613 def EnsureDaemon(name):
1614 """Check for and start daemon if not alive.
1617 result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
1619 logging.error("Can't start daemon '%s', failure %s, output: %s",
1620 name, result.fail_reason, result.output)
1626 def WritePidFile(name):
1627 """Write the current process pidfile.
1629 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1632 @param name: the daemon name to use
1633 @raise errors.GenericError: if the pid file already exists and
1634 points to a live process
1638 pidfilename = DaemonPidFileName(name)
1639 if IsProcessAlive(ReadPidFile(pidfilename)):
1640 raise errors.GenericError("%s contains a live process" % pidfilename)
1642 WriteFile(pidfilename, data="%d\n" % pid)
1645 def RemovePidFile(name):
1646 """Remove the current process pidfile.
1648 Any errors are ignored.
1651 @param name: the daemon name used to derive the pidfile name
1654 pidfilename = DaemonPidFileName(name)
1655 # TODO: we could check here that the file contains our pid
1657 RemoveFile(pidfilename)
1658 except: # pylint: disable-msg=W0702
1662 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1664 """Kill a process given by its pid.
1667 @param pid: The PID to terminate.
1669 @param signal_: The signal to send, by default SIGTERM
1671 @param timeout: The timeout after which, if the process is still alive,
1672 a SIGKILL will be sent. If not positive, no such checking
1674 @type waitpid: boolean
1675 @param waitpid: If true, we should waitpid on this process after
1676 sending signals, since it's our own child and otherwise it
1677 would remain as zombie
1680 def _helper(pid, signal_, wait):
1681 """Simple helper to encapsulate the kill/waitpid sequence"""
1682 os.kill(pid, signal_)
1685 os.waitpid(pid, os.WNOHANG)
1690 # kill with pid=0 == suicide
1691 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1693 if not IsProcessAlive(pid):
1696 _helper(pid, signal_, waitpid)
1701 def _CheckProcess():
1702 if not IsProcessAlive(pid):
1706 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1716 # Wait up to $timeout seconds
1717 Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1718 except RetryTimeout:
1721 if IsProcessAlive(pid):
1722 # Kill process if it's still alive
1723 _helper(pid, signal.SIGKILL, waitpid)
1726 def FindFile(name, search_path, test=os.path.exists):
1727 """Look for a filesystem object in a given path.
1729 This is an abstract method to search for filesystem object (files,
1730 dirs) under a given search path.
1733 @param name: the name to look for
1734 @type search_path: str
1735 @param search_path: location to start at
1736 @type test: callable
1737 @param test: a function taking one argument that should return True
1738 if the a given object is valid; the default value is
1739 os.path.exists, causing only existing files to be returned
1741 @return: full path to the object if found, None otherwise
1744 # validate the filename mask
1745 if constants.EXT_PLUGIN_MASK.match(name) is None:
1746 logging.critical("Invalid value passed for external script name: '%s'",
1750 for dir_name in search_path:
1751 # FIXME: investigate switch to PathJoin
1752 item_name = os.path.sep.join([dir_name, name])
1753 # check the user test and that we're indeed resolving to the given
1755 if test(item_name) and os.path.basename(item_name) == name:
1760 def CheckVolumeGroupSize(vglist, vgname, minsize):
1761 """Checks if the volume group list is valid.
1763 The function will check if a given volume group is in the list of
1764 volume groups and has a minimum size.
1767 @param vglist: dictionary of volume group names and their size
1769 @param vgname: the volume group we should check
1771 @param minsize: the minimum size we accept
1773 @return: None for success, otherwise the error message
1776 vgsize = vglist.get(vgname, None)
1778 return "volume group '%s' missing" % vgname
1779 elif vgsize < minsize:
1780 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1781 (vgname, minsize, vgsize))
1785 def SplitTime(value):
1786 """Splits time as floating point number into a tuple.
1788 @param value: Time in seconds
1789 @type value: int or float
1790 @return: Tuple containing (seconds, microseconds)
1793 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1795 assert 0 <= seconds, \
1796 "Seconds must be larger than or equal to 0, but are %s" % seconds
1797 assert 0 <= microseconds <= 999999, \
1798 "Microseconds must be 0-999999, but are %s" % microseconds
1800 return (int(seconds), int(microseconds))
1803 def MergeTime(timetuple):
1804 """Merges a tuple into time as a floating point number.
1806 @param timetuple: Time as tuple, (seconds, microseconds)
1807 @type timetuple: tuple
1808 @return: Time as a floating point number expressed in seconds
1811 (seconds, microseconds) = timetuple
1813 assert 0 <= seconds, \
1814 "Seconds must be larger than or equal to 0, but are %s" % seconds
1815 assert 0 <= microseconds <= 999999, \
1816 "Microseconds must be 0-999999, but are %s" % microseconds
1818 return float(seconds) + (float(microseconds) * 0.000001)
1821 def GetDaemonPort(daemon_name):
1822 """Get the daemon port for this cluster.
1824 Note that this routine does not read a ganeti-specific file, but
1825 instead uses C{socket.getservbyname} to allow pre-customization of
1826 this parameter outside of Ganeti.
1828 @type daemon_name: string
1829 @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1833 if daemon_name not in constants.DAEMONS_PORTS:
1834 raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1836 (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1838 port = socket.getservbyname(daemon_name, proto)
1839 except socket.error:
1845 def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
1846 multithreaded=False, syslog=constants.SYSLOG_USAGE):
1847 """Configures the logging module.
1850 @param logfile: the filename to which we should log
1851 @type debug: integer
1852 @param debug: if greater than zero, enable debug messages, otherwise
1853 only those at C{INFO} and above level
1854 @type stderr_logging: boolean
1855 @param stderr_logging: whether we should also log to the standard error
1857 @param program: the name under which we should log messages
1858 @type multithreaded: boolean
1859 @param multithreaded: if True, will add the thread name to the log file
1860 @type syslog: string
1861 @param syslog: one of 'no', 'yes', 'only':
1862 - if no, syslog is not used
1863 - if yes, syslog is used (in addition to file-logging)
1864 - if only, only syslog is used
1865 @raise EnvironmentError: if we can't open the log file and
1866 syslog/stderr logging is disabled
1869 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1870 sft = program + "[%(process)d]:"
1872 fmt += "/%(threadName)s"
1873 sft += " (%(threadName)s)"
1875 fmt += " %(module)s:%(lineno)s"
1876 # no debug info for syslog loggers
1877 fmt += " %(levelname)s %(message)s"
1878 # yes, we do want the textual level, as remote syslog will probably
1879 # lose the error level, and it's easier to grep for it
1880 sft += " %(levelname)s %(message)s"
1881 formatter = logging.Formatter(fmt)
1882 sys_fmt = logging.Formatter(sft)
1884 root_logger = logging.getLogger("")
1885 root_logger.setLevel(logging.NOTSET)
1887 # Remove all previously setup handlers
1888 for handler in root_logger.handlers:
1890 root_logger.removeHandler(handler)
1893 stderr_handler = logging.StreamHandler()
1894 stderr_handler.setFormatter(formatter)
1896 stderr_handler.setLevel(logging.NOTSET)
1898 stderr_handler.setLevel(logging.CRITICAL)
1899 root_logger.addHandler(stderr_handler)
1901 if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
1902 facility = logging.handlers.SysLogHandler.LOG_DAEMON
1903 syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
1905 syslog_handler.setFormatter(sys_fmt)
1906 # Never enable debug over syslog
1907 syslog_handler.setLevel(logging.INFO)
1908 root_logger.addHandler(syslog_handler)
1910 if syslog != constants.SYSLOG_ONLY:
1911 # this can fail, if the logging directories are not setup or we have
1912 # a permisssion problem; in this case, it's best to log but ignore
1913 # the error if stderr_logging is True, and if false we re-raise the
1914 # exception since otherwise we could run but without any logs at all
1916 logfile_handler = logging.FileHandler(logfile)
1917 logfile_handler.setFormatter(formatter)
1919 logfile_handler.setLevel(logging.DEBUG)
1921 logfile_handler.setLevel(logging.INFO)
1922 root_logger.addHandler(logfile_handler)
1923 except EnvironmentError:
1924 if stderr_logging or syslog == constants.SYSLOG_YES:
1925 logging.exception("Failed to enable logging to file '%s'", logfile)
1927 # we need to re-raise the exception
1931 def IsNormAbsPath(path):
1932 """Check whether a path is absolute and also normalized
1934 This avoids things like /dir/../../other/path to be valid.
1937 return os.path.normpath(path) == path and os.path.isabs(path)
1940 def PathJoin(*args):
1941 """Safe-join a list of path components.
1944 - the first argument must be an absolute path
1945 - no component in the path must have backtracking (e.g. /../),
1946 since we check for normalization at the end
1948 @param args: the path components to be joined
1949 @raise ValueError: for invalid paths
1952 # ensure we're having at least one path passed in
1954 # ensure the first component is an absolute and normalized path name
1956 if not IsNormAbsPath(root):
1957 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
1958 result = os.path.join(*args)
1959 # ensure that the whole path is normalized
1960 if not IsNormAbsPath(result):
1961 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
1962 # check that we're still under the original prefix
1963 prefix = os.path.commonprefix([root, result])
1965 raise ValueError("Error: path joining resulted in different prefix"
1966 " (%s != %s)" % (prefix, root))
1970 def TailFile(fname, lines=20):
1971 """Return the last lines from a file.
1973 @note: this function will only read and parse the last 4KB of
1974 the file; if the lines are very long, it could be that less
1975 than the requested number of lines are returned
1977 @param fname: the file name
1979 @param lines: the (maximum) number of lines to return
1982 fd = open(fname, "r")
1986 pos = max(0, pos-4096)
1988 raw_data = fd.read()
1992 rows = raw_data.splitlines()
1993 return rows[-lines:]
1996 def SafeEncode(text):
1997 """Return a 'safe' version of a source string.
1999 This function mangles the input string and returns a version that
2000 should be safe to display/encode as ASCII. To this end, we first
2001 convert it to ASCII using the 'backslashreplace' encoding which
2002 should get rid of any non-ASCII chars, and then we process it
2003 through a loop copied from the string repr sources in the python; we
2004 don't use string_escape anymore since that escape single quotes and
2005 backslashes too, and that is too much; and that escaping is not
2006 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2008 @type text: str or unicode
2009 @param text: input data
2011 @return: a safe version of text
2014 if isinstance(text, unicode):
2015 # only if unicode; if str already, we handle it below
2016 text = text.encode('ascii', 'backslashreplace')
2026 elif c < 32 or c >= 127: # non-printable
2027 resu += "\\x%02x" % (c & 0xff)
2033 def UnescapeAndSplit(text, sep=","):
2034 """Split and unescape a string based on a given separator.
2036 This function splits a string based on a separator where the
2037 separator itself can be escape in order to be an element of the
2038 elements. The escaping rules are (assuming coma being the
2040 - a plain , separates the elements
2041 - a sequence \\\\, (double backslash plus comma) is handled as a
2042 backslash plus a separator comma
2043 - a sequence \, (backslash plus comma) is handled as a
2047 @param text: the string to split
2049 @param text: the separator
2051 @return: a list of strings
2054 # we split the list by sep (with no escaping at this stage)
2055 slist = text.split(sep)
2056 # next, we revisit the elements and if any of them ended with an odd
2057 # number of backslashes, then we join it with the next
2061 if e1.endswith("\\"):
2062 num_b = len(e1) - len(e1.rstrip("\\"))
2065 # here the backslashes remain (all), and will be reduced in
2067 rlist.append(e1 + sep + e2)
2070 # finally, replace backslash-something with something
2071 rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2075 def CommaJoin(names):
2076 """Nicely join a set of identifiers.
2078 @param names: set, list or tuple
2079 @return: a string with the formatted results
2082 return ", ".join([str(val) for val in names])
2085 def BytesToMebibyte(value):
2086 """Converts bytes to mebibytes.
2089 @param value: Value in bytes
2091 @return: Value in mebibytes
2094 return int(round(value / (1024.0 * 1024.0), 0))
2097 def CalculateDirectorySize(path):
2098 """Calculates the size of a directory recursively.
2101 @param path: Path to directory
2103 @return: Size in mebibytes
2108 for (curpath, _, files) in os.walk(path):
2109 for filename in files:
2110 st = os.lstat(PathJoin(curpath, filename))
2113 return BytesToMebibyte(size)
2116 def GetFilesystemStats(path):
2117 """Returns the total and free space on a filesystem.
2120 @param path: Path on filesystem to be examined
2122 @return: tuple of (Total space, Free space) in mebibytes
2125 st = os.statvfs(path)
2127 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2128 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2129 return (tsize, fsize)
2132 def RunInSeparateProcess(fn):
2133 """Runs a function in a separate process.
2135 Note: Only boolean return values are supported.
2138 @param fn: Function to be called
2139 @rtype: tuple of (int/None, int/None)
2140 @return: Exit code and signal number
2147 # In case the function uses temporary files
2148 ResetTempfileModule()
2151 result = int(bool(fn()))
2152 assert result in (0, 1)
2153 except: # pylint: disable-msg=W0702
2154 logging.exception("Error while calling function in separate process")
2155 # 0 and 1 are reserved for the return value
2158 os._exit(result) # pylint: disable-msg=W0212
2162 # Avoid zombies and check exit code
2163 (_, status) = os.waitpid(pid, 0)
2165 if os.WIFSIGNALED(status):
2167 signum = os.WTERMSIG(status)
2169 exitcode = os.WEXITSTATUS(status)
2172 if not (exitcode in (0, 1) and signum is None):
2173 raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2176 return bool(exitcode)
2179 def LockedMethod(fn):
2180 """Synchronized object access decorator.
2182 This decorator is intended to protect access to an object using the
2183 object's own lock which is hardcoded to '_lock'.
2186 def _LockDebug(*args, **kwargs):
2188 logging.debug(*args, **kwargs)
2190 def wrapper(self, *args, **kwargs):
2191 # pylint: disable-msg=W0212
2192 assert hasattr(self, '_lock')
2194 _LockDebug("Waiting for %s", lock)
2197 _LockDebug("Acquired %s", lock)
2198 result = fn(self, *args, **kwargs)
2200 _LockDebug("Releasing %s", lock)
2202 _LockDebug("Released %s", lock)
2208 """Locks a file using POSIX locks.
2211 @param fd: the file descriptor we need to lock
2215 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2216 except IOError, err:
2217 if err.errno == errno.EAGAIN:
2218 raise errors.LockError("File already locked")
2222 def FormatTime(val):
2223 """Formats a time value.
2225 @type val: float or None
2226 @param val: the timestamp as returned by time.time()
2227 @return: a string value or N/A if we don't have a valid timestamp
2230 if val is None or not isinstance(val, (int, float)):
2232 # these two codes works on Linux, but they are not guaranteed on all
2234 return time.strftime("%F %T", time.localtime(val))
2237 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2238 """Reads the watcher pause file.
2240 @type filename: string
2241 @param filename: Path to watcher pause file
2242 @type now: None, float or int
2243 @param now: Current time as Unix timestamp
2244 @type remove_after: int
2245 @param remove_after: Remove watcher pause file after specified amount of
2246 seconds past the pause end time
2253 value = ReadFile(filename)
2254 except IOError, err:
2255 if err.errno != errno.ENOENT:
2259 if value is not None:
2263 logging.warning(("Watcher pause file (%s) contains invalid value,"
2264 " removing it"), filename)
2265 RemoveFile(filename)
2268 if value is not None:
2269 # Remove file if it's outdated
2270 if now > (value + remove_after):
2271 RemoveFile(filename)
2280 class RetryTimeout(Exception):
2281 """Retry loop timed out.
2286 class RetryAgain(Exception):
2292 class _RetryDelayCalculator(object):
2293 """Calculator for increasing delays.
2303 def __init__(self, start, factor, limit):
2304 """Initializes this class.
2307 @param start: Initial delay
2309 @param factor: Factor for delay increase
2310 @type limit: float or None
2311 @param limit: Upper limit for delay or None for no limit
2315 assert factor >= 1.0
2316 assert limit is None or limit >= 0.0
2319 self._factor = factor
2325 """Returns current delay and calculates the next one.
2328 current = self._next
2330 # Update for next run
2331 if self._limit is None or self._next < self._limit:
2332 self._next = min(self._limit, self._next * self._factor)
2337 #: Special delay to specify whole remaining timeout
2338 RETRY_REMAINING_TIME = object()
2341 def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2342 _time_fn=time.time):
2343 """Call a function repeatedly until it succeeds.
2345 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2346 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2347 total of C{timeout} seconds, this function throws L{RetryTimeout}.
2349 C{delay} can be one of the following:
2350 - callable returning the delay length as a float
2351 - Tuple of (start, factor, limit)
2352 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2353 useful when overriding L{wait_fn} to wait for an external event)
2354 - A static delay as a number (int or float)
2357 @param fn: Function to be called
2358 @param delay: Either a callable (returning the delay), a tuple of (start,
2359 factor, limit) (see L{_RetryDelayCalculator}),
2360 L{RETRY_REMAINING_TIME} or a number (int or float)
2361 @type timeout: float
2362 @param timeout: Total timeout
2363 @type wait_fn: callable
2364 @param wait_fn: Waiting function
2365 @return: Return value of function
2369 assert callable(wait_fn)
2370 assert callable(_time_fn)
2375 end_time = _time_fn() + timeout
2378 # External function to calculate delay
2381 elif isinstance(delay, (tuple, list)):
2382 # Increasing delay with optional upper boundary
2383 (start, factor, limit) = delay
2384 calc_delay = _RetryDelayCalculator(start, factor, limit)
2386 elif delay is RETRY_REMAINING_TIME:
2387 # Always use the remaining time
2392 calc_delay = lambda: delay
2394 assert calc_delay is None or callable(calc_delay)
2398 # pylint: disable-msg=W0142
2403 remaining_time = end_time - _time_fn()
2405 if remaining_time < 0.0:
2406 raise RetryTimeout()
2408 assert remaining_time >= 0.0
2410 if calc_delay is None:
2411 wait_fn(remaining_time)
2413 current_delay = calc_delay()
2414 if current_delay > 0.0:
2415 wait_fn(current_delay)
2418 class FileLock(object):
2419 """Utility class for file locks.
2422 def __init__(self, filename):
2423 """Constructor for FileLock.
2425 This will open the file denoted by the I{filename} argument.
2428 @param filename: path to the file to be locked
2431 self.filename = filename
2432 self.fd = open(self.filename, "w")
2438 """Close the file and release the lock.
2441 if hasattr(self, "fd") and self.fd:
2445 def _flock(self, flag, blocking, timeout, errmsg):
2446 """Wrapper for fcntl.flock.
2449 @param flag: operation flag
2450 @type blocking: bool
2451 @param blocking: whether the operation should be done in blocking mode.
2452 @type timeout: None or float
2453 @param timeout: for how long the operation should be retried (implies
2455 @type errmsg: string
2456 @param errmsg: error message in case operation fails.
2459 assert self.fd, "Lock was closed"
2460 assert timeout is None or timeout >= 0, \
2461 "If specified, timeout must be positive"
2463 if timeout is not None:
2464 flag |= fcntl.LOCK_NB
2465 timeout_end = time.time() + timeout
2467 # Blocking doesn't have effect with timeout
2469 flag |= fcntl.LOCK_NB
2472 # TODO: Convert to utils.Retry
2477 fcntl.flock(self.fd, flag)
2479 except IOError, err:
2480 if err.errno in (errno.EAGAIN, ):
2481 if timeout_end is not None and time.time() < timeout_end:
2482 # Wait before trying again
2483 time.sleep(max(0.1, min(1.0, timeout)))
2485 raise errors.LockError(errmsg)
2487 logging.exception("fcntl.flock failed")
2490 def Exclusive(self, blocking=False, timeout=None):
2491 """Locks the file in exclusive mode.
2493 @type blocking: boolean
2494 @param blocking: whether to block and wait until we
2495 can lock the file or return immediately
2496 @type timeout: int or None
2497 @param timeout: if not None, the duration to wait for the lock
2501 self._flock(fcntl.LOCK_EX, blocking, timeout,
2502 "Failed to lock %s in exclusive mode" % self.filename)
2504 def Shared(self, blocking=False, timeout=None):
2505 """Locks the file in shared mode.
2507 @type blocking: boolean
2508 @param blocking: whether to block and wait until we
2509 can lock the file or return immediately
2510 @type timeout: int or None
2511 @param timeout: if not None, the duration to wait for the lock
2515 self._flock(fcntl.LOCK_SH, blocking, timeout,
2516 "Failed to lock %s in shared mode" % self.filename)
2518 def Unlock(self, blocking=True, timeout=None):
2519 """Unlocks the file.
2521 According to C{flock(2)}, unlocking can also be a nonblocking
2524 To make a non-blocking request, include LOCK_NB with any of the above
2527 @type blocking: boolean
2528 @param blocking: whether to block and wait until we
2529 can lock the file or return immediately
2530 @type timeout: int or None
2531 @param timeout: if not None, the duration to wait for the lock
2535 self._flock(fcntl.LOCK_UN, blocking, timeout,
2536 "Failed to unlock %s" % self.filename)
2539 def SignalHandled(signums):
2540 """Signal Handled decoration.
2542 This special decorator installs a signal handler and then calls the target
2543 function. The function must accept a 'signal_handlers' keyword argument,
2544 which will contain a dict indexed by signal number, with SignalHandler
2547 The decorator can be safely stacked with iself, to handle multiple signals
2548 with different handlers.
2551 @param signums: signals to intercept
2555 def sig_function(*args, **kwargs):
2556 assert 'signal_handlers' not in kwargs or \
2557 kwargs['signal_handlers'] is None or \
2558 isinstance(kwargs['signal_handlers'], dict), \
2559 "Wrong signal_handlers parameter in original function call"
2560 if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2561 signal_handlers = kwargs['signal_handlers']
2563 signal_handlers = {}
2564 kwargs['signal_handlers'] = signal_handlers
2565 sighandler = SignalHandler(signums)
2568 signal_handlers[sig] = sighandler
2569 return fn(*args, **kwargs)
2576 class SignalHandler(object):
2577 """Generic signal handler class.
2579 It automatically restores the original handler when deconstructed or
2580 when L{Reset} is called. You can either pass your own handler
2581 function in or query the L{called} attribute to detect whether the
2585 @ivar signum: the signals we handle
2586 @type called: boolean
2587 @ivar called: tracks whether any of the signals have been raised
2590 def __init__(self, signum):
2591 """Constructs a new SignalHandler instance.
2593 @type signum: int or list of ints
2594 @param signum: Single signal number or set of signal numbers
2597 self.signum = set(signum)
2602 for signum in self.signum:
2604 prev_handler = signal.signal(signum, self._HandleSignal)
2606 self._previous[signum] = prev_handler
2608 # Restore previous handler
2609 signal.signal(signum, prev_handler)
2612 # Reset all handlers
2614 # Here we have a race condition: a handler may have already been called,
2615 # but there's not much we can do about it at this point.
2622 """Restore previous handler.
2624 This will reset all the signals to their previous handlers.
2627 for signum, prev_handler in self._previous.items():
2628 signal.signal(signum, prev_handler)
2629 # If successful, remove from dict
2630 del self._previous[signum]
2633 """Unsets the L{called} flag.
2635 This function can be used in case a signal may arrive several times.
2640 # we don't care about arguments, but we leave them named for the future
2641 def _HandleSignal(self, signum, frame): # pylint: disable-msg=W0613
2642 """Actual signal handling function.
2645 # This is not nice and not absolutely atomic, but it appears to be the only
2646 # solution in Python -- there are no atomic types.
2650 class FieldSet(object):
2651 """A simple field set.
2653 Among the features are:
2654 - checking if a string is among a list of static string or regex objects
2655 - checking if a whole list of string matches
2656 - returning the matching groups from a regex match
2658 Internally, all fields are held as regular expression objects.
2661 def __init__(self, *items):
2662 self.items = [re.compile("^%s$" % value) for value in items]
2664 def Extend(self, other_set):
2665 """Extend the field set with the items from another one"""
2666 self.items.extend(other_set.items)
2668 def Matches(self, field):
2669 """Checks if a field matches the current set
2672 @param field: the string to match
2673 @return: either None or a regular expression match object
2676 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2680 def NonMatching(self, items):
2681 """Returns the list of fields not matching the current set
2684 @param items: the list of fields to check
2686 @return: list of non-matching fields
2689 return [val for val in items if not self.Matches(val)]