4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
46 from cStringIO import StringIO
49 from hashlib import sha1
54 from ganeti import errors
55 from ganeti import constants
59 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
63 #: when set to True, L{RunCmd} is disabled
66 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
69 class RunResult(object):
70 """Holds the result of running external programs.
73 @ivar exit_code: the exit code of the program, or None (if the program
75 @type signal: int or None
76 @ivar signal: the signal that caused the program to finish, or None
77 (if the program wasn't terminated by a signal)
79 @ivar stdout: the standard output of the program
81 @ivar stderr: the standard error of the program
83 @ivar failed: True in case the program was
84 terminated by a signal or exited with a non-zero exit code
85 @ivar fail_reason: a string detailing the termination reason
88 __slots__ = ["exit_code", "signal", "stdout", "stderr",
89 "failed", "fail_reason", "cmd"]
92 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
94 self.exit_code = exit_code
98 self.failed = (signal_ is not None or exit_code != 0)
100 if self.signal is not None:
101 self.fail_reason = "terminated by signal %s" % self.signal
102 elif self.exit_code is not None:
103 self.fail_reason = "exited with exit code %s" % self.exit_code
105 self.fail_reason = "unable to determine termination reason"
108 logging.debug("Command '%s' failed (%s); output: %s",
109 self.cmd, self.fail_reason, self.output)
111 def _GetOutput(self):
112 """Returns the combined stdout and stderr for easier usage.
115 return self.stdout + self.stderr
117 output = property(_GetOutput, None, None, "Return full output")
120 def RunCmd(cmd, env=None, output=None, cwd='/'):
121 """Execute a (shell) command.
123 The command should not read from its standard input, as it will be
126 @type cmd: string or list
127 @param cmd: Command to run
129 @param env: Additional environment
131 @param output: if desired, the output of the command can be
132 saved in a file instead of the RunResult instance; this
133 parameter denotes the file name (if not None)
135 @param cwd: if specified, will be used as the working
136 directory for the command; the default will be /
138 @return: RunResult instance
139 @raise errors.ProgrammerError: if we call this when forks are disabled
143 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
145 if isinstance(cmd, list):
146 cmd = [str(val) for val in cmd]
147 strcmd = " ".join(cmd)
152 logging.debug("RunCmd '%s'", strcmd)
154 cmd_env = os.environ.copy()
155 cmd_env["LC_ALL"] = "C"
161 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
163 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
166 if err.errno == errno.ENOENT:
167 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
179 return RunResult(exitcode, signal_, out, err, strcmd)
182 def _RunCmdPipe(cmd, env, via_shell, cwd):
183 """Run a command and return its output.
185 @type cmd: string or list
186 @param cmd: Command to run
188 @param env: The environment to use
189 @type via_shell: bool
190 @param via_shell: if we should run via the shell
192 @param cwd: the working directory for the program
194 @return: (out, err, status)
197 poller = select.poll()
198 child = subprocess.Popen(cmd, shell=via_shell,
199 stderr=subprocess.PIPE,
200 stdout=subprocess.PIPE,
201 stdin=subprocess.PIPE,
202 close_fds=True, env=env,
206 poller.register(child.stdout, select.POLLIN)
207 poller.register(child.stderr, select.POLLIN)
211 child.stdout.fileno(): (out, child.stdout),
212 child.stderr.fileno(): (err, child.stderr),
215 status = fcntl.fcntl(fd, fcntl.F_GETFL)
216 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
220 pollresult = poller.poll()
221 except EnvironmentError, eerr:
222 if eerr.errno == errno.EINTR:
225 except select.error, serr:
226 if serr[0] == errno.EINTR:
230 for fd, event in pollresult:
231 if event & select.POLLIN or event & select.POLLPRI:
232 data = fdmap[fd][1].read()
233 # no data from read signifies EOF (the same as POLLHUP)
235 poller.unregister(fd)
238 fdmap[fd][0].write(data)
239 if (event & select.POLLNVAL or event & select.POLLHUP or
240 event & select.POLLERR):
241 poller.unregister(fd)
247 status = child.wait()
248 return out, err, status
251 def _RunCmdFile(cmd, env, via_shell, output, cwd):
252 """Run a command and save its output to a file.
254 @type cmd: string or list
255 @param cmd: Command to run
257 @param env: The environment to use
258 @type via_shell: bool
259 @param via_shell: if we should run via the shell
261 @param output: the filename in which to save the output
263 @param cwd: the working directory for the program
265 @return: the exit status
268 fh = open(output, "a")
270 child = subprocess.Popen(cmd, shell=via_shell,
271 stderr=subprocess.STDOUT,
273 stdin=subprocess.PIPE,
274 close_fds=True, env=env,
278 status = child.wait()
284 def RemoveFile(filename):
285 """Remove a file ignoring some errors.
287 Remove a file, ignoring non-existing ones or directories. Other
291 @param filename: the file to be removed
297 if err.errno not in (errno.ENOENT, errno.EISDIR):
301 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
305 @param old: Original path
309 @param mkdir: Whether to create target directory if it doesn't exist
310 @type mkdir_mode: int
311 @param mkdir_mode: Mode for newly created directories
315 return os.rename(old, new)
317 # In at least one use case of this function, the job queue, directory
318 # creation is very rare. Checking for the directory before renaming is not
320 if mkdir and err.errno == errno.ENOENT:
321 # Create directory and try again
322 os.makedirs(os.path.dirname(new), mkdir_mode)
323 return os.rename(old, new)
327 def _FingerprintFile(filename):
328 """Compute the fingerprint of a file.
330 If the file does not exist, a None will be returned
334 @param filename: the filename to checksum
336 @return: the hex digest of the sha checksum of the contents
340 if not (os.path.exists(filename) and os.path.isfile(filename)):
353 return fp.hexdigest()
356 def FingerprintFiles(files):
357 """Compute fingerprints for a list of files.
360 @param files: the list of filename to fingerprint
362 @return: a dictionary filename: fingerprint, holding only
368 for filename in files:
369 cksum = _FingerprintFile(filename)
371 ret[filename] = cksum
376 def ForceDictType(target, key_types, allowed_values=None):
377 """Force the values of a dict to have certain types.
380 @param target: the dict to update
381 @type key_types: dict
382 @param key_types: dict mapping target dict keys to types
383 in constants.ENFORCEABLE_TYPES
384 @type allowed_values: list
385 @keyword allowed_values: list of specially allowed values
388 if allowed_values is None:
391 if not isinstance(target, dict):
392 msg = "Expected dictionary, got '%s'" % target
393 raise errors.TypeEnforcementError(msg)
396 if key not in key_types:
397 msg = "Unknown key '%s'" % key
398 raise errors.TypeEnforcementError(msg)
400 if target[key] in allowed_values:
403 ktype = key_types[key]
404 if ktype not in constants.ENFORCEABLE_TYPES:
405 msg = "'%s' has non-enforceable type %s" % (key, ktype)
406 raise errors.ProgrammerError(msg)
408 if ktype == constants.VTYPE_STRING:
409 if not isinstance(target[key], basestring):
410 if isinstance(target[key], bool) and not target[key]:
413 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
414 raise errors.TypeEnforcementError(msg)
415 elif ktype == constants.VTYPE_BOOL:
416 if isinstance(target[key], basestring) and target[key]:
417 if target[key].lower() == constants.VALUE_FALSE:
419 elif target[key].lower() == constants.VALUE_TRUE:
422 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
423 raise errors.TypeEnforcementError(msg)
428 elif ktype == constants.VTYPE_SIZE:
430 target[key] = ParseUnit(target[key])
431 except errors.UnitParseError, err:
432 msg = "'%s' (value %s) is not a valid size. error: %s" % \
433 (key, target[key], err)
434 raise errors.TypeEnforcementError(msg)
435 elif ktype == constants.VTYPE_INT:
437 target[key] = int(target[key])
438 except (ValueError, TypeError):
439 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
440 raise errors.TypeEnforcementError(msg)
443 def IsProcessAlive(pid):
444 """Check if a given pid exists on the system.
446 @note: zombie status is not handled, so zombie processes
447 will be returned as alive
449 @param pid: the process ID to check
451 @return: True if the process exists
458 os.stat("/proc/%d/status" % pid)
460 except EnvironmentError, err:
461 if err.errno in (errno.ENOENT, errno.ENOTDIR):
466 def ReadPidFile(pidfile):
467 """Read a pid from a file.
469 @type pidfile: string
470 @param pidfile: path to the file containing the pid
472 @return: The process id, if the file exists and contains a valid PID,
477 raw_data = ReadFile(pidfile)
478 except EnvironmentError, err:
479 if err.errno != errno.ENOENT:
480 logging.exception("Can't read pid file")
485 except ValueError, err:
486 logging.info("Can't parse pid file contents", exc_info=True)
492 def MatchNameComponent(key, name_list, case_sensitive=True):
493 """Try to match a name against a list.
495 This function will try to match a name like test1 against a list
496 like C{['test1.example.com', 'test2.example.com', ...]}. Against
497 this list, I{'test1'} as well as I{'test1.example'} will match, but
498 not I{'test1.ex'}. A multiple match will be considered as no match
499 at all (e.g. I{'test1'} against C{['test1.example.com',
500 'test1.example.org']}), except when the key fully matches an entry
501 (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
504 @param key: the name to be searched
505 @type name_list: list
506 @param name_list: the list of strings against which to search the key
507 @type case_sensitive: boolean
508 @param case_sensitive: whether to provide a case-sensitive match
511 @return: None if there is no match I{or} if there are multiple matches,
512 otherwise the element from the list which matches
519 if not case_sensitive:
520 re_flags |= re.IGNORECASE
522 mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
525 for name in name_list:
526 if mo.match(name) is not None:
527 names_filtered.append(name)
528 if not case_sensitive and key == name.upper():
529 string_matches.append(name)
531 if len(string_matches) == 1:
532 return string_matches[0]
533 if len(names_filtered) == 1:
534 return names_filtered[0]
539 """Class implementing resolver and hostname functionality
542 def __init__(self, name=None):
543 """Initialize the host name object.
545 If the name argument is not passed, it will use this system's
550 name = self.SysName()
553 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
554 self.ip = self.ipaddrs[0]
557 """Returns the hostname without domain.
560 return self.name.split('.')[0]
564 """Return the current system's name.
566 This is simply a wrapper over C{socket.gethostname()}.
569 return socket.gethostname()
572 def LookupHostname(hostname):
576 @param hostname: hostname to look up
579 @return: a tuple (name, aliases, ipaddrs) as returned by
580 C{socket.gethostbyname_ex}
581 @raise errors.ResolverError: in case of errors in resolving
585 result = socket.gethostbyname_ex(hostname)
586 except socket.gaierror, err:
587 # hostname not found in DNS
588 raise errors.ResolverError(hostname, err.args[0], err.args[1])
593 def GetHostInfo(name=None):
594 """Lookup host name and raise an OpPrereqError for failures"""
597 return HostInfo(name)
598 except errors.ResolverError, err:
599 raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
600 (err[0], err[2]), errors.ECODE_RESOLVER)
603 def ListVolumeGroups():
604 """List volume groups and their size
608 Dictionary with keys volume name and values
609 the size of the volume
612 command = "vgs --noheadings --units m --nosuffix -o name,size"
613 result = RunCmd(command)
618 for line in result.stdout.splitlines():
620 name, size = line.split()
621 size = int(float(size))
622 except (IndexError, ValueError), err:
623 logging.error("Invalid output from vgs (%s): %s", err, line)
631 def BridgeExists(bridge):
632 """Check whether the given bridge exists in the system
635 @param bridge: the bridge name to check
637 @return: True if it does
640 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
643 def NiceSort(name_list):
644 """Sort a list of strings based on digit and non-digit groupings.
646 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
647 will sort the list in the logical order C{['a1', 'a2', 'a10',
650 The sort algorithm breaks each name in groups of either only-digits
651 or no-digits. Only the first eight such groups are considered, and
652 after that we just use what's left of the string.
654 @type name_list: list
655 @param name_list: the names to be sorted
657 @return: a copy of the name list sorted with our algorithm
660 _SORTER_BASE = "(\D+|\d+)"
661 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
662 _SORTER_BASE, _SORTER_BASE,
663 _SORTER_BASE, _SORTER_BASE,
664 _SORTER_BASE, _SORTER_BASE)
665 _SORTER_RE = re.compile(_SORTER_FULL)
666 _SORTER_NODIGIT = re.compile("^\D*$")
668 """Attempts to convert a variable to integer."""
669 if val is None or _SORTER_NODIGIT.match(val):
674 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
675 for name in name_list]
677 return [tup[1] for tup in to_sort]
680 def TryConvert(fn, val):
681 """Try to convert a value ignoring errors.
683 This function tries to apply function I{fn} to I{val}. If no
684 C{ValueError} or C{TypeError} exceptions are raised, it will return
685 the result, else it will return the original value. Any other
686 exceptions are propagated to the caller.
689 @param fn: function to apply to the value
690 @param val: the value to be converted
691 @return: The converted value if the conversion was successful,
692 otherwise the original value.
697 except (ValueError, TypeError):
703 """Verifies the syntax of an IPv4 address.
705 This function checks if the IPv4 address passes is valid or not based
706 on syntax (not IP range, class calculations, etc.).
709 @param ip: the address to be checked
710 @rtype: a regular expression match object
711 @return: a regular expression match object, or None if the
715 unit = "(0|[1-9]\d{0,2})"
716 #TODO: convert and return only boolean
717 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
720 def IsValidShellParam(word):
721 """Verifies is the given word is safe from the shell's p.o.v.
723 This means that we can pass this to a command via the shell and be
724 sure that it doesn't alter the command line and is passed as such to
727 Note that we are overly restrictive here, in order to be on the safe
731 @param word: the word to check
733 @return: True if the word is 'safe'
736 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
739 def BuildShellCmd(template, *args):
740 """Build a safe shell command line from the given arguments.
742 This function will check all arguments in the args list so that they
743 are valid shell parameters (i.e. they don't contain shell
744 metacharacters). If everything is ok, it will return the result of
748 @param template: the string holding the template for the
751 @return: the expanded command line
755 if not IsValidShellParam(word):
756 raise errors.ProgrammerError("Shell argument '%s' contains"
757 " invalid characters" % word)
758 return template % args
761 def FormatUnit(value, units):
762 """Formats an incoming number of MiB with the appropriate unit.
765 @param value: integer representing the value in MiB (1048576)
767 @param units: the type of formatting we should do:
768 - 'h' for automatic scaling
773 @return: the formatted value (with suffix)
776 if units not in ('m', 'g', 't', 'h'):
777 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
781 if units == 'm' or (units == 'h' and value < 1024):
784 return "%d%s" % (round(value, 0), suffix)
786 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
789 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
794 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
797 def ParseUnit(input_string):
798 """Tries to extract number and scale from the given string.
800 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
801 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
802 is always an int in MiB.
805 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
807 raise errors.UnitParseError("Invalid format")
809 value = float(m.groups()[0])
813 lcunit = unit.lower()
817 if lcunit in ('m', 'mb', 'mib'):
818 # Value already in MiB
821 elif lcunit in ('g', 'gb', 'gib'):
824 elif lcunit in ('t', 'tb', 'tib'):
828 raise errors.UnitParseError("Unknown unit: %s" % unit)
830 # Make sure we round up
831 if int(value) < value:
834 # Round up to the next multiple of 4
837 value += 4 - value % 4
842 def AddAuthorizedKey(file_name, key):
843 """Adds an SSH public key to an authorized_keys file.
846 @param file_name: path to authorized_keys file
848 @param key: string containing key
851 key_fields = key.split()
853 f = open(file_name, 'a+')
857 # Ignore whitespace changes
858 if line.split() == key_fields:
860 nl = line.endswith('\n')
864 f.write(key.rstrip('\r\n'))
871 def RemoveAuthorizedKey(file_name, key):
872 """Removes an SSH public key from an authorized_keys file.
875 @param file_name: path to authorized_keys file
877 @param key: string containing key
880 key_fields = key.split()
882 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
884 out = os.fdopen(fd, 'w')
886 f = open(file_name, 'r')
889 # Ignore whitespace changes while comparing lines
890 if line.split() != key_fields:
894 os.rename(tmpname, file_name)
904 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
905 """Sets the name of an IP address and hostname in /etc/hosts.
908 @param file_name: path to the file to modify (usually C{/etc/hosts})
910 @param ip: the IP address
912 @param hostname: the hostname to be added
914 @param aliases: the list of aliases to add for the hostname
917 # FIXME: use WriteFile + fn rather than duplicating its efforts
918 # Ensure aliases are unique
919 aliases = UniqueSequence([hostname] + aliases)[1:]
921 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
923 out = os.fdopen(fd, 'w')
925 f = open(file_name, 'r')
928 fields = line.split()
929 if fields and not fields[0].startswith('#') and ip == fields[0]:
933 out.write("%s\t%s" % (ip, hostname))
935 out.write(" %s" % ' '.join(aliases))
940 os.chmod(tmpname, 0644)
941 os.rename(tmpname, file_name)
951 def AddHostToEtcHosts(hostname):
952 """Wrapper around SetEtcHostsEntry.
955 @param hostname: a hostname that will be resolved and added to
956 L{constants.ETC_HOSTS}
959 hi = HostInfo(name=hostname)
960 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
963 def RemoveEtcHostsEntry(file_name, hostname):
964 """Removes a hostname from /etc/hosts.
966 IP addresses without names are removed from the file.
969 @param file_name: path to the file to modify (usually C{/etc/hosts})
971 @param hostname: the hostname to be removed
974 # FIXME: use WriteFile + fn rather than duplicating its efforts
975 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
977 out = os.fdopen(fd, 'w')
979 f = open(file_name, 'r')
982 fields = line.split()
983 if len(fields) > 1 and not fields[0].startswith('#'):
985 if hostname in names:
986 while hostname in names:
987 names.remove(hostname)
989 out.write("%s %s\n" % (fields[0], ' '.join(names)))
996 os.chmod(tmpname, 0644)
997 os.rename(tmpname, file_name)
1007 def RemoveHostFromEtcHosts(hostname):
1008 """Wrapper around RemoveEtcHostsEntry.
1011 @param hostname: hostname that will be resolved and its
1012 full and shot name will be removed from
1013 L{constants.ETC_HOSTS}
1016 hi = HostInfo(name=hostname)
1017 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1018 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1021 def CreateBackup(file_name):
1022 """Creates a backup of a file.
1024 @type file_name: str
1025 @param file_name: file to be backed up
1027 @return: the path to the newly created backup
1028 @raise errors.ProgrammerError: for invalid file names
1031 if not os.path.isfile(file_name):
1032 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1035 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1036 dir_name = os.path.dirname(file_name)
1038 fsrc = open(file_name, 'rb')
1040 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1041 fdst = os.fdopen(fd, 'wb')
1043 shutil.copyfileobj(fsrc, fdst)
1052 def ShellQuote(value):
1053 """Quotes shell argument according to POSIX.
1056 @param value: the argument to be quoted
1058 @return: the quoted value
1061 if _re_shell_unquoted.match(value):
1064 return "'%s'" % value.replace("'", "'\\''")
1067 def ShellQuoteArgs(args):
1068 """Quotes a list of shell arguments.
1071 @param args: list of arguments to be quoted
1073 @return: the quoted arguments concatenated with spaces
1076 return ' '.join([ShellQuote(i) for i in args])
1079 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1080 """Simple ping implementation using TCP connect(2).
1082 Check if the given IP is reachable by doing attempting a TCP connect
1086 @param target: the IP or hostname to ping
1088 @param port: the port to connect to
1090 @param timeout: the timeout on the connection attempt
1091 @type live_port_needed: boolean
1092 @param live_port_needed: whether a closed port will cause the
1093 function to return failure, as if there was a timeout
1094 @type source: str or None
1095 @param source: if specified, will cause the connect to be made
1096 from this specific source address; failures to bind other
1097 than C{EADDRNOTAVAIL} will be ignored
1100 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1104 if source is not None:
1106 sock.bind((source, 0))
1107 except socket.error, (errcode, _):
1108 if errcode == errno.EADDRNOTAVAIL:
1111 sock.settimeout(timeout)
1114 sock.connect((target, port))
1117 except socket.timeout:
1119 except socket.error, (errcode, _):
1120 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1125 def OwnIpAddress(address):
1126 """Check if the current host has the the given IP address.
1128 Currently this is done by TCP-pinging the address from the loopback
1131 @type address: string
1132 @param address: the address to check
1134 @return: True if we own the address
1137 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1138 source=constants.LOCALHOST_IP_ADDRESS)
1141 def ListVisibleFiles(path):
1142 """Returns a list of visible files in a directory.
1145 @param path: the directory to enumerate
1147 @return: the list of all files not starting with a dot
1150 files = [i for i in os.listdir(path) if not i.startswith(".")]
1155 def GetHomeDir(user, default=None):
1156 """Try to get the homedir of the given user.
1158 The user can be passed either as a string (denoting the name) or as
1159 an integer (denoting the user id). If the user is not found, the
1160 'default' argument is returned, which defaults to None.
1164 if isinstance(user, basestring):
1165 result = pwd.getpwnam(user)
1166 elif isinstance(user, (int, long)):
1167 result = pwd.getpwuid(user)
1169 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1173 return result.pw_dir
1177 """Returns a random UUID.
1179 @note: This is a Linux-specific method as it uses the /proc
1184 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1187 def GenerateSecret(numbytes=20):
1188 """Generates a random secret.
1190 This will generate a pseudo-random secret returning an hex string
1191 (so that it can be used where an ASCII string is needed).
1193 @param numbytes: the number of bytes which will be represented by the returned
1194 string (defaulting to 20, the length of a SHA1 hash)
1196 @return: an hex representation of the pseudo-random sequence
1199 return os.urandom(numbytes).encode('hex')
1202 def EnsureDirs(dirs):
1203 """Make required directories, if they don't exist.
1205 @param dirs: list of tuples (dir_name, dir_mode)
1206 @type dirs: list of (string, integer)
1209 for dir_name, dir_mode in dirs:
1211 os.mkdir(dir_name, dir_mode)
1212 except EnvironmentError, err:
1213 if err.errno != errno.EEXIST:
1214 raise errors.GenericError("Cannot create needed directory"
1215 " '%s': %s" % (dir_name, err))
1216 if not os.path.isdir(dir_name):
1217 raise errors.GenericError("%s is not a directory" % dir_name)
1220 def ReadFile(file_name, size=None):
1223 @type size: None or int
1224 @param size: Read at most size bytes
1226 @return: the (possibly partial) content of the file
1229 f = open(file_name, "r")
1239 def WriteFile(file_name, fn=None, data=None,
1240 mode=None, uid=-1, gid=-1,
1241 atime=None, mtime=None, close=True,
1242 dry_run=False, backup=False,
1243 prewrite=None, postwrite=None):
1244 """(Over)write a file atomically.
1246 The file_name and either fn (a function taking one argument, the
1247 file descriptor, and which should write the data to it) or data (the
1248 contents of the file) must be passed. The other arguments are
1249 optional and allow setting the file mode, owner and group, and the
1250 mtime/atime of the file.
1252 If the function doesn't raise an exception, it has succeeded and the
1253 target file has the new contents. If the function has raised an
1254 exception, an existing target file should be unmodified and the
1255 temporary file should be removed.
1257 @type file_name: str
1258 @param file_name: the target filename
1260 @param fn: content writing function, called with
1261 file descriptor as parameter
1263 @param data: contents of the file
1265 @param mode: file mode
1267 @param uid: the owner of the file
1269 @param gid: the group of the file
1271 @param atime: a custom access time to be set on the file
1273 @param mtime: a custom modification time to be set on the file
1274 @type close: boolean
1275 @param close: whether to close file after writing it
1276 @type prewrite: callable
1277 @param prewrite: function to be called before writing content
1278 @type postwrite: callable
1279 @param postwrite: function to be called after writing content
1282 @return: None if the 'close' parameter evaluates to True,
1283 otherwise the file descriptor
1285 @raise errors.ProgrammerError: if any of the arguments are not valid
1288 if not os.path.isabs(file_name):
1289 raise errors.ProgrammerError("Path passed to WriteFile is not"
1290 " absolute: '%s'" % file_name)
1292 if [fn, data].count(None) != 1:
1293 raise errors.ProgrammerError("fn or data required")
1295 if [atime, mtime].count(None) == 1:
1296 raise errors.ProgrammerError("Both atime and mtime must be either"
1299 if backup and not dry_run and os.path.isfile(file_name):
1300 CreateBackup(file_name)
1302 dir_name, base_name = os.path.split(file_name)
1303 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1305 # here we need to make sure we remove the temp file, if any error
1306 # leaves it in place
1308 if uid != -1 or gid != -1:
1309 os.chown(new_name, uid, gid)
1311 os.chmod(new_name, mode)
1312 if callable(prewrite):
1314 if data is not None:
1318 if callable(postwrite):
1321 if atime is not None and mtime is not None:
1322 os.utime(new_name, (atime, mtime))
1324 os.rename(new_name, file_name)
1333 RemoveFile(new_name)
1338 def FirstFree(seq, base=0):
1339 """Returns the first non-existing integer from seq.
1341 The seq argument should be a sorted list of positive integers. The
1342 first time the index of an element is smaller than the element
1343 value, the index will be returned.
1345 The base argument is used to start at a different offset,
1346 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1348 Example: C{[0, 1, 3]} will return I{2}.
1351 @param seq: the sequence to be analyzed.
1353 @param base: use this value as the base index of the sequence
1355 @return: the first non-used index in the sequence
1358 for idx, elem in enumerate(seq):
1359 assert elem >= base, "Passed element is higher than base offset"
1360 if elem > idx + base:
1366 def all(seq, pred=bool):
1367 "Returns True if pred(x) is True for every element in the iterable"
1368 for _ in itertools.ifilterfalse(pred, seq):
1373 def any(seq, pred=bool):
1374 "Returns True if pred(x) is True for at least one element in the iterable"
1375 for _ in itertools.ifilter(pred, seq):
1380 def UniqueSequence(seq):
1381 """Returns a list with unique elements.
1383 Element order is preserved.
1386 @param seq: the sequence with the source elements
1388 @return: list of unique elements from seq
1392 return [i for i in seq if i not in seen and not seen.add(i)]
1395 def IsValidMac(mac):
1396 """Predicate to check if a MAC address is valid.
1398 Checks whether the supplied MAC address is formally correct, only
1399 accepts colon separated format.
1402 @param mac: the MAC to be validated
1404 @return: True is the MAC seems valid
1407 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1408 return mac_check.match(mac) is not None
1411 def TestDelay(duration):
1412 """Sleep for a fixed amount of time.
1414 @type duration: float
1415 @param duration: the sleep duration
1417 @return: False for negative value, True otherwise
1421 return False, "Invalid sleep duration"
1422 time.sleep(duration)
1426 def _CloseFDNoErr(fd, retries=5):
1427 """Close a file descriptor ignoring errors.
1430 @param fd: the file descriptor
1432 @param retries: how many retries to make, in case we get any
1433 other error than EBADF
1438 except OSError, err:
1439 if err.errno != errno.EBADF:
1441 _CloseFDNoErr(fd, retries - 1)
1442 # else either it's closed already or we're out of retries, so we
1443 # ignore this and go on
1446 def CloseFDs(noclose_fds=None):
1447 """Close file descriptors.
1449 This closes all file descriptors above 2 (i.e. except
1452 @type noclose_fds: list or None
1453 @param noclose_fds: if given, it denotes a list of file descriptor
1454 that should not be closed
1457 # Default maximum for the number of available file descriptors.
1458 if 'SC_OPEN_MAX' in os.sysconf_names:
1460 MAXFD = os.sysconf('SC_OPEN_MAX')
1467 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1468 if (maxfd == resource.RLIM_INFINITY):
1471 # Iterate through and close all file descriptors (except the standard ones)
1472 for fd in range(3, maxfd):
1473 if noclose_fds and fd in noclose_fds:
1478 def Daemonize(logfile):
1479 """Daemonize the current process.
1481 This detaches the current process from the controlling terminal and
1482 runs it in the background as a daemon.
1485 @param logfile: the logfile to which we should redirect stdout/stderr
1487 @return: the value zero
1495 if (pid == 0): # The first child.
1498 pid = os.fork() # Fork a second child.
1499 if (pid == 0): # The second child.
1503 # exit() or _exit()? See below.
1504 os._exit(0) # Exit parent (the first child) of the second child.
1506 os._exit(0) # Exit parent of the first child.
1510 i = os.open("/dev/null", os.O_RDONLY) # stdin
1511 assert i == 0, "Can't close/reopen stdin"
1512 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1513 assert i == 1, "Can't close/reopen stdout"
1514 # Duplicate standard output to standard error.
1519 def DaemonPidFileName(name):
1520 """Compute a ganeti pid file absolute path
1523 @param name: the daemon name
1525 @return: the full path to the pidfile corresponding to the given
1529 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1532 def WritePidFile(name):
1533 """Write the current process pidfile.
1535 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1538 @param name: the daemon name to use
1539 @raise errors.GenericError: if the pid file already exists and
1540 points to a live process
1544 pidfilename = DaemonPidFileName(name)
1545 if IsProcessAlive(ReadPidFile(pidfilename)):
1546 raise errors.GenericError("%s contains a live process" % pidfilename)
1548 WriteFile(pidfilename, data="%d\n" % pid)
1551 def RemovePidFile(name):
1552 """Remove the current process pidfile.
1554 Any errors are ignored.
1557 @param name: the daemon name used to derive the pidfile name
1560 pidfilename = DaemonPidFileName(name)
1561 # TODO: we could check here that the file contains our pid
1563 RemoveFile(pidfilename)
1568 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1570 """Kill a process given by its pid.
1573 @param pid: The PID to terminate.
1575 @param signal_: The signal to send, by default SIGTERM
1577 @param timeout: The timeout after which, if the process is still alive,
1578 a SIGKILL will be sent. If not positive, no such checking
1580 @type waitpid: boolean
1581 @param waitpid: If true, we should waitpid on this process after
1582 sending signals, since it's our own child and otherwise it
1583 would remain as zombie
1586 def _helper(pid, signal_, wait):
1587 """Simple helper to encapsulate the kill/waitpid sequence"""
1588 os.kill(pid, signal_)
1591 os.waitpid(pid, os.WNOHANG)
1596 # kill with pid=0 == suicide
1597 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1599 if not IsProcessAlive(pid):
1602 _helper(pid, signal_, waitpid)
1607 def _CheckProcess():
1608 if not IsProcessAlive(pid):
1612 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1622 # Wait up to $timeout seconds
1623 Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1624 except RetryTimeout:
1627 if IsProcessAlive(pid):
1628 # Kill process if it's still alive
1629 _helper(pid, signal.SIGKILL, waitpid)
1632 def FindFile(name, search_path, test=os.path.exists):
1633 """Look for a filesystem object in a given path.
1635 This is an abstract method to search for filesystem object (files,
1636 dirs) under a given search path.
1639 @param name: the name to look for
1640 @type search_path: str
1641 @param search_path: location to start at
1642 @type test: callable
1643 @param test: a function taking one argument that should return True
1644 if the a given object is valid; the default value is
1645 os.path.exists, causing only existing files to be returned
1647 @return: full path to the object if found, None otherwise
1650 for dir_name in search_path:
1651 item_name = os.path.sep.join([dir_name, name])
1657 def CheckVolumeGroupSize(vglist, vgname, minsize):
1658 """Checks if the volume group list is valid.
1660 The function will check if a given volume group is in the list of
1661 volume groups and has a minimum size.
1664 @param vglist: dictionary of volume group names and their size
1666 @param vgname: the volume group we should check
1668 @param minsize: the minimum size we accept
1670 @return: None for success, otherwise the error message
1673 vgsize = vglist.get(vgname, None)
1675 return "volume group '%s' missing" % vgname
1676 elif vgsize < minsize:
1677 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1678 (vgname, minsize, vgsize))
1682 def SplitTime(value):
1683 """Splits time as floating point number into a tuple.
1685 @param value: Time in seconds
1686 @type value: int or float
1687 @return: Tuple containing (seconds, microseconds)
1690 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1692 assert 0 <= seconds, \
1693 "Seconds must be larger than or equal to 0, but are %s" % seconds
1694 assert 0 <= microseconds <= 999999, \
1695 "Microseconds must be 0-999999, but are %s" % microseconds
1697 return (int(seconds), int(microseconds))
1700 def MergeTime(timetuple):
1701 """Merges a tuple into time as a floating point number.
1703 @param timetuple: Time as tuple, (seconds, microseconds)
1704 @type timetuple: tuple
1705 @return: Time as a floating point number expressed in seconds
1708 (seconds, microseconds) = timetuple
1710 assert 0 <= seconds, \
1711 "Seconds must be larger than or equal to 0, but are %s" % seconds
1712 assert 0 <= microseconds <= 999999, \
1713 "Microseconds must be 0-999999, but are %s" % microseconds
1715 return float(seconds) + (float(microseconds) * 0.000001)
1718 def GetDaemonPort(daemon_name):
1719 """Get the daemon port for this cluster.
1721 Note that this routine does not read a ganeti-specific file, but
1722 instead uses C{socket.getservbyname} to allow pre-customization of
1723 this parameter outside of Ganeti.
1725 @type daemon_name: string
1726 @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1730 if daemon_name not in constants.DAEMONS_PORTS:
1731 raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1733 (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1735 port = socket.getservbyname(daemon_name, proto)
1736 except socket.error:
1742 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1743 multithreaded=False):
1744 """Configures the logging module.
1747 @param logfile: the filename to which we should log
1748 @type debug: boolean
1749 @param debug: whether to enable debug messages too or
1750 only those at C{INFO} and above level
1751 @type stderr_logging: boolean
1752 @param stderr_logging: whether we should also log to the standard error
1754 @param program: the name under which we should log messages
1755 @type multithreaded: boolean
1756 @param multithreaded: if True, will add the thread name to the log file
1757 @raise EnvironmentError: if we can't open the log file and
1758 stderr logging is disabled
1761 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1763 fmt += "/%(threadName)s"
1765 fmt += " %(module)s:%(lineno)s"
1766 fmt += " %(levelname)s %(message)s"
1767 formatter = logging.Formatter(fmt)
1769 root_logger = logging.getLogger("")
1770 root_logger.setLevel(logging.NOTSET)
1772 # Remove all previously setup handlers
1773 for handler in root_logger.handlers:
1775 root_logger.removeHandler(handler)
1778 stderr_handler = logging.StreamHandler()
1779 stderr_handler.setFormatter(formatter)
1781 stderr_handler.setLevel(logging.NOTSET)
1783 stderr_handler.setLevel(logging.CRITICAL)
1784 root_logger.addHandler(stderr_handler)
1786 # this can fail, if the logging directories are not setup or we have
1787 # a permisssion problem; in this case, it's best to log but ignore
1788 # the error if stderr_logging is True, and if false we re-raise the
1789 # exception since otherwise we could run but without any logs at all
1791 logfile_handler = logging.FileHandler(logfile)
1792 logfile_handler.setFormatter(formatter)
1794 logfile_handler.setLevel(logging.DEBUG)
1796 logfile_handler.setLevel(logging.INFO)
1797 root_logger.addHandler(logfile_handler)
1798 except EnvironmentError:
1800 logging.exception("Failed to enable logging to file '%s'", logfile)
1802 # we need to re-raise the exception
1806 def IsNormAbsPath(path):
1807 """Check whether a path is absolute and also normalized
1809 This avoids things like /dir/../../other/path to be valid.
1812 return os.path.normpath(path) == path and os.path.isabs(path)
1815 def TailFile(fname, lines=20):
1816 """Return the last lines from a file.
1818 @note: this function will only read and parse the last 4KB of
1819 the file; if the lines are very long, it could be that less
1820 than the requested number of lines are returned
1822 @param fname: the file name
1824 @param lines: the (maximum) number of lines to return
1827 fd = open(fname, "r")
1831 pos = max(0, pos-4096)
1833 raw_data = fd.read()
1837 rows = raw_data.splitlines()
1838 return rows[-lines:]
1841 def SafeEncode(text):
1842 """Return a 'safe' version of a source string.
1844 This function mangles the input string and returns a version that
1845 should be safe to display/encode as ASCII. To this end, we first
1846 convert it to ASCII using the 'backslashreplace' encoding which
1847 should get rid of any non-ASCII chars, and then we process it
1848 through a loop copied from the string repr sources in the python; we
1849 don't use string_escape anymore since that escape single quotes and
1850 backslashes too, and that is too much; and that escaping is not
1851 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
1853 @type text: str or unicode
1854 @param text: input data
1856 @return: a safe version of text
1859 if isinstance(text, unicode):
1860 # only if unicode; if str already, we handle it below
1861 text = text.encode('ascii', 'backslashreplace')
1871 elif c < 32 or c >= 127: # non-printable
1872 resu += "\\x%02x" % (c & 0xff)
1878 def BytesToMebibyte(value):
1879 """Converts bytes to mebibytes.
1882 @param value: Value in bytes
1884 @return: Value in mebibytes
1887 return int(round(value / (1024.0 * 1024.0), 0))
1890 def CalculateDirectorySize(path):
1891 """Calculates the size of a directory recursively.
1894 @param path: Path to directory
1896 @return: Size in mebibytes
1901 for (curpath, _, files) in os.walk(path):
1902 for filename in files:
1903 st = os.lstat(os.path.join(curpath, filename))
1906 return BytesToMebibyte(size)
1909 def GetFilesystemStats(path):
1910 """Returns the total and free space on a filesystem.
1913 @param path: Path on filesystem to be examined
1915 @return: tuple of (Total space, Free space) in mebibytes
1918 st = os.statvfs(path)
1920 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
1921 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
1922 return (tsize, fsize)
1925 def LockedMethod(fn):
1926 """Synchronized object access decorator.
1928 This decorator is intended to protect access to an object using the
1929 object's own lock which is hardcoded to '_lock'.
1932 def _LockDebug(*args, **kwargs):
1934 logging.debug(*args, **kwargs)
1936 def wrapper(self, *args, **kwargs):
1937 assert hasattr(self, '_lock')
1939 _LockDebug("Waiting for %s", lock)
1942 _LockDebug("Acquired %s", lock)
1943 result = fn(self, *args, **kwargs)
1945 _LockDebug("Releasing %s", lock)
1947 _LockDebug("Released %s", lock)
1953 """Locks a file using POSIX locks.
1956 @param fd: the file descriptor we need to lock
1960 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1961 except IOError, err:
1962 if err.errno == errno.EAGAIN:
1963 raise errors.LockError("File already locked")
1967 def FormatTime(val):
1968 """Formats a time value.
1970 @type val: float or None
1971 @param val: the timestamp as returned by time.time()
1972 @return: a string value or N/A if we don't have a valid timestamp
1975 if val is None or not isinstance(val, (int, float)):
1977 # these two codes works on Linux, but they are not guaranteed on all
1979 return time.strftime("%F %T", time.localtime(val))
1982 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
1983 """Reads the watcher pause file.
1985 @type filename: string
1986 @param filename: Path to watcher pause file
1987 @type now: None, float or int
1988 @param now: Current time as Unix timestamp
1989 @type remove_after: int
1990 @param remove_after: Remove watcher pause file after specified amount of
1991 seconds past the pause end time
1998 value = ReadFile(filename)
1999 except IOError, err:
2000 if err.errno != errno.ENOENT:
2004 if value is not None:
2008 logging.warning(("Watcher pause file (%s) contains invalid value,"
2009 " removing it"), filename)
2010 RemoveFile(filename)
2013 if value is not None:
2014 # Remove file if it's outdated
2015 if now > (value + remove_after):
2016 RemoveFile(filename)
2025 class RetryTimeout(Exception):
2026 """Retry loop timed out.
2031 class RetryAgain(Exception):
2037 class _RetryDelayCalculator(object):
2038 """Calculator for increasing delays.
2048 def __init__(self, start, factor, limit):
2049 """Initializes this class.
2052 @param start: Initial delay
2054 @param factor: Factor for delay increase
2055 @type limit: float or None
2056 @param limit: Upper limit for delay or None for no limit
2060 assert factor >= 1.0
2061 assert limit is None or limit >= 0.0
2064 self._factor = factor
2070 """Returns current delay and calculates the next one.
2073 current = self._next
2075 # Update for next run
2076 if self._limit is None or self._next < self._limit:
2077 self._next = max(self._limit, self._next * self._factor)
2082 #: Special delay to specify whole remaining timeout
2083 RETRY_REMAINING_TIME = object()
2086 def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2087 _time_fn=time.time):
2088 """Call a function repeatedly until it succeeds.
2090 The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2091 anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2092 total of C{timeout} seconds, this function throws L{RetryTimeout}.
2094 C{delay} can be one of the following:
2095 - callable returning the delay length as a float
2096 - Tuple of (start, factor, limit)
2097 - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2098 useful when overriding L{wait_fn} to wait for an external event)
2099 - A static delay as a number (int or float)
2102 @param fn: Function to be called
2103 @param delay: Either a callable (returning the delay), a tuple of (start,
2104 factor, limit) (see L{_RetryDelayCalculator}),
2105 L{RETRY_REMAINING_TIME} or a number (int or float)
2106 @type timeout: float
2107 @param timeout: Total timeout
2108 @type wait_fn: callable
2109 @param wait_fn: Waiting function
2110 @return: Return value of function
2114 assert callable(wait_fn)
2115 assert callable(_time_fn)
2120 end_time = _time_fn() + timeout
2123 # External function to calculate delay
2126 elif isinstance(delay, (tuple, list)):
2127 # Increasing delay with optional upper boundary
2128 (start, factor, limit) = delay
2129 calc_delay = _RetryDelayCalculator(start, factor, limit)
2131 elif delay is RETRY_REMAINING_TIME:
2132 # Always use the remaining time
2137 calc_delay = lambda: delay
2139 assert calc_delay is None or callable(calc_delay)
2147 remaining_time = end_time - _time_fn()
2149 if remaining_time < 0.0:
2150 raise RetryTimeout()
2152 assert remaining_time >= 0.0
2154 if calc_delay is None:
2155 wait_fn(remaining_time)
2157 current_delay = calc_delay()
2158 if current_delay > 0.0:
2159 wait_fn(current_delay)
2162 class FileLock(object):
2163 """Utility class for file locks.
2166 def __init__(self, filename):
2167 """Constructor for FileLock.
2169 This will open the file denoted by the I{filename} argument.
2172 @param filename: path to the file to be locked
2175 self.filename = filename
2176 self.fd = open(self.filename, "w")
2182 """Close the file and release the lock.
2189 def _flock(self, flag, blocking, timeout, errmsg):
2190 """Wrapper for fcntl.flock.
2193 @param flag: operation flag
2194 @type blocking: bool
2195 @param blocking: whether the operation should be done in blocking mode.
2196 @type timeout: None or float
2197 @param timeout: for how long the operation should be retried (implies
2199 @type errmsg: string
2200 @param errmsg: error message in case operation fails.
2203 assert self.fd, "Lock was closed"
2204 assert timeout is None or timeout >= 0, \
2205 "If specified, timeout must be positive"
2207 if timeout is not None:
2208 flag |= fcntl.LOCK_NB
2209 timeout_end = time.time() + timeout
2211 # Blocking doesn't have effect with timeout
2213 flag |= fcntl.LOCK_NB
2216 # TODO: Convert to utils.Retry
2221 fcntl.flock(self.fd, flag)
2223 except IOError, err:
2224 if err.errno in (errno.EAGAIN, ):
2225 if timeout_end is not None and time.time() < timeout_end:
2226 # Wait before trying again
2227 time.sleep(max(0.1, min(1.0, timeout)))
2229 raise errors.LockError(errmsg)
2231 logging.exception("fcntl.flock failed")
2234 def Exclusive(self, blocking=False, timeout=None):
2235 """Locks the file in exclusive mode.
2237 @type blocking: boolean
2238 @param blocking: whether to block and wait until we
2239 can lock the file or return immediately
2240 @type timeout: int or None
2241 @param timeout: if not None, the duration to wait for the lock
2245 self._flock(fcntl.LOCK_EX, blocking, timeout,
2246 "Failed to lock %s in exclusive mode" % self.filename)
2248 def Shared(self, blocking=False, timeout=None):
2249 """Locks the file in shared mode.
2251 @type blocking: boolean
2252 @param blocking: whether to block and wait until we
2253 can lock the file or return immediately
2254 @type timeout: int or None
2255 @param timeout: if not None, the duration to wait for the lock
2259 self._flock(fcntl.LOCK_SH, blocking, timeout,
2260 "Failed to lock %s in shared mode" % self.filename)
2262 def Unlock(self, blocking=True, timeout=None):
2263 """Unlocks the file.
2265 According to C{flock(2)}, unlocking can also be a nonblocking
2268 To make a non-blocking request, include LOCK_NB with any of the above
2271 @type blocking: boolean
2272 @param blocking: whether to block and wait until we
2273 can lock the file or return immediately
2274 @type timeout: int or None
2275 @param timeout: if not None, the duration to wait for the lock
2279 self._flock(fcntl.LOCK_UN, blocking, timeout,
2280 "Failed to unlock %s" % self.filename)
2283 def SignalHandled(signums):
2284 """Signal Handled decoration.
2286 This special decorator installs a signal handler and then calls the target
2287 function. The function must accept a 'signal_handlers' keyword argument,
2288 which will contain a dict indexed by signal number, with SignalHandler
2291 The decorator can be safely stacked with iself, to handle multiple signals
2292 with different handlers.
2295 @param signums: signals to intercept
2299 def sig_function(*args, **kwargs):
2300 assert 'signal_handlers' not in kwargs or \
2301 kwargs['signal_handlers'] is None or \
2302 isinstance(kwargs['signal_handlers'], dict), \
2303 "Wrong signal_handlers parameter in original function call"
2304 if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2305 signal_handlers = kwargs['signal_handlers']
2307 signal_handlers = {}
2308 kwargs['signal_handlers'] = signal_handlers
2309 sighandler = SignalHandler(signums)
2312 signal_handlers[sig] = sighandler
2313 return fn(*args, **kwargs)
2320 class SignalHandler(object):
2321 """Generic signal handler class.
2323 It automatically restores the original handler when deconstructed or
2324 when L{Reset} is called. You can either pass your own handler
2325 function in or query the L{called} attribute to detect whether the
2329 @ivar signum: the signals we handle
2330 @type called: boolean
2331 @ivar called: tracks whether any of the signals have been raised
2334 def __init__(self, signum):
2335 """Constructs a new SignalHandler instance.
2337 @type signum: int or list of ints
2338 @param signum: Single signal number or set of signal numbers
2341 self.signum = set(signum)
2346 for signum in self.signum:
2348 prev_handler = signal.signal(signum, self._HandleSignal)
2350 self._previous[signum] = prev_handler
2352 # Restore previous handler
2353 signal.signal(signum, prev_handler)
2356 # Reset all handlers
2358 # Here we have a race condition: a handler may have already been called,
2359 # but there's not much we can do about it at this point.
2366 """Restore previous handler.
2368 This will reset all the signals to their previous handlers.
2371 for signum, prev_handler in self._previous.items():
2372 signal.signal(signum, prev_handler)
2373 # If successful, remove from dict
2374 del self._previous[signum]
2377 """Unsets the L{called} flag.
2379 This function can be used in case a signal may arrive several times.
2384 def _HandleSignal(self, signum, frame):
2385 """Actual signal handling function.
2388 # This is not nice and not absolutely atomic, but it appears to be the only
2389 # solution in Python -- there are no atomic types.
2393 class FieldSet(object):
2394 """A simple field set.
2396 Among the features are:
2397 - checking if a string is among a list of static string or regex objects
2398 - checking if a whole list of string matches
2399 - returning the matching groups from a regex match
2401 Internally, all fields are held as regular expression objects.
2404 def __init__(self, *items):
2405 self.items = [re.compile("^%s$" % value) for value in items]
2407 def Extend(self, other_set):
2408 """Extend the field set with the items from another one"""
2409 self.items.extend(other_set.items)
2411 def Matches(self, field):
2412 """Checks if a field matches the current set
2415 @param field: the string to match
2416 @return: either False or a regular expression match object
2419 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2423 def NonMatching(self, items):
2424 """Returns the list of fields not matching the current set
2427 @param items: the list of fields to check
2429 @return: list of non-matching fields
2432 return [val for val in items if not self.Matches(val)]