4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
47 from cStringIO import StringIO
50 from hashlib import sha1
55 from ganeti import errors
56 from ganeti import constants
60 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
65 #: when set to True, L{RunCmd} is disabled
69 class RunResult(object):
70 """Holds the result of running external programs.
73 @ivar exit_code: the exit code of the program, or None (if the program
75 @type signal: int or None
76 @ivar signal: the signal that caused the program to finish, or None
77 (if the program wasn't terminated by a signal)
79 @ivar stdout: the standard output of the program
81 @ivar stderr: the standard error of the program
83 @ivar failed: True in case the program was
84 terminated by a signal or exited with a non-zero exit code
85 @ivar fail_reason: a string detailing the termination reason
88 __slots__ = ["exit_code", "signal", "stdout", "stderr",
89 "failed", "fail_reason", "cmd"]
92 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
94 self.exit_code = exit_code
98 self.failed = (signal_ is not None or exit_code != 0)
100 if self.signal is not None:
101 self.fail_reason = "terminated by signal %s" % self.signal
102 elif self.exit_code is not None:
103 self.fail_reason = "exited with exit code %s" % self.exit_code
105 self.fail_reason = "unable to determine termination reason"
108 logging.debug("Command '%s' failed (%s); output: %s",
109 self.cmd, self.fail_reason, self.output)
111 def _GetOutput(self):
112 """Returns the combined stdout and stderr for easier usage.
115 return self.stdout + self.stderr
117 output = property(_GetOutput, None, None, "Return full output")
120 def RunCmd(cmd, env=None, output=None, cwd='/'):
121 """Execute a (shell) command.
123 The command should not read from its standard input, as it will be
126 @type cmd: string or list
127 @param cmd: Command to run
129 @param env: Additional environment
131 @param output: if desired, the output of the command can be
132 saved in a file instead of the RunResult instance; this
133 parameter denotes the file name (if not None)
135 @param cwd: if specified, will be used as the working
136 directory for the command; the default will be /
138 @return: RunResult instance
139 @raise errors.ProgrammerError: if we call this when forks are disabled
143 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
145 if isinstance(cmd, list):
146 cmd = [str(val) for val in cmd]
147 strcmd = " ".join(cmd)
152 logging.debug("RunCmd '%s'", strcmd)
154 cmd_env = os.environ.copy()
155 cmd_env["LC_ALL"] = "C"
161 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
163 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
166 if err.errno == errno.ENOENT:
167 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
179 return RunResult(exitcode, signal_, out, err, strcmd)
182 def _RunCmdPipe(cmd, env, via_shell, cwd):
183 """Run a command and return its output.
185 @type cmd: string or list
186 @param cmd: Command to run
188 @param env: The environment to use
189 @type via_shell: bool
190 @param via_shell: if we should run via the shell
192 @param cwd: the working directory for the program
194 @return: (out, err, status)
197 poller = select.poll()
198 child = subprocess.Popen(cmd, shell=via_shell,
199 stderr=subprocess.PIPE,
200 stdout=subprocess.PIPE,
201 stdin=subprocess.PIPE,
202 close_fds=True, env=env,
206 poller.register(child.stdout, select.POLLIN)
207 poller.register(child.stderr, select.POLLIN)
211 child.stdout.fileno(): (out, child.stdout),
212 child.stderr.fileno(): (err, child.stderr),
215 status = fcntl.fcntl(fd, fcntl.F_GETFL)
216 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
220 pollresult = poller.poll()
221 except EnvironmentError, eerr:
222 if eerr.errno == errno.EINTR:
225 except select.error, serr:
226 if serr[0] == errno.EINTR:
230 for fd, event in pollresult:
231 if event & select.POLLIN or event & select.POLLPRI:
232 data = fdmap[fd][1].read()
233 # no data from read signifies EOF (the same as POLLHUP)
235 poller.unregister(fd)
238 fdmap[fd][0].write(data)
239 if (event & select.POLLNVAL or event & select.POLLHUP or
240 event & select.POLLERR):
241 poller.unregister(fd)
247 status = child.wait()
248 return out, err, status
251 def _RunCmdFile(cmd, env, via_shell, output, cwd):
252 """Run a command and save its output to a file.
254 @type cmd: string or list
255 @param cmd: Command to run
257 @param env: The environment to use
258 @type via_shell: bool
259 @param via_shell: if we should run via the shell
261 @param output: the filename in which to save the output
263 @param cwd: the working directory for the program
265 @return: the exit status
268 fh = open(output, "a")
270 child = subprocess.Popen(cmd, shell=via_shell,
271 stderr=subprocess.STDOUT,
273 stdin=subprocess.PIPE,
274 close_fds=True, env=env,
278 status = child.wait()
284 def RemoveFile(filename):
285 """Remove a file ignoring some errors.
287 Remove a file, ignoring non-existing ones or directories. Other
291 @param filename: the file to be removed
297 if err.errno not in (errno.ENOENT, errno.EISDIR):
301 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
305 @param old: Original path
309 @param mkdir: Whether to create target directory if it doesn't exist
310 @type mkdir_mode: int
311 @param mkdir_mode: Mode for newly created directories
315 return os.rename(old, new)
317 # In at least one use case of this function, the job queue, directory
318 # creation is very rare. Checking for the directory before renaming is not
320 if mkdir and err.errno == errno.ENOENT:
321 # Create directory and try again
322 os.makedirs(os.path.dirname(new), mkdir_mode)
323 return os.rename(old, new)
327 def _FingerprintFile(filename):
328 """Compute the fingerprint of a file.
330 If the file does not exist, a None will be returned
334 @param filename: the filename to checksum
336 @return: the hex digest of the sha checksum of the contents
340 if not (os.path.exists(filename) and os.path.isfile(filename)):
353 return fp.hexdigest()
356 def FingerprintFiles(files):
357 """Compute fingerprints for a list of files.
360 @param files: the list of filename to fingerprint
362 @return: a dictionary filename: fingerprint, holding only
368 for filename in files:
369 cksum = _FingerprintFile(filename)
371 ret[filename] = cksum
376 def CheckDict(target, template, logname=None):
377 """Ensure a dictionary has a required set of keys.
379 For the given dictionaries I{target} and I{template}, ensure
380 I{target} has all the keys from I{template}. Missing keys are added
381 with values from template.
384 @param target: the dictionary to update
386 @param template: the dictionary holding the default values
387 @type logname: str or None
388 @param logname: if not None, causes the missing keys to be
389 logged with this name
396 target[k] = template[k]
398 if missing and logname:
399 logging.warning('%s missing keys %s', logname, ', '.join(missing))
402 def ForceDictType(target, key_types, allowed_values=None):
403 """Force the values of a dict to have certain types.
406 @param target: the dict to update
407 @type key_types: dict
408 @param key_types: dict mapping target dict keys to types
409 in constants.ENFORCEABLE_TYPES
410 @type allowed_values: list
411 @keyword allowed_values: list of specially allowed values
414 if allowed_values is None:
418 if key not in key_types:
419 msg = "Unknown key '%s'" % key
420 raise errors.TypeEnforcementError(msg)
422 if target[key] in allowed_values:
425 type = key_types[key]
426 if type not in constants.ENFORCEABLE_TYPES:
427 msg = "'%s' has non-enforceable type %s" % (key, type)
428 raise errors.ProgrammerError(msg)
430 if type == constants.VTYPE_STRING:
431 if not isinstance(target[key], basestring):
432 if isinstance(target[key], bool) and not target[key]:
435 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
436 raise errors.TypeEnforcementError(msg)
437 elif type == constants.VTYPE_BOOL:
438 if isinstance(target[key], basestring) and target[key]:
439 if target[key].lower() == constants.VALUE_FALSE:
441 elif target[key].lower() == constants.VALUE_TRUE:
444 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
445 raise errors.TypeEnforcementError(msg)
450 elif type == constants.VTYPE_SIZE:
452 target[key] = ParseUnit(target[key])
453 except errors.UnitParseError, err:
454 msg = "'%s' (value %s) is not a valid size. error: %s" % \
455 (key, target[key], err)
456 raise errors.TypeEnforcementError(msg)
457 elif type == constants.VTYPE_INT:
459 target[key] = int(target[key])
460 except (ValueError, TypeError):
461 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
462 raise errors.TypeEnforcementError(msg)
465 def IsProcessAlive(pid):
466 """Check if a given pid exists on the system.
468 @note: zombie status is not handled, so zombie processes
469 will be returned as alive
471 @param pid: the process ID to check
473 @return: True if the process exists
480 os.stat("/proc/%d/status" % pid)
482 except EnvironmentError, err:
483 if err.errno in (errno.ENOENT, errno.ENOTDIR):
488 def ReadPidFile(pidfile):
489 """Read a pid from a file.
491 @type pidfile: string
492 @param pidfile: path to the file containing the pid
494 @return: The process id, if the file exists and contains a valid PID,
499 pf = open(pidfile, 'r')
500 except EnvironmentError, err:
501 if err.errno != errno.ENOENT:
502 logging.exception("Can't read pid file?!")
507 except ValueError, err:
508 logging.info("Can't parse pid file contents", exc_info=True)
514 def MatchNameComponent(key, name_list):
515 """Try to match a name against a list.
517 This function will try to match a name like test1 against a list
518 like C{['test1.example.com', 'test2.example.com', ...]}. Against
519 this list, I{'test1'} as well as I{'test1.example'} will match, but
520 not I{'test1.ex'}. A multiple match will be considered as no match
521 at all (e.g. I{'test1'} against C{['test1.example.com',
522 'test1.example.org']}).
525 @param key: the name to be searched
526 @type name_list: list
527 @param name_list: the list of strings against which to search the key
530 @return: None if there is no match I{or} if there are multiple matches,
531 otherwise the element from the list which matches
534 mo = re.compile("^%s(\..*)?$" % re.escape(key))
535 names_filtered = [name for name in name_list if mo.match(name) is not None]
536 if len(names_filtered) != 1:
538 return names_filtered[0]
542 """Class implementing resolver and hostname functionality
545 def __init__(self, name=None):
546 """Initialize the host name object.
548 If the name argument is not passed, it will use this system's
553 name = self.SysName()
556 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
557 self.ip = self.ipaddrs[0]
560 """Returns the hostname without domain.
563 return self.name.split('.')[0]
567 """Return the current system's name.
569 This is simply a wrapper over C{socket.gethostname()}.
572 return socket.gethostname()
575 def LookupHostname(hostname):
579 @param hostname: hostname to look up
582 @return: a tuple (name, aliases, ipaddrs) as returned by
583 C{socket.gethostbyname_ex}
584 @raise errors.ResolverError: in case of errors in resolving
588 result = socket.gethostbyname_ex(hostname)
589 except socket.gaierror, err:
590 # hostname not found in DNS
591 raise errors.ResolverError(hostname, err.args[0], err.args[1])
596 def ListVolumeGroups():
597 """List volume groups and their size
601 Dictionary with keys volume name and values
602 the size of the volume
605 command = "vgs --noheadings --units m --nosuffix -o name,size"
606 result = RunCmd(command)
611 for line in result.stdout.splitlines():
613 name, size = line.split()
614 size = int(float(size))
615 except (IndexError, ValueError), err:
616 logging.error("Invalid output from vgs (%s): %s", err, line)
624 def BridgeExists(bridge):
625 """Check whether the given bridge exists in the system
628 @param bridge: the bridge name to check
630 @return: True if it does
633 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
636 def NiceSort(name_list):
637 """Sort a list of strings based on digit and non-digit groupings.
639 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
640 will sort the list in the logical order C{['a1', 'a2', 'a10',
643 The sort algorithm breaks each name in groups of either only-digits
644 or no-digits. Only the first eight such groups are considered, and
645 after that we just use what's left of the string.
647 @type name_list: list
648 @param name_list: the names to be sorted
650 @return: a copy of the name list sorted with our algorithm
653 _SORTER_BASE = "(\D+|\d+)"
654 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
655 _SORTER_BASE, _SORTER_BASE,
656 _SORTER_BASE, _SORTER_BASE,
657 _SORTER_BASE, _SORTER_BASE)
658 _SORTER_RE = re.compile(_SORTER_FULL)
659 _SORTER_NODIGIT = re.compile("^\D*$")
661 """Attempts to convert a variable to integer."""
662 if val is None or _SORTER_NODIGIT.match(val):
667 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
668 for name in name_list]
670 return [tup[1] for tup in to_sort]
673 def TryConvert(fn, val):
674 """Try to convert a value ignoring errors.
676 This function tries to apply function I{fn} to I{val}. If no
677 C{ValueError} or C{TypeError} exceptions are raised, it will return
678 the result, else it will return the original value. Any other
679 exceptions are propagated to the caller.
682 @param fn: function to apply to the value
683 @param val: the value to be converted
684 @return: The converted value if the conversion was successful,
685 otherwise the original value.
690 except (ValueError, TypeError), err:
696 """Verifies the syntax of an IPv4 address.
698 This function checks if the IPv4 address passes is valid or not based
699 on syntax (not IP range, class calculations, etc.).
702 @param ip: the address to be checked
703 @rtype: a regular expression match object
704 @return: a regular expression match object, or None if the
708 unit = "(0|[1-9]\d{0,2})"
709 #TODO: convert and return only boolean
710 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
713 def IsValidShellParam(word):
714 """Verifies is the given word is safe from the shell's p.o.v.
716 This means that we can pass this to a command via the shell and be
717 sure that it doesn't alter the command line and is passed as such to
720 Note that we are overly restrictive here, in order to be on the safe
724 @param word: the word to check
726 @return: True if the word is 'safe'
729 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
732 def BuildShellCmd(template, *args):
733 """Build a safe shell command line from the given arguments.
735 This function will check all arguments in the args list so that they
736 are valid shell parameters (i.e. they don't contain shell
737 metacharacters). If everything is ok, it will return the result of
741 @param template: the string holding the template for the
744 @return: the expanded command line
748 if not IsValidShellParam(word):
749 raise errors.ProgrammerError("Shell argument '%s' contains"
750 " invalid characters" % word)
751 return template % args
754 def FormatUnit(value, units):
755 """Formats an incoming number of MiB with the appropriate unit.
758 @param value: integer representing the value in MiB (1048576)
760 @param units: the type of formatting we should do:
761 - 'h' for automatic scaling
766 @return: the formatted value (with suffix)
769 if units not in ('m', 'g', 't', 'h'):
770 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
774 if units == 'm' or (units == 'h' and value < 1024):
777 return "%d%s" % (round(value, 0), suffix)
779 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
782 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
787 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
790 def ParseUnit(input_string):
791 """Tries to extract number and scale from the given string.
793 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
794 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
795 is always an int in MiB.
798 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
800 raise errors.UnitParseError("Invalid format")
802 value = float(m.groups()[0])
806 lcunit = unit.lower()
810 if lcunit in ('m', 'mb', 'mib'):
811 # Value already in MiB
814 elif lcunit in ('g', 'gb', 'gib'):
817 elif lcunit in ('t', 'tb', 'tib'):
821 raise errors.UnitParseError("Unknown unit: %s" % unit)
823 # Make sure we round up
824 if int(value) < value:
827 # Round up to the next multiple of 4
830 value += 4 - value % 4
835 def AddAuthorizedKey(file_name, key):
836 """Adds an SSH public key to an authorized_keys file.
839 @param file_name: path to authorized_keys file
841 @param key: string containing key
844 key_fields = key.split()
846 f = open(file_name, 'a+')
850 # Ignore whitespace changes
851 if line.split() == key_fields:
853 nl = line.endswith('\n')
857 f.write(key.rstrip('\r\n'))
864 def RemoveAuthorizedKey(file_name, key):
865 """Removes an SSH public key from an authorized_keys file.
868 @param file_name: path to authorized_keys file
870 @param key: string containing key
873 key_fields = key.split()
875 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
877 out = os.fdopen(fd, 'w')
879 f = open(file_name, 'r')
882 # Ignore whitespace changes while comparing lines
883 if line.split() != key_fields:
887 os.rename(tmpname, file_name)
897 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
898 """Sets the name of an IP address and hostname in /etc/hosts.
901 @param file_name: path to the file to modify (usually C{/etc/hosts})
903 @param ip: the IP address
905 @param hostname: the hostname to be added
907 @param aliases: the list of aliases to add for the hostname
910 # FIXME: use WriteFile + fn rather than duplicating its efforts
911 # Ensure aliases are unique
912 aliases = UniqueSequence([hostname] + aliases)[1:]
914 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
916 out = os.fdopen(fd, 'w')
918 f = open(file_name, 'r')
921 fields = line.split()
922 if fields and not fields[0].startswith('#') and ip == fields[0]:
926 out.write("%s\t%s" % (ip, hostname))
928 out.write(" %s" % ' '.join(aliases))
933 os.chmod(tmpname, 0644)
934 os.rename(tmpname, file_name)
944 def AddHostToEtcHosts(hostname):
945 """Wrapper around SetEtcHostsEntry.
948 @param hostname: a hostname that will be resolved and added to
949 L{constants.ETC_HOSTS}
952 hi = HostInfo(name=hostname)
953 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
956 def RemoveEtcHostsEntry(file_name, hostname):
957 """Removes a hostname from /etc/hosts.
959 IP addresses without names are removed from the file.
962 @param file_name: path to the file to modify (usually C{/etc/hosts})
964 @param hostname: the hostname to be removed
967 # FIXME: use WriteFile + fn rather than duplicating its efforts
968 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
970 out = os.fdopen(fd, 'w')
972 f = open(file_name, 'r')
975 fields = line.split()
976 if len(fields) > 1 and not fields[0].startswith('#'):
978 if hostname in names:
979 while hostname in names:
980 names.remove(hostname)
982 out.write("%s %s\n" % (fields[0], ' '.join(names)))
989 os.chmod(tmpname, 0644)
990 os.rename(tmpname, file_name)
1000 def RemoveHostFromEtcHosts(hostname):
1001 """Wrapper around RemoveEtcHostsEntry.
1004 @param hostname: hostname that will be resolved and its
1005 full and shot name will be removed from
1006 L{constants.ETC_HOSTS}
1009 hi = HostInfo(name=hostname)
1010 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1011 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1014 def CreateBackup(file_name):
1015 """Creates a backup of a file.
1017 @type file_name: str
1018 @param file_name: file to be backed up
1020 @return: the path to the newly created backup
1021 @raise errors.ProgrammerError: for invalid file names
1024 if not os.path.isfile(file_name):
1025 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1028 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1029 dir_name = os.path.dirname(file_name)
1031 fsrc = open(file_name, 'rb')
1033 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1034 fdst = os.fdopen(fd, 'wb')
1036 shutil.copyfileobj(fsrc, fdst)
1045 def ShellQuote(value):
1046 """Quotes shell argument according to POSIX.
1049 @param value: the argument to be quoted
1051 @return: the quoted value
1054 if _re_shell_unquoted.match(value):
1057 return "'%s'" % value.replace("'", "'\\''")
1060 def ShellQuoteArgs(args):
1061 """Quotes a list of shell arguments.
1064 @param args: list of arguments to be quoted
1066 @return: the quoted arguments concatenated with spaces
1069 return ' '.join([ShellQuote(i) for i in args])
1072 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1073 """Simple ping implementation using TCP connect(2).
1075 Check if the given IP is reachable by doing attempting a TCP connect
1079 @param target: the IP or hostname to ping
1081 @param port: the port to connect to
1083 @param timeout: the timeout on the connection attempt
1084 @type live_port_needed: boolean
1085 @param live_port_needed: whether a closed port will cause the
1086 function to return failure, as if there was a timeout
1087 @type source: str or None
1088 @param source: if specified, will cause the connect to be made
1089 from this specific source address; failures to bind other
1090 than C{EADDRNOTAVAIL} will be ignored
1093 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1097 if source is not None:
1099 sock.bind((source, 0))
1100 except socket.error, (errcode, errstring):
1101 if errcode == errno.EADDRNOTAVAIL:
1104 sock.settimeout(timeout)
1107 sock.connect((target, port))
1110 except socket.timeout:
1112 except socket.error, (errcode, errstring):
1113 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1118 def OwnIpAddress(address):
1119 """Check if the current host has the the given IP address.
1121 Currently this is done by TCP-pinging the address from the loopback
1124 @type address: string
1125 @param address: the address to check
1127 @return: True if we own the address
1130 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1131 source=constants.LOCALHOST_IP_ADDRESS)
1134 def ListVisibleFiles(path):
1135 """Returns a list of visible files in a directory.
1138 @param path: the directory to enumerate
1140 @return: the list of all files not starting with a dot
1143 files = [i for i in os.listdir(path) if not i.startswith(".")]
1148 def GetHomeDir(user, default=None):
1149 """Try to get the homedir of the given user.
1151 The user can be passed either as a string (denoting the name) or as
1152 an integer (denoting the user id). If the user is not found, the
1153 'default' argument is returned, which defaults to None.
1157 if isinstance(user, basestring):
1158 result = pwd.getpwnam(user)
1159 elif isinstance(user, (int, long)):
1160 result = pwd.getpwuid(user)
1162 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1166 return result.pw_dir
1170 """Returns a random UUID.
1172 @note: This is a Linux-specific method as it uses the /proc
1177 f = open("/proc/sys/kernel/random/uuid", "r")
1179 return f.read(128).rstrip("\n")
1184 def GenerateSecret():
1185 """Generates a random secret.
1187 This will generate a pseudo-random secret, and return its sha digest
1188 (so that it can be used where an ASCII string is needed).
1191 @return: a sha1 hexdigest of a block of 64 random bytes
1194 return sha1(os.urandom(64)).hexdigest()
1197 def EnsureDirs(dirs):
1198 """Make required directories, if they don't exist.
1200 @param dirs: list of tuples (dir_name, dir_mode)
1201 @type dirs: list of (string, integer)
1204 for dir_name, dir_mode in dirs:
1206 os.mkdir(dir_name, dir_mode)
1207 except EnvironmentError, err:
1208 if err.errno != errno.EEXIST:
1209 raise errors.GenericError("Cannot create needed directory"
1210 " '%s': %s" % (dir_name, err))
1211 if not os.path.isdir(dir_name):
1212 raise errors.GenericError("%s is not a directory" % dir_name)
1215 def ReadFile(file_name, size=None):
1218 @type size: None or int
1219 @param size: Read at most size bytes
1221 @return: the (possibly partial) content of the file
1224 f = open(file_name, "r")
1234 def WriteFile(file_name, fn=None, data=None,
1235 mode=None, uid=-1, gid=-1,
1236 atime=None, mtime=None, close=True,
1237 dry_run=False, backup=False,
1238 prewrite=None, postwrite=None):
1239 """(Over)write a file atomically.
1241 The file_name and either fn (a function taking one argument, the
1242 file descriptor, and which should write the data to it) or data (the
1243 contents of the file) must be passed. The other arguments are
1244 optional and allow setting the file mode, owner and group, and the
1245 mtime/atime of the file.
1247 If the function doesn't raise an exception, it has succeeded and the
1248 target file has the new contents. If the function has raised an
1249 exception, an existing target file should be unmodified and the
1250 temporary file should be removed.
1252 @type file_name: str
1253 @param file_name: the target filename
1255 @param fn: content writing function, called with
1256 file descriptor as parameter
1258 @param data: contents of the file
1260 @param mode: file mode
1262 @param uid: the owner of the file
1264 @param gid: the group of the file
1266 @param atime: a custom access time to be set on the file
1268 @param mtime: a custom modification time to be set on the file
1269 @type close: boolean
1270 @param close: whether to close file after writing it
1271 @type prewrite: callable
1272 @param prewrite: function to be called before writing content
1273 @type postwrite: callable
1274 @param postwrite: function to be called after writing content
1277 @return: None if the 'close' parameter evaluates to True,
1278 otherwise the file descriptor
1280 @raise errors.ProgrammerError: if any of the arguments are not valid
1283 if not os.path.isabs(file_name):
1284 raise errors.ProgrammerError("Path passed to WriteFile is not"
1285 " absolute: '%s'" % file_name)
1287 if [fn, data].count(None) != 1:
1288 raise errors.ProgrammerError("fn or data required")
1290 if [atime, mtime].count(None) == 1:
1291 raise errors.ProgrammerError("Both atime and mtime must be either"
1294 if backup and not dry_run and os.path.isfile(file_name):
1295 CreateBackup(file_name)
1297 dir_name, base_name = os.path.split(file_name)
1298 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1300 # here we need to make sure we remove the temp file, if any error
1301 # leaves it in place
1303 if uid != -1 or gid != -1:
1304 os.chown(new_name, uid, gid)
1306 os.chmod(new_name, mode)
1307 if callable(prewrite):
1309 if data is not None:
1313 if callable(postwrite):
1316 if atime is not None and mtime is not None:
1317 os.utime(new_name, (atime, mtime))
1319 os.rename(new_name, file_name)
1328 RemoveFile(new_name)
1333 def FirstFree(seq, base=0):
1334 """Returns the first non-existing integer from seq.
1336 The seq argument should be a sorted list of positive integers. The
1337 first time the index of an element is smaller than the element
1338 value, the index will be returned.
1340 The base argument is used to start at a different offset,
1341 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1343 Example: C{[0, 1, 3]} will return I{2}.
1346 @param seq: the sequence to be analyzed.
1348 @param base: use this value as the base index of the sequence
1350 @return: the first non-used index in the sequence
1353 for idx, elem in enumerate(seq):
1354 assert elem >= base, "Passed element is higher than base offset"
1355 if elem > idx + base:
1361 def all(seq, pred=bool):
1362 "Returns True if pred(x) is True for every element in the iterable"
1363 for elem in itertools.ifilterfalse(pred, seq):
1368 def any(seq, pred=bool):
1369 "Returns True if pred(x) is True for at least one element in the iterable"
1370 for elem in itertools.ifilter(pred, seq):
1375 def UniqueSequence(seq):
1376 """Returns a list with unique elements.
1378 Element order is preserved.
1381 @param seq: the sequence with the source elements
1383 @return: list of unique elements from seq
1387 return [i for i in seq if i not in seen and not seen.add(i)]
1390 def IsValidMac(mac):
1391 """Predicate to check if a MAC address is valid.
1393 Checks whether the supplied MAC address is formally correct, only
1394 accepts colon separated format.
1397 @param mac: the MAC to be validated
1399 @return: True is the MAC seems valid
1402 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1403 return mac_check.match(mac) is not None
1406 def TestDelay(duration):
1407 """Sleep for a fixed amount of time.
1409 @type duration: float
1410 @param duration: the sleep duration
1412 @return: False for negative value, True otherwise
1417 time.sleep(duration)
1421 def _CloseFDNoErr(fd, retries=5):
1422 """Close a file descriptor ignoring errors.
1425 @param fd: the file descriptor
1427 @param retries: how many retries to make, in case we get any
1428 other error than EBADF
1433 except OSError, err:
1434 if err.errno != errno.EBADF:
1436 _CloseFDNoErr(fd, retries - 1)
1437 # else either it's closed already or we're out of retries, so we
1438 # ignore this and go on
1441 def CloseFDs(noclose_fds=None):
1442 """Close file descriptors.
1444 This closes all file descriptors above 2 (i.e. except
1447 @type noclose_fds: list or None
1448 @param noclose_fds: if given, it denotes a list of file descriptor
1449 that should not be closed
1452 # Default maximum for the number of available file descriptors.
1453 if 'SC_OPEN_MAX' in os.sysconf_names:
1455 MAXFD = os.sysconf('SC_OPEN_MAX')
1462 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1463 if (maxfd == resource.RLIM_INFINITY):
1466 # Iterate through and close all file descriptors (except the standard ones)
1467 for fd in range(3, maxfd):
1468 if noclose_fds and fd in noclose_fds:
1473 def Daemonize(logfile):
1474 """Daemonize the current process.
1476 This detaches the current process from the controlling terminal and
1477 runs it in the background as a daemon.
1480 @param logfile: the logfile to which we should redirect stdout/stderr
1482 @return: the value zero
1490 if (pid == 0): # The first child.
1493 pid = os.fork() # Fork a second child.
1494 if (pid == 0): # The second child.
1498 # exit() or _exit()? See below.
1499 os._exit(0) # Exit parent (the first child) of the second child.
1501 os._exit(0) # Exit parent of the first child.
1505 i = os.open("/dev/null", os.O_RDONLY) # stdin
1506 assert i == 0, "Can't close/reopen stdin"
1507 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1508 assert i == 1, "Can't close/reopen stdout"
1509 # Duplicate standard output to standard error.
1514 def DaemonPidFileName(name):
1515 """Compute a ganeti pid file absolute path
1518 @param name: the daemon name
1520 @return: the full path to the pidfile corresponding to the given
1524 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1527 def WritePidFile(name):
1528 """Write the current process pidfile.
1530 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1533 @param name: the daemon name to use
1534 @raise errors.GenericError: if the pid file already exists and
1535 points to a live process
1539 pidfilename = DaemonPidFileName(name)
1540 if IsProcessAlive(ReadPidFile(pidfilename)):
1541 raise errors.GenericError("%s contains a live process" % pidfilename)
1543 WriteFile(pidfilename, data="%d\n" % pid)
1546 def RemovePidFile(name):
1547 """Remove the current process pidfile.
1549 Any errors are ignored.
1552 @param name: the daemon name used to derive the pidfile name
1556 pidfilename = DaemonPidFileName(name)
1557 # TODO: we could check here that the file contains our pid
1559 RemoveFile(pidfilename)
1564 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1566 """Kill a process given by its pid.
1569 @param pid: The PID to terminate.
1571 @param signal_: The signal to send, by default SIGTERM
1573 @param timeout: The timeout after which, if the process is still alive,
1574 a SIGKILL will be sent. If not positive, no such checking
1576 @type waitpid: boolean
1577 @param waitpid: If true, we should waitpid on this process after
1578 sending signals, since it's our own child and otherwise it
1579 would remain as zombie
1582 def _helper(pid, signal_, wait):
1583 """Simple helper to encapsulate the kill/waitpid sequence"""
1584 os.kill(pid, signal_)
1587 os.waitpid(pid, os.WNOHANG)
1592 # kill with pid=0 == suicide
1593 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1595 if not IsProcessAlive(pid):
1597 _helper(pid, signal_, waitpid)
1601 # Wait up to $timeout seconds
1602 end = time.time() + timeout
1604 while time.time() < end and IsProcessAlive(pid):
1606 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1612 # Make wait time longer for next try
1616 if IsProcessAlive(pid):
1617 # Kill process if it's still alive
1618 _helper(pid, signal.SIGKILL, waitpid)
1621 def FindFile(name, search_path, test=os.path.exists):
1622 """Look for a filesystem object in a given path.
1624 This is an abstract method to search for filesystem object (files,
1625 dirs) under a given search path.
1628 @param name: the name to look for
1629 @type search_path: str
1630 @param search_path: location to start at
1631 @type test: callable
1632 @param test: a function taking one argument that should return True
1633 if the a given object is valid; the default value is
1634 os.path.exists, causing only existing files to be returned
1636 @return: full path to the object if found, None otherwise
1639 for dir_name in search_path:
1640 item_name = os.path.sep.join([dir_name, name])
1646 def CheckVolumeGroupSize(vglist, vgname, minsize):
1647 """Checks if the volume group list is valid.
1649 The function will check if a given volume group is in the list of
1650 volume groups and has a minimum size.
1653 @param vglist: dictionary of volume group names and their size
1655 @param vgname: the volume group we should check
1657 @param minsize: the minimum size we accept
1659 @return: None for success, otherwise the error message
1662 vgsize = vglist.get(vgname, None)
1664 return "volume group '%s' missing" % vgname
1665 elif vgsize < minsize:
1666 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1667 (vgname, minsize, vgsize))
1671 def SplitTime(value):
1672 """Splits time as floating point number into a tuple.
1674 @param value: Time in seconds
1675 @type value: int or float
1676 @return: Tuple containing (seconds, microseconds)
1679 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1681 assert 0 <= seconds, \
1682 "Seconds must be larger than or equal to 0, but are %s" % seconds
1683 assert 0 <= microseconds <= 999999, \
1684 "Microseconds must be 0-999999, but are %s" % microseconds
1686 return (int(seconds), int(microseconds))
1689 def MergeTime(timetuple):
1690 """Merges a tuple into time as a floating point number.
1692 @param timetuple: Time as tuple, (seconds, microseconds)
1693 @type timetuple: tuple
1694 @return: Time as a floating point number expressed in seconds
1697 (seconds, microseconds) = timetuple
1699 assert 0 <= seconds, \
1700 "Seconds must be larger than or equal to 0, but are %s" % seconds
1701 assert 0 <= microseconds <= 999999, \
1702 "Microseconds must be 0-999999, but are %s" % microseconds
1704 return float(seconds) + (float(microseconds) * 0.000001)
1707 def GetNodeDaemonPort():
1708 """Get the node daemon port for this cluster.
1710 Note that this routine does not read a ganeti-specific file, but
1711 instead uses C{socket.getservbyname} to allow pre-customization of
1712 this parameter outside of Ganeti.
1718 port = socket.getservbyname("ganeti-noded", "tcp")
1719 except socket.error:
1720 port = constants.DEFAULT_NODED_PORT
1725 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1726 multithreaded=False):
1727 """Configures the logging module.
1730 @param logfile: the filename to which we should log
1731 @type debug: boolean
1732 @param debug: whether to enable debug messages too or
1733 only those at C{INFO} and above level
1734 @type stderr_logging: boolean
1735 @param stderr_logging: whether we should also log to the standard error
1737 @param program: the name under which we should log messages
1738 @type multithreaded: boolean
1739 @param multithreaded: if True, will add the thread name to the log file
1740 @raise EnvironmentError: if we can't open the log file and
1741 stderr logging is disabled
1744 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1746 fmt += "/%(threadName)s"
1748 fmt += " %(module)s:%(lineno)s"
1749 fmt += " %(levelname)s %(message)s"
1750 formatter = logging.Formatter(fmt)
1752 root_logger = logging.getLogger("")
1753 root_logger.setLevel(logging.NOTSET)
1755 # Remove all previously setup handlers
1756 for handler in root_logger.handlers:
1758 root_logger.removeHandler(handler)
1761 stderr_handler = logging.StreamHandler()
1762 stderr_handler.setFormatter(formatter)
1764 stderr_handler.setLevel(logging.NOTSET)
1766 stderr_handler.setLevel(logging.CRITICAL)
1767 root_logger.addHandler(stderr_handler)
1769 # this can fail, if the logging directories are not setup or we have
1770 # a permisssion problem; in this case, it's best to log but ignore
1771 # the error if stderr_logging is True, and if false we re-raise the
1772 # exception since otherwise we could run but without any logs at all
1774 logfile_handler = logging.FileHandler(logfile)
1775 logfile_handler.setFormatter(formatter)
1777 logfile_handler.setLevel(logging.DEBUG)
1779 logfile_handler.setLevel(logging.INFO)
1780 root_logger.addHandler(logfile_handler)
1781 except EnvironmentError:
1783 logging.exception("Failed to enable logging to file '%s'", logfile)
1785 # we need to re-raise the exception
1789 def TailFile(fname, lines=20):
1790 """Return the last lines from a file.
1792 @note: this function will only read and parse the last 4KB of
1793 the file; if the lines are very long, it could be that less
1794 than the requested number of lines are returned
1796 @param fname: the file name
1798 @param lines: the (maximum) number of lines to return
1801 fd = open(fname, "r")
1805 pos = max(0, pos-4096)
1807 raw_data = fd.read()
1811 rows = raw_data.splitlines()
1812 return rows[-lines:]
1815 def SafeEncode(text):
1816 """Return a 'safe' version of a source string.
1818 This function mangles the input string and returns a version that
1819 should be safe to display/encode as ASCII. To this end, we first
1820 convert it to ASCII using the 'backslashreplace' encoding which
1821 should get rid of any non-ASCII chars, and then we process it
1822 through a loop copied from the string repr sources in the python; we
1823 don't use string_escape anymore since that escape single quotes and
1824 backslashes too, and that is too much; and that escaping is not
1825 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
1827 @type text: str or unicode
1828 @param text: input data
1830 @return: a safe version of text
1833 if isinstance(text, unicode):
1834 # only if unicode; if str already, we handle it below
1835 text = text.encode('ascii', 'backslashreplace')
1845 elif c < 32 or c >= 127: # non-printable
1846 resu += "\\x%02x" % (c & 0xff)
1852 def CommaJoin(names):
1853 """Nicely join a set of identifiers.
1855 @param names: set, list or tuple
1856 @return: a string with the formatted results
1859 return ", ".join(["'%s'" % val for val in names])
1862 def LockedMethod(fn):
1863 """Synchronized object access decorator.
1865 This decorator is intended to protect access to an object using the
1866 object's own lock which is hardcoded to '_lock'.
1869 def _LockDebug(*args, **kwargs):
1871 logging.debug(*args, **kwargs)
1873 def wrapper(self, *args, **kwargs):
1874 assert hasattr(self, '_lock')
1876 _LockDebug("Waiting for %s", lock)
1879 _LockDebug("Acquired %s", lock)
1880 result = fn(self, *args, **kwargs)
1882 _LockDebug("Releasing %s", lock)
1884 _LockDebug("Released %s", lock)
1890 """Locks a file using POSIX locks.
1893 @param fd: the file descriptor we need to lock
1897 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1898 except IOError, err:
1899 if err.errno == errno.EAGAIN:
1900 raise errors.LockError("File already locked")
1904 class FileLock(object):
1905 """Utility class for file locks.
1908 def __init__(self, filename):
1909 """Constructor for FileLock.
1911 This will open the file denoted by the I{filename} argument.
1914 @param filename: path to the file to be locked
1917 self.filename = filename
1918 self.fd = open(self.filename, "w")
1924 """Close the file and release the lock.
1931 def _flock(self, flag, blocking, timeout, errmsg):
1932 """Wrapper for fcntl.flock.
1935 @param flag: operation flag
1936 @type blocking: bool
1937 @param blocking: whether the operation should be done in blocking mode.
1938 @type timeout: None or float
1939 @param timeout: for how long the operation should be retried (implies
1941 @type errmsg: string
1942 @param errmsg: error message in case operation fails.
1945 assert self.fd, "Lock was closed"
1946 assert timeout is None or timeout >= 0, \
1947 "If specified, timeout must be positive"
1949 if timeout is not None:
1950 flag |= fcntl.LOCK_NB
1951 timeout_end = time.time() + timeout
1953 # Blocking doesn't have effect with timeout
1955 flag |= fcntl.LOCK_NB
1961 fcntl.flock(self.fd, flag)
1963 except IOError, err:
1964 if err.errno in (errno.EAGAIN, ):
1965 if timeout_end is not None and time.time() < timeout_end:
1966 # Wait before trying again
1967 time.sleep(max(0.1, min(1.0, timeout)))
1969 raise errors.LockError(errmsg)
1971 logging.exception("fcntl.flock failed")
1974 def Exclusive(self, blocking=False, timeout=None):
1975 """Locks the file in exclusive mode.
1977 @type blocking: boolean
1978 @param blocking: whether to block and wait until we
1979 can lock the file or return immediately
1980 @type timeout: int or None
1981 @param timeout: if not None, the duration to wait for the lock
1985 self._flock(fcntl.LOCK_EX, blocking, timeout,
1986 "Failed to lock %s in exclusive mode" % self.filename)
1988 def Shared(self, blocking=False, timeout=None):
1989 """Locks the file in shared mode.
1991 @type blocking: boolean
1992 @param blocking: whether to block and wait until we
1993 can lock the file or return immediately
1994 @type timeout: int or None
1995 @param timeout: if not None, the duration to wait for the lock
1999 self._flock(fcntl.LOCK_SH, blocking, timeout,
2000 "Failed to lock %s in shared mode" % self.filename)
2002 def Unlock(self, blocking=True, timeout=None):
2003 """Unlocks the file.
2005 According to C{flock(2)}, unlocking can also be a nonblocking
2008 To make a non-blocking request, include LOCK_NB with any of the above
2011 @type blocking: boolean
2012 @param blocking: whether to block and wait until we
2013 can lock the file or return immediately
2014 @type timeout: int or None
2015 @param timeout: if not None, the duration to wait for the lock
2019 self._flock(fcntl.LOCK_UN, blocking, timeout,
2020 "Failed to unlock %s" % self.filename)
2023 class SignalHandler(object):
2024 """Generic signal handler class.
2026 It automatically restores the original handler when deconstructed or
2027 when L{Reset} is called. You can either pass your own handler
2028 function in or query the L{called} attribute to detect whether the
2032 @ivar signum: the signals we handle
2033 @type called: boolean
2034 @ivar called: tracks whether any of the signals have been raised
2037 def __init__(self, signum):
2038 """Constructs a new SignalHandler instance.
2040 @type signum: int or list of ints
2041 @param signum: Single signal number or set of signal numbers
2044 if isinstance(signum, (int, long)):
2045 self.signum = set([signum])
2047 self.signum = set(signum)
2053 for signum in self.signum:
2055 prev_handler = signal.signal(signum, self._HandleSignal)
2057 self._previous[signum] = prev_handler
2059 # Restore previous handler
2060 signal.signal(signum, prev_handler)
2063 # Reset all handlers
2065 # Here we have a race condition: a handler may have already been called,
2066 # but there's not much we can do about it at this point.
2073 """Restore previous handler.
2075 This will reset all the signals to their previous handlers.
2078 for signum, prev_handler in self._previous.items():
2079 signal.signal(signum, prev_handler)
2080 # If successful, remove from dict
2081 del self._previous[signum]
2084 """Unsets the L{called} flag.
2086 This function can be used in case a signal may arrive several times.
2091 def _HandleSignal(self, signum, frame):
2092 """Actual signal handling function.
2095 # This is not nice and not absolutely atomic, but it appears to be the only
2096 # solution in Python -- there are no atomic types.
2100 class FieldSet(object):
2101 """A simple field set.
2103 Among the features are:
2104 - checking if a string is among a list of static string or regex objects
2105 - checking if a whole list of string matches
2106 - returning the matching groups from a regex match
2108 Internally, all fields are held as regular expression objects.
2111 def __init__(self, *items):
2112 self.items = [re.compile("^%s$" % value) for value in items]
2114 def Extend(self, other_set):
2115 """Extend the field set with the items from another one"""
2116 self.items.extend(other_set.items)
2118 def Matches(self, field):
2119 """Checks if a field matches the current set
2122 @param field: the string to match
2123 @return: either False or a regular expression match object
2126 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2130 def NonMatching(self, items):
2131 """Returns the list of fields not matching the current set
2134 @param items: the list of fields to check
2136 @return: list of non-matching fields
2139 return [val for val in items if not self.Matches(val)]