4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
46 from cStringIO import StringIO
49 from hashlib import sha1
54 from ganeti import errors
55 from ganeti import constants
59 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
63 #: when set to True, L{RunCmd} is disabled
67 class RunResult(object):
68 """Holds the result of running external programs.
71 @ivar exit_code: the exit code of the program, or None (if the program
73 @type signal: int or None
74 @ivar signal: the signal that caused the program to finish, or None
75 (if the program wasn't terminated by a signal)
77 @ivar stdout: the standard output of the program
79 @ivar stderr: the standard error of the program
81 @ivar failed: True in case the program was
82 terminated by a signal or exited with a non-zero exit code
83 @ivar fail_reason: a string detailing the termination reason
86 __slots__ = ["exit_code", "signal", "stdout", "stderr",
87 "failed", "fail_reason", "cmd"]
90 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
92 self.exit_code = exit_code
96 self.failed = (signal_ is not None or exit_code != 0)
98 if self.signal is not None:
99 self.fail_reason = "terminated by signal %s" % self.signal
100 elif self.exit_code is not None:
101 self.fail_reason = "exited with exit code %s" % self.exit_code
103 self.fail_reason = "unable to determine termination reason"
106 logging.debug("Command '%s' failed (%s); output: %s",
107 self.cmd, self.fail_reason, self.output)
109 def _GetOutput(self):
110 """Returns the combined stdout and stderr for easier usage.
113 return self.stdout + self.stderr
115 output = property(_GetOutput, None, None, "Return full output")
118 def RunCmd(cmd, env=None, output=None, cwd='/'):
119 """Execute a (shell) command.
121 The command should not read from its standard input, as it will be
124 @type cmd: string or list
125 @param cmd: Command to run
127 @param env: Additional environment
129 @param output: if desired, the output of the command can be
130 saved in a file instead of the RunResult instance; this
131 parameter denotes the file name (if not None)
133 @param cwd: if specified, will be used as the working
134 directory for the command; the default will be /
136 @return: RunResult instance
137 @raise errors.ProgrammerError: if we call this when forks are disabled
141 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
143 if isinstance(cmd, list):
144 cmd = [str(val) for val in cmd]
145 strcmd = " ".join(cmd)
150 logging.debug("RunCmd '%s'", strcmd)
152 cmd_env = os.environ.copy()
153 cmd_env["LC_ALL"] = "C"
159 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
161 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
164 if err.errno == errno.ENOENT:
165 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
177 return RunResult(exitcode, signal_, out, err, strcmd)
180 def _RunCmdPipe(cmd, env, via_shell, cwd):
181 """Run a command and return its output.
183 @type cmd: string or list
184 @param cmd: Command to run
186 @param env: The environment to use
187 @type via_shell: bool
188 @param via_shell: if we should run via the shell
190 @param cwd: the working directory for the program
192 @return: (out, err, status)
195 poller = select.poll()
196 child = subprocess.Popen(cmd, shell=via_shell,
197 stderr=subprocess.PIPE,
198 stdout=subprocess.PIPE,
199 stdin=subprocess.PIPE,
200 close_fds=True, env=env,
204 poller.register(child.stdout, select.POLLIN)
205 poller.register(child.stderr, select.POLLIN)
209 child.stdout.fileno(): (out, child.stdout),
210 child.stderr.fileno(): (err, child.stderr),
213 status = fcntl.fcntl(fd, fcntl.F_GETFL)
214 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
218 pollresult = poller.poll()
219 except EnvironmentError, eerr:
220 if eerr.errno == errno.EINTR:
223 except select.error, serr:
224 if serr[0] == errno.EINTR:
228 for fd, event in pollresult:
229 if event & select.POLLIN or event & select.POLLPRI:
230 data = fdmap[fd][1].read()
231 # no data from read signifies EOF (the same as POLLHUP)
233 poller.unregister(fd)
236 fdmap[fd][0].write(data)
237 if (event & select.POLLNVAL or event & select.POLLHUP or
238 event & select.POLLERR):
239 poller.unregister(fd)
245 status = child.wait()
246 return out, err, status
249 def _RunCmdFile(cmd, env, via_shell, output, cwd):
250 """Run a command and save its output to a file.
252 @type cmd: string or list
253 @param cmd: Command to run
255 @param env: The environment to use
256 @type via_shell: bool
257 @param via_shell: if we should run via the shell
259 @param output: the filename in which to save the output
261 @param cwd: the working directory for the program
263 @return: the exit status
266 fh = open(output, "a")
268 child = subprocess.Popen(cmd, shell=via_shell,
269 stderr=subprocess.STDOUT,
271 stdin=subprocess.PIPE,
272 close_fds=True, env=env,
276 status = child.wait()
282 def RemoveFile(filename):
283 """Remove a file ignoring some errors.
285 Remove a file, ignoring non-existing ones or directories. Other
289 @param filename: the file to be removed
295 if err.errno not in (errno.ENOENT, errno.EISDIR):
299 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
303 @param old: Original path
307 @param mkdir: Whether to create target directory if it doesn't exist
308 @type mkdir_mode: int
309 @param mkdir_mode: Mode for newly created directories
313 return os.rename(old, new)
315 # In at least one use case of this function, the job queue, directory
316 # creation is very rare. Checking for the directory before renaming is not
318 if mkdir and err.errno == errno.ENOENT:
319 # Create directory and try again
320 os.makedirs(os.path.dirname(new), mkdir_mode)
321 return os.rename(old, new)
325 def _FingerprintFile(filename):
326 """Compute the fingerprint of a file.
328 If the file does not exist, a None will be returned
332 @param filename: the filename to checksum
334 @return: the hex digest of the sha checksum of the contents
338 if not (os.path.exists(filename) and os.path.isfile(filename)):
351 return fp.hexdigest()
354 def FingerprintFiles(files):
355 """Compute fingerprints for a list of files.
358 @param files: the list of filename to fingerprint
360 @return: a dictionary filename: fingerprint, holding only
366 for filename in files:
367 cksum = _FingerprintFile(filename)
369 ret[filename] = cksum
374 def CheckDict(target, template, logname=None):
375 """Ensure a dictionary has a required set of keys.
377 For the given dictionaries I{target} and I{template}, ensure
378 I{target} has all the keys from I{template}. Missing keys are added
379 with values from template.
382 @param target: the dictionary to update
384 @param template: the dictionary holding the default values
385 @type logname: str or None
386 @param logname: if not None, causes the missing keys to be
387 logged with this name
394 target[k] = template[k]
396 if missing and logname:
397 logging.warning('%s missing keys %s', logname, ', '.join(missing))
400 def ForceDictType(target, key_types, allowed_values=None):
401 """Force the values of a dict to have certain types.
404 @param target: the dict to update
405 @type key_types: dict
406 @param key_types: dict mapping target dict keys to types
407 in constants.ENFORCEABLE_TYPES
408 @type allowed_values: list
409 @keyword allowed_values: list of specially allowed values
412 if allowed_values is None:
416 if key not in key_types:
417 msg = "Unknown key '%s'" % key
418 raise errors.TypeEnforcementError(msg)
420 if target[key] in allowed_values:
423 type = key_types[key]
424 if type not in constants.ENFORCEABLE_TYPES:
425 msg = "'%s' has non-enforceable type %s" % (key, type)
426 raise errors.ProgrammerError(msg)
428 if type == constants.VTYPE_STRING:
429 if not isinstance(target[key], basestring):
430 if isinstance(target[key], bool) and not target[key]:
433 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
434 raise errors.TypeEnforcementError(msg)
435 elif type == constants.VTYPE_BOOL:
436 if isinstance(target[key], basestring) and target[key]:
437 if target[key].lower() == constants.VALUE_FALSE:
439 elif target[key].lower() == constants.VALUE_TRUE:
442 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
443 raise errors.TypeEnforcementError(msg)
448 elif type == constants.VTYPE_SIZE:
450 target[key] = ParseUnit(target[key])
451 except errors.UnitParseError, err:
452 msg = "'%s' (value %s) is not a valid size. error: %s" % \
453 (key, target[key], err)
454 raise errors.TypeEnforcementError(msg)
455 elif type == constants.VTYPE_INT:
457 target[key] = int(target[key])
458 except (ValueError, TypeError):
459 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
460 raise errors.TypeEnforcementError(msg)
463 def IsProcessAlive(pid):
464 """Check if a given pid exists on the system.
466 @note: zombie status is not handled, so zombie processes
467 will be returned as alive
469 @param pid: the process ID to check
471 @return: True if the process exists
478 os.stat("/proc/%d/status" % pid)
480 except EnvironmentError, err:
481 if err.errno in (errno.ENOENT, errno.ENOTDIR):
486 def ReadPidFile(pidfile):
487 """Read a pid from a file.
489 @type pidfile: string
490 @param pidfile: path to the file containing the pid
492 @return: The process id, if the file exists and contains a valid PID,
497 pf = open(pidfile, 'r')
498 except EnvironmentError, err:
499 if err.errno != errno.ENOENT:
500 logging.exception("Can't read pid file?!")
505 except ValueError, err:
506 logging.info("Can't parse pid file contents", exc_info=True)
512 def MatchNameComponent(key, name_list):
513 """Try to match a name against a list.
515 This function will try to match a name like test1 against a list
516 like C{['test1.example.com', 'test2.example.com', ...]}. Against
517 this list, I{'test1'} as well as I{'test1.example'} will match, but
518 not I{'test1.ex'}. A multiple match will be considered as no match
519 at all (e.g. I{'test1'} against C{['test1.example.com',
520 'test1.example.org']}).
523 @param key: the name to be searched
524 @type name_list: list
525 @param name_list: the list of strings against which to search the key
528 @return: None if there is no match I{or} if there are multiple matches,
529 otherwise the element from the list which matches
532 mo = re.compile("^%s(\..*)?$" % re.escape(key))
533 names_filtered = [name for name in name_list if mo.match(name) is not None]
534 if len(names_filtered) != 1:
536 return names_filtered[0]
540 """Class implementing resolver and hostname functionality
543 def __init__(self, name=None):
544 """Initialize the host name object.
546 If the name argument is not passed, it will use this system's
551 name = self.SysName()
554 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
555 self.ip = self.ipaddrs[0]
558 """Returns the hostname without domain.
561 return self.name.split('.')[0]
565 """Return the current system's name.
567 This is simply a wrapper over C{socket.gethostname()}.
570 return socket.gethostname()
573 def LookupHostname(hostname):
577 @param hostname: hostname to look up
580 @return: a tuple (name, aliases, ipaddrs) as returned by
581 C{socket.gethostbyname_ex}
582 @raise errors.ResolverError: in case of errors in resolving
586 result = socket.gethostbyname_ex(hostname)
587 except socket.gaierror, err:
588 # hostname not found in DNS
589 raise errors.ResolverError(hostname, err.args[0], err.args[1])
594 def ListVolumeGroups():
595 """List volume groups and their size
599 Dictionary with keys volume name and values
600 the size of the volume
603 command = "vgs --noheadings --units m --nosuffix -o name,size"
604 result = RunCmd(command)
609 for line in result.stdout.splitlines():
611 name, size = line.split()
612 size = int(float(size))
613 except (IndexError, ValueError), err:
614 logging.error("Invalid output from vgs (%s): %s", err, line)
622 def BridgeExists(bridge):
623 """Check whether the given bridge exists in the system
626 @param bridge: the bridge name to check
628 @return: True if it does
631 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
634 def NiceSort(name_list):
635 """Sort a list of strings based on digit and non-digit groupings.
637 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
638 will sort the list in the logical order C{['a1', 'a2', 'a10',
641 The sort algorithm breaks each name in groups of either only-digits
642 or no-digits. Only the first eight such groups are considered, and
643 after that we just use what's left of the string.
645 @type name_list: list
646 @param name_list: the names to be sorted
648 @return: a copy of the name list sorted with our algorithm
651 _SORTER_BASE = "(\D+|\d+)"
652 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
653 _SORTER_BASE, _SORTER_BASE,
654 _SORTER_BASE, _SORTER_BASE,
655 _SORTER_BASE, _SORTER_BASE)
656 _SORTER_RE = re.compile(_SORTER_FULL)
657 _SORTER_NODIGIT = re.compile("^\D*$")
659 """Attempts to convert a variable to integer."""
660 if val is None or _SORTER_NODIGIT.match(val):
665 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
666 for name in name_list]
668 return [tup[1] for tup in to_sort]
671 def TryConvert(fn, val):
672 """Try to convert a value ignoring errors.
674 This function tries to apply function I{fn} to I{val}. If no
675 C{ValueError} or C{TypeError} exceptions are raised, it will return
676 the result, else it will return the original value. Any other
677 exceptions are propagated to the caller.
680 @param fn: function to apply to the value
681 @param val: the value to be converted
682 @return: The converted value if the conversion was successful,
683 otherwise the original value.
688 except (ValueError, TypeError):
694 """Verifies the syntax of an IPv4 address.
696 This function checks if the IPv4 address passes is valid or not based
697 on syntax (not IP range, class calculations, etc.).
700 @param ip: the address to be checked
701 @rtype: a regular expression match object
702 @return: a regular expression match object, or None if the
706 unit = "(0|[1-9]\d{0,2})"
707 #TODO: convert and return only boolean
708 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
711 def IsValidShellParam(word):
712 """Verifies is the given word is safe from the shell's p.o.v.
714 This means that we can pass this to a command via the shell and be
715 sure that it doesn't alter the command line and is passed as such to
718 Note that we are overly restrictive here, in order to be on the safe
722 @param word: the word to check
724 @return: True if the word is 'safe'
727 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
730 def BuildShellCmd(template, *args):
731 """Build a safe shell command line from the given arguments.
733 This function will check all arguments in the args list so that they
734 are valid shell parameters (i.e. they don't contain shell
735 metacharacters). If everything is ok, it will return the result of
739 @param template: the string holding the template for the
742 @return: the expanded command line
746 if not IsValidShellParam(word):
747 raise errors.ProgrammerError("Shell argument '%s' contains"
748 " invalid characters" % word)
749 return template % args
752 def FormatUnit(value, units):
753 """Formats an incoming number of MiB with the appropriate unit.
756 @param value: integer representing the value in MiB (1048576)
758 @param units: the type of formatting we should do:
759 - 'h' for automatic scaling
764 @return: the formatted value (with suffix)
767 if units not in ('m', 'g', 't', 'h'):
768 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
772 if units == 'm' or (units == 'h' and value < 1024):
775 return "%d%s" % (round(value, 0), suffix)
777 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
780 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
785 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
788 def ParseUnit(input_string):
789 """Tries to extract number and scale from the given string.
791 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
792 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
793 is always an int in MiB.
796 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
798 raise errors.UnitParseError("Invalid format")
800 value = float(m.groups()[0])
804 lcunit = unit.lower()
808 if lcunit in ('m', 'mb', 'mib'):
809 # Value already in MiB
812 elif lcunit in ('g', 'gb', 'gib'):
815 elif lcunit in ('t', 'tb', 'tib'):
819 raise errors.UnitParseError("Unknown unit: %s" % unit)
821 # Make sure we round up
822 if int(value) < value:
825 # Round up to the next multiple of 4
828 value += 4 - value % 4
833 def AddAuthorizedKey(file_name, key):
834 """Adds an SSH public key to an authorized_keys file.
837 @param file_name: path to authorized_keys file
839 @param key: string containing key
842 key_fields = key.split()
844 f = open(file_name, 'a+')
848 # Ignore whitespace changes
849 if line.split() == key_fields:
851 nl = line.endswith('\n')
855 f.write(key.rstrip('\r\n'))
862 def RemoveAuthorizedKey(file_name, key):
863 """Removes an SSH public key from an authorized_keys file.
866 @param file_name: path to authorized_keys file
868 @param key: string containing key
871 key_fields = key.split()
873 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
875 out = os.fdopen(fd, 'w')
877 f = open(file_name, 'r')
880 # Ignore whitespace changes while comparing lines
881 if line.split() != key_fields:
885 os.rename(tmpname, file_name)
895 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
896 """Sets the name of an IP address and hostname in /etc/hosts.
899 @param file_name: path to the file to modify (usually C{/etc/hosts})
901 @param ip: the IP address
903 @param hostname: the hostname to be added
905 @param aliases: the list of aliases to add for the hostname
908 # FIXME: use WriteFile + fn rather than duplicating its efforts
909 # Ensure aliases are unique
910 aliases = UniqueSequence([hostname] + aliases)[1:]
912 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
914 out = os.fdopen(fd, 'w')
916 f = open(file_name, 'r')
919 fields = line.split()
920 if fields and not fields[0].startswith('#') and ip == fields[0]:
924 out.write("%s\t%s" % (ip, hostname))
926 out.write(" %s" % ' '.join(aliases))
931 os.chmod(tmpname, 0644)
932 os.rename(tmpname, file_name)
942 def AddHostToEtcHosts(hostname):
943 """Wrapper around SetEtcHostsEntry.
946 @param hostname: a hostname that will be resolved and added to
947 L{constants.ETC_HOSTS}
950 hi = HostInfo(name=hostname)
951 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
954 def RemoveEtcHostsEntry(file_name, hostname):
955 """Removes a hostname from /etc/hosts.
957 IP addresses without names are removed from the file.
960 @param file_name: path to the file to modify (usually C{/etc/hosts})
962 @param hostname: the hostname to be removed
965 # FIXME: use WriteFile + fn rather than duplicating its efforts
966 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
968 out = os.fdopen(fd, 'w')
970 f = open(file_name, 'r')
973 fields = line.split()
974 if len(fields) > 1 and not fields[0].startswith('#'):
976 if hostname in names:
977 while hostname in names:
978 names.remove(hostname)
980 out.write("%s %s\n" % (fields[0], ' '.join(names)))
987 os.chmod(tmpname, 0644)
988 os.rename(tmpname, file_name)
998 def RemoveHostFromEtcHosts(hostname):
999 """Wrapper around RemoveEtcHostsEntry.
1002 @param hostname: hostname that will be resolved and its
1003 full and shot name will be removed from
1004 L{constants.ETC_HOSTS}
1007 hi = HostInfo(name=hostname)
1008 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1009 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1012 def CreateBackup(file_name):
1013 """Creates a backup of a file.
1015 @type file_name: str
1016 @param file_name: file to be backed up
1018 @return: the path to the newly created backup
1019 @raise errors.ProgrammerError: for invalid file names
1022 if not os.path.isfile(file_name):
1023 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1026 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1027 dir_name = os.path.dirname(file_name)
1029 fsrc = open(file_name, 'rb')
1031 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1032 fdst = os.fdopen(fd, 'wb')
1034 shutil.copyfileobj(fsrc, fdst)
1043 def ShellQuote(value):
1044 """Quotes shell argument according to POSIX.
1047 @param value: the argument to be quoted
1049 @return: the quoted value
1052 if _re_shell_unquoted.match(value):
1055 return "'%s'" % value.replace("'", "'\\''")
1058 def ShellQuoteArgs(args):
1059 """Quotes a list of shell arguments.
1062 @param args: list of arguments to be quoted
1064 @return: the quoted arguments concatenated with spaces
1067 return ' '.join([ShellQuote(i) for i in args])
1070 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1071 """Simple ping implementation using TCP connect(2).
1073 Check if the given IP is reachable by doing attempting a TCP connect
1077 @param target: the IP or hostname to ping
1079 @param port: the port to connect to
1081 @param timeout: the timeout on the connection attempt
1082 @type live_port_needed: boolean
1083 @param live_port_needed: whether a closed port will cause the
1084 function to return failure, as if there was a timeout
1085 @type source: str or None
1086 @param source: if specified, will cause the connect to be made
1087 from this specific source address; failures to bind other
1088 than C{EADDRNOTAVAIL} will be ignored
1091 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1095 if source is not None:
1097 sock.bind((source, 0))
1098 except socket.error, (errcode, _):
1099 if errcode == errno.EADDRNOTAVAIL:
1102 sock.settimeout(timeout)
1105 sock.connect((target, port))
1108 except socket.timeout:
1110 except socket.error, (errcode, errstring):
1111 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1116 def OwnIpAddress(address):
1117 """Check if the current host has the the given IP address.
1119 Currently this is done by TCP-pinging the address from the loopback
1122 @type address: string
1123 @param address: the address to check
1125 @return: True if we own the address
1128 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1129 source=constants.LOCALHOST_IP_ADDRESS)
1132 def ListVisibleFiles(path):
1133 """Returns a list of visible files in a directory.
1136 @param path: the directory to enumerate
1138 @return: the list of all files not starting with a dot
1141 files = [i for i in os.listdir(path) if not i.startswith(".")]
1146 def GetHomeDir(user, default=None):
1147 """Try to get the homedir of the given user.
1149 The user can be passed either as a string (denoting the name) or as
1150 an integer (denoting the user id). If the user is not found, the
1151 'default' argument is returned, which defaults to None.
1155 if isinstance(user, basestring):
1156 result = pwd.getpwnam(user)
1157 elif isinstance(user, (int, long)):
1158 result = pwd.getpwuid(user)
1160 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1164 return result.pw_dir
1168 """Returns a random UUID.
1170 @note: This is a Linux-specific method as it uses the /proc
1175 f = open("/proc/sys/kernel/random/uuid", "r")
1177 return f.read(128).rstrip("\n")
1182 def GenerateSecret():
1183 """Generates a random secret.
1185 This will generate a pseudo-random secret, and return its sha digest
1186 (so that it can be used where an ASCII string is needed).
1189 @return: a sha1 hexdigest of a block of 64 random bytes
1192 return sha1(os.urandom(64)).hexdigest()
1195 def EnsureDirs(dirs):
1196 """Make required directories, if they don't exist.
1198 @param dirs: list of tuples (dir_name, dir_mode)
1199 @type dirs: list of (string, integer)
1202 for dir_name, dir_mode in dirs:
1204 os.mkdir(dir_name, dir_mode)
1205 except EnvironmentError, err:
1206 if err.errno != errno.EEXIST:
1207 raise errors.GenericError("Cannot create needed directory"
1208 " '%s': %s" % (dir_name, err))
1209 if not os.path.isdir(dir_name):
1210 raise errors.GenericError("%s is not a directory" % dir_name)
1213 def ReadFile(file_name, size=None):
1216 @type size: None or int
1217 @param size: Read at most size bytes
1219 @return: the (possibly partial) content of the file
1222 f = open(file_name, "r")
1232 def WriteFile(file_name, fn=None, data=None,
1233 mode=None, uid=-1, gid=-1,
1234 atime=None, mtime=None, close=True,
1235 dry_run=False, backup=False,
1236 prewrite=None, postwrite=None):
1237 """(Over)write a file atomically.
1239 The file_name and either fn (a function taking one argument, the
1240 file descriptor, and which should write the data to it) or data (the
1241 contents of the file) must be passed. The other arguments are
1242 optional and allow setting the file mode, owner and group, and the
1243 mtime/atime of the file.
1245 If the function doesn't raise an exception, it has succeeded and the
1246 target file has the new contents. If the function has raised an
1247 exception, an existing target file should be unmodified and the
1248 temporary file should be removed.
1250 @type file_name: str
1251 @param file_name: the target filename
1253 @param fn: content writing function, called with
1254 file descriptor as parameter
1256 @param data: contents of the file
1258 @param mode: file mode
1260 @param uid: the owner of the file
1262 @param gid: the group of the file
1264 @param atime: a custom access time to be set on the file
1266 @param mtime: a custom modification time to be set on the file
1267 @type close: boolean
1268 @param close: whether to close file after writing it
1269 @type prewrite: callable
1270 @param prewrite: function to be called before writing content
1271 @type postwrite: callable
1272 @param postwrite: function to be called after writing content
1275 @return: None if the 'close' parameter evaluates to True,
1276 otherwise the file descriptor
1278 @raise errors.ProgrammerError: if any of the arguments are not valid
1281 if not os.path.isabs(file_name):
1282 raise errors.ProgrammerError("Path passed to WriteFile is not"
1283 " absolute: '%s'" % file_name)
1285 if [fn, data].count(None) != 1:
1286 raise errors.ProgrammerError("fn or data required")
1288 if [atime, mtime].count(None) == 1:
1289 raise errors.ProgrammerError("Both atime and mtime must be either"
1292 if backup and not dry_run and os.path.isfile(file_name):
1293 CreateBackup(file_name)
1295 dir_name, base_name = os.path.split(file_name)
1296 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1298 # here we need to make sure we remove the temp file, if any error
1299 # leaves it in place
1301 if uid != -1 or gid != -1:
1302 os.chown(new_name, uid, gid)
1304 os.chmod(new_name, mode)
1305 if callable(prewrite):
1307 if data is not None:
1311 if callable(postwrite):
1314 if atime is not None and mtime is not None:
1315 os.utime(new_name, (atime, mtime))
1317 os.rename(new_name, file_name)
1326 RemoveFile(new_name)
1331 def FirstFree(seq, base=0):
1332 """Returns the first non-existing integer from seq.
1334 The seq argument should be a sorted list of positive integers. The
1335 first time the index of an element is smaller than the element
1336 value, the index will be returned.
1338 The base argument is used to start at a different offset,
1339 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1341 Example: C{[0, 1, 3]} will return I{2}.
1344 @param seq: the sequence to be analyzed.
1346 @param base: use this value as the base index of the sequence
1348 @return: the first non-used index in the sequence
1351 for idx, elem in enumerate(seq):
1352 assert elem >= base, "Passed element is higher than base offset"
1353 if elem > idx + base:
1359 def all(seq, pred=bool):
1360 "Returns True if pred(x) is True for every element in the iterable"
1361 for _ in itertools.ifilterfalse(pred, seq):
1366 def any(seq, pred=bool):
1367 "Returns True if pred(x) is True for at least one element in the iterable"
1368 for _ in itertools.ifilter(pred, seq):
1373 def UniqueSequence(seq):
1374 """Returns a list with unique elements.
1376 Element order is preserved.
1379 @param seq: the sequence with the source elements
1381 @return: list of unique elements from seq
1385 return [i for i in seq if i not in seen and not seen.add(i)]
1388 def IsValidMac(mac):
1389 """Predicate to check if a MAC address is valid.
1391 Checks whether the supplied MAC address is formally correct, only
1392 accepts colon separated format.
1395 @param mac: the MAC to be validated
1397 @return: True is the MAC seems valid
1400 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1401 return mac_check.match(mac) is not None
1404 def TestDelay(duration):
1405 """Sleep for a fixed amount of time.
1407 @type duration: float
1408 @param duration: the sleep duration
1410 @return: False for negative value, True otherwise
1415 time.sleep(duration)
1419 def _CloseFDNoErr(fd, retries=5):
1420 """Close a file descriptor ignoring errors.
1423 @param fd: the file descriptor
1425 @param retries: how many retries to make, in case we get any
1426 other error than EBADF
1431 except OSError, err:
1432 if err.errno != errno.EBADF:
1434 _CloseFDNoErr(fd, retries - 1)
1435 # else either it's closed already or we're out of retries, so we
1436 # ignore this and go on
1439 def CloseFDs(noclose_fds=None):
1440 """Close file descriptors.
1442 This closes all file descriptors above 2 (i.e. except
1445 @type noclose_fds: list or None
1446 @param noclose_fds: if given, it denotes a list of file descriptor
1447 that should not be closed
1450 # Default maximum for the number of available file descriptors.
1451 if 'SC_OPEN_MAX' in os.sysconf_names:
1453 MAXFD = os.sysconf('SC_OPEN_MAX')
1460 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1461 if (maxfd == resource.RLIM_INFINITY):
1464 # Iterate through and close all file descriptors (except the standard ones)
1465 for fd in range(3, maxfd):
1466 if noclose_fds and fd in noclose_fds:
1471 def Daemonize(logfile):
1472 """Daemonize the current process.
1474 This detaches the current process from the controlling terminal and
1475 runs it in the background as a daemon.
1478 @param logfile: the logfile to which we should redirect stdout/stderr
1480 @return: the value zero
1488 if (pid == 0): # The first child.
1491 pid = os.fork() # Fork a second child.
1492 if (pid == 0): # The second child.
1496 # exit() or _exit()? See below.
1497 os._exit(0) # Exit parent (the first child) of the second child.
1499 os._exit(0) # Exit parent of the first child.
1503 i = os.open("/dev/null", os.O_RDONLY) # stdin
1504 assert i == 0, "Can't close/reopen stdin"
1505 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1506 assert i == 1, "Can't close/reopen stdout"
1507 # Duplicate standard output to standard error.
1512 def DaemonPidFileName(name):
1513 """Compute a ganeti pid file absolute path
1516 @param name: the daemon name
1518 @return: the full path to the pidfile corresponding to the given
1522 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1525 def WritePidFile(name):
1526 """Write the current process pidfile.
1528 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1531 @param name: the daemon name to use
1532 @raise errors.GenericError: if the pid file already exists and
1533 points to a live process
1537 pidfilename = DaemonPidFileName(name)
1538 if IsProcessAlive(ReadPidFile(pidfilename)):
1539 raise errors.GenericError("%s contains a live process" % pidfilename)
1541 WriteFile(pidfilename, data="%d\n" % pid)
1544 def RemovePidFile(name):
1545 """Remove the current process pidfile.
1547 Any errors are ignored.
1550 @param name: the daemon name used to derive the pidfile name
1553 pidfilename = DaemonPidFileName(name)
1554 # TODO: we could check here that the file contains our pid
1556 RemoveFile(pidfilename)
1561 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1563 """Kill a process given by its pid.
1566 @param pid: The PID to terminate.
1568 @param signal_: The signal to send, by default SIGTERM
1570 @param timeout: The timeout after which, if the process is still alive,
1571 a SIGKILL will be sent. If not positive, no such checking
1573 @type waitpid: boolean
1574 @param waitpid: If true, we should waitpid on this process after
1575 sending signals, since it's our own child and otherwise it
1576 would remain as zombie
1579 def _helper(pid, signal_, wait):
1580 """Simple helper to encapsulate the kill/waitpid sequence"""
1581 os.kill(pid, signal_)
1584 os.waitpid(pid, os.WNOHANG)
1589 # kill with pid=0 == suicide
1590 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1592 if not IsProcessAlive(pid):
1594 _helper(pid, signal_, waitpid)
1598 # Wait up to $timeout seconds
1599 end = time.time() + timeout
1601 while time.time() < end and IsProcessAlive(pid):
1603 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1609 # Make wait time longer for next try
1613 if IsProcessAlive(pid):
1614 # Kill process if it's still alive
1615 _helper(pid, signal.SIGKILL, waitpid)
1618 def FindFile(name, search_path, test=os.path.exists):
1619 """Look for a filesystem object in a given path.
1621 This is an abstract method to search for filesystem object (files,
1622 dirs) under a given search path.
1625 @param name: the name to look for
1626 @type search_path: str
1627 @param search_path: location to start at
1628 @type test: callable
1629 @param test: a function taking one argument that should return True
1630 if the a given object is valid; the default value is
1631 os.path.exists, causing only existing files to be returned
1633 @return: full path to the object if found, None otherwise
1636 for dir_name in search_path:
1637 item_name = os.path.sep.join([dir_name, name])
1643 def CheckVolumeGroupSize(vglist, vgname, minsize):
1644 """Checks if the volume group list is valid.
1646 The function will check if a given volume group is in the list of
1647 volume groups and has a minimum size.
1650 @param vglist: dictionary of volume group names and their size
1652 @param vgname: the volume group we should check
1654 @param minsize: the minimum size we accept
1656 @return: None for success, otherwise the error message
1659 vgsize = vglist.get(vgname, None)
1661 return "volume group '%s' missing" % vgname
1662 elif vgsize < minsize:
1663 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1664 (vgname, minsize, vgsize))
1668 def SplitTime(value):
1669 """Splits time as floating point number into a tuple.
1671 @param value: Time in seconds
1672 @type value: int or float
1673 @return: Tuple containing (seconds, microseconds)
1676 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1678 assert 0 <= seconds, \
1679 "Seconds must be larger than or equal to 0, but are %s" % seconds
1680 assert 0 <= microseconds <= 999999, \
1681 "Microseconds must be 0-999999, but are %s" % microseconds
1683 return (int(seconds), int(microseconds))
1686 def MergeTime(timetuple):
1687 """Merges a tuple into time as a floating point number.
1689 @param timetuple: Time as tuple, (seconds, microseconds)
1690 @type timetuple: tuple
1691 @return: Time as a floating point number expressed in seconds
1694 (seconds, microseconds) = timetuple
1696 assert 0 <= seconds, \
1697 "Seconds must be larger than or equal to 0, but are %s" % seconds
1698 assert 0 <= microseconds <= 999999, \
1699 "Microseconds must be 0-999999, but are %s" % microseconds
1701 return float(seconds) + (float(microseconds) * 0.000001)
1704 def GetNodeDaemonPort():
1705 """Get the node daemon port for this cluster.
1707 Note that this routine does not read a ganeti-specific file, but
1708 instead uses C{socket.getservbyname} to allow pre-customization of
1709 this parameter outside of Ganeti.
1715 port = socket.getservbyname("ganeti-noded", "tcp")
1716 except socket.error:
1717 port = constants.DEFAULT_NODED_PORT
1722 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1723 multithreaded=False):
1724 """Configures the logging module.
1727 @param logfile: the filename to which we should log
1728 @type debug: boolean
1729 @param debug: whether to enable debug messages too or
1730 only those at C{INFO} and above level
1731 @type stderr_logging: boolean
1732 @param stderr_logging: whether we should also log to the standard error
1734 @param program: the name under which we should log messages
1735 @type multithreaded: boolean
1736 @param multithreaded: if True, will add the thread name to the log file
1737 @raise EnvironmentError: if we can't open the log file and
1738 stderr logging is disabled
1741 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1743 fmt += "/%(threadName)s"
1745 fmt += " %(module)s:%(lineno)s"
1746 fmt += " %(levelname)s %(message)s"
1747 formatter = logging.Formatter(fmt)
1749 root_logger = logging.getLogger("")
1750 root_logger.setLevel(logging.NOTSET)
1752 # Remove all previously setup handlers
1753 for handler in root_logger.handlers:
1755 root_logger.removeHandler(handler)
1758 stderr_handler = logging.StreamHandler()
1759 stderr_handler.setFormatter(formatter)
1761 stderr_handler.setLevel(logging.NOTSET)
1763 stderr_handler.setLevel(logging.CRITICAL)
1764 root_logger.addHandler(stderr_handler)
1766 # this can fail, if the logging directories are not setup or we have
1767 # a permisssion problem; in this case, it's best to log but ignore
1768 # the error if stderr_logging is True, and if false we re-raise the
1769 # exception since otherwise we could run but without any logs at all
1771 logfile_handler = logging.FileHandler(logfile)
1772 logfile_handler.setFormatter(formatter)
1774 logfile_handler.setLevel(logging.DEBUG)
1776 logfile_handler.setLevel(logging.INFO)
1777 root_logger.addHandler(logfile_handler)
1778 except EnvironmentError:
1780 logging.exception("Failed to enable logging to file '%s'", logfile)
1782 # we need to re-raise the exception
1786 def TailFile(fname, lines=20):
1787 """Return the last lines from a file.
1789 @note: this function will only read and parse the last 4KB of
1790 the file; if the lines are very long, it could be that less
1791 than the requested number of lines are returned
1793 @param fname: the file name
1795 @param lines: the (maximum) number of lines to return
1798 fd = open(fname, "r")
1802 pos = max(0, pos-4096)
1804 raw_data = fd.read()
1808 rows = raw_data.splitlines()
1809 return rows[-lines:]
1812 def SafeEncode(text):
1813 """Return a 'safe' version of a source string.
1815 This function mangles the input string and returns a version that
1816 should be safe to display/encode as ASCII. To this end, we first
1817 convert it to ASCII using the 'backslashreplace' encoding which
1818 should get rid of any non-ASCII chars, and then we process it
1819 through a loop copied from the string repr sources in the python; we
1820 don't use string_escape anymore since that escape single quotes and
1821 backslashes too, and that is too much; and that escaping is not
1822 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
1824 @type text: str or unicode
1825 @param text: input data
1827 @return: a safe version of text
1830 if isinstance(text, unicode):
1831 # only if unicode; if str already, we handle it below
1832 text = text.encode('ascii', 'backslashreplace')
1842 elif c < 32 or c >= 127: # non-printable
1843 resu += "\\x%02x" % (c & 0xff)
1849 def CommaJoin(names):
1850 """Nicely join a set of identifiers.
1852 @param names: set, list or tuple
1853 @return: a string with the formatted results
1856 return ", ".join(["'%s'" % val for val in names])
1859 def LockedMethod(fn):
1860 """Synchronized object access decorator.
1862 This decorator is intended to protect access to an object using the
1863 object's own lock which is hardcoded to '_lock'.
1866 def _LockDebug(*args, **kwargs):
1868 logging.debug(*args, **kwargs)
1870 def wrapper(self, *args, **kwargs):
1871 assert hasattr(self, '_lock')
1873 _LockDebug("Waiting for %s", lock)
1876 _LockDebug("Acquired %s", lock)
1877 result = fn(self, *args, **kwargs)
1879 _LockDebug("Releasing %s", lock)
1881 _LockDebug("Released %s", lock)
1887 """Locks a file using POSIX locks.
1890 @param fd: the file descriptor we need to lock
1894 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1895 except IOError, err:
1896 if err.errno == errno.EAGAIN:
1897 raise errors.LockError("File already locked")
1901 class FileLock(object):
1902 """Utility class for file locks.
1905 def __init__(self, filename):
1906 """Constructor for FileLock.
1908 This will open the file denoted by the I{filename} argument.
1911 @param filename: path to the file to be locked
1914 self.filename = filename
1915 self.fd = open(self.filename, "w")
1921 """Close the file and release the lock.
1928 def _flock(self, flag, blocking, timeout, errmsg):
1929 """Wrapper for fcntl.flock.
1932 @param flag: operation flag
1933 @type blocking: bool
1934 @param blocking: whether the operation should be done in blocking mode.
1935 @type timeout: None or float
1936 @param timeout: for how long the operation should be retried (implies
1938 @type errmsg: string
1939 @param errmsg: error message in case operation fails.
1942 assert self.fd, "Lock was closed"
1943 assert timeout is None or timeout >= 0, \
1944 "If specified, timeout must be positive"
1946 if timeout is not None:
1947 flag |= fcntl.LOCK_NB
1948 timeout_end = time.time() + timeout
1950 # Blocking doesn't have effect with timeout
1952 flag |= fcntl.LOCK_NB
1958 fcntl.flock(self.fd, flag)
1960 except IOError, err:
1961 if err.errno in (errno.EAGAIN, ):
1962 if timeout_end is not None and time.time() < timeout_end:
1963 # Wait before trying again
1964 time.sleep(max(0.1, min(1.0, timeout)))
1966 raise errors.LockError(errmsg)
1968 logging.exception("fcntl.flock failed")
1971 def Exclusive(self, blocking=False, timeout=None):
1972 """Locks the file in exclusive mode.
1974 @type blocking: boolean
1975 @param blocking: whether to block and wait until we
1976 can lock the file or return immediately
1977 @type timeout: int or None
1978 @param timeout: if not None, the duration to wait for the lock
1982 self._flock(fcntl.LOCK_EX, blocking, timeout,
1983 "Failed to lock %s in exclusive mode" % self.filename)
1985 def Shared(self, blocking=False, timeout=None):
1986 """Locks the file in shared mode.
1988 @type blocking: boolean
1989 @param blocking: whether to block and wait until we
1990 can lock the file or return immediately
1991 @type timeout: int or None
1992 @param timeout: if not None, the duration to wait for the lock
1996 self._flock(fcntl.LOCK_SH, blocking, timeout,
1997 "Failed to lock %s in shared mode" % self.filename)
1999 def Unlock(self, blocking=True, timeout=None):
2000 """Unlocks the file.
2002 According to C{flock(2)}, unlocking can also be a nonblocking
2005 To make a non-blocking request, include LOCK_NB with any of the above
2008 @type blocking: boolean
2009 @param blocking: whether to block and wait until we
2010 can lock the file or return immediately
2011 @type timeout: int or None
2012 @param timeout: if not None, the duration to wait for the lock
2016 self._flock(fcntl.LOCK_UN, blocking, timeout,
2017 "Failed to unlock %s" % self.filename)
2020 class SignalHandler(object):
2021 """Generic signal handler class.
2023 It automatically restores the original handler when deconstructed or
2024 when L{Reset} is called. You can either pass your own handler
2025 function in or query the L{called} attribute to detect whether the
2029 @ivar signum: the signals we handle
2030 @type called: boolean
2031 @ivar called: tracks whether any of the signals have been raised
2034 def __init__(self, signum):
2035 """Constructs a new SignalHandler instance.
2037 @type signum: int or list of ints
2038 @param signum: Single signal number or set of signal numbers
2041 if isinstance(signum, (int, long)):
2042 self.signum = set([signum])
2044 self.signum = set(signum)
2050 for signum in self.signum:
2052 prev_handler = signal.signal(signum, self._HandleSignal)
2054 self._previous[signum] = prev_handler
2056 # Restore previous handler
2057 signal.signal(signum, prev_handler)
2060 # Reset all handlers
2062 # Here we have a race condition: a handler may have already been called,
2063 # but there's not much we can do about it at this point.
2070 """Restore previous handler.
2072 This will reset all the signals to their previous handlers.
2075 for signum, prev_handler in self._previous.items():
2076 signal.signal(signum, prev_handler)
2077 # If successful, remove from dict
2078 del self._previous[signum]
2081 """Unsets the L{called} flag.
2083 This function can be used in case a signal may arrive several times.
2088 def _HandleSignal(self, signum, frame):
2089 """Actual signal handling function.
2092 # This is not nice and not absolutely atomic, but it appears to be the only
2093 # solution in Python -- there are no atomic types.
2097 class FieldSet(object):
2098 """A simple field set.
2100 Among the features are:
2101 - checking if a string is among a list of static string or regex objects
2102 - checking if a whole list of string matches
2103 - returning the matching groups from a regex match
2105 Internally, all fields are held as regular expression objects.
2108 def __init__(self, *items):
2109 self.items = [re.compile("^%s$" % value) for value in items]
2111 def Extend(self, other_set):
2112 """Extend the field set with the items from another one"""
2113 self.items.extend(other_set.items)
2115 def Matches(self, field):
2116 """Checks if a field matches the current set
2119 @param field: the string to match
2120 @return: either False or a regular expression match object
2123 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2127 def NonMatching(self, items):
2128 """Returns the list of fields not matching the current set
2131 @param items: the list of fields to check
2133 @return: list of non-matching fields
2136 return [val for val in items if not self.Matches(val)]