4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
48 from cStringIO import StringIO
50 from ganeti import errors
51 from ganeti import constants
55 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60 #: when set to True, L{RunCmd} is disabled
64 class RunResult(object):
65 """Holds the result of running external programs.
68 @ivar exit_code: the exit code of the program, or None (if the program
70 @type signal: int or None
71 @ivar signal: the signal that caused the program to finish, or None
72 (if the program wasn't terminated by a signal)
74 @ivar stdout: the standard output of the program
76 @ivar stderr: the standard error of the program
78 @ivar failed: True in case the program was
79 terminated by a signal or exited with a non-zero exit code
80 @ivar fail_reason: a string detailing the termination reason
83 __slots__ = ["exit_code", "signal", "stdout", "stderr",
84 "failed", "fail_reason", "cmd"]
87 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
89 self.exit_code = exit_code
93 self.failed = (signal_ is not None or exit_code != 0)
95 if self.signal is not None:
96 self.fail_reason = "terminated by signal %s" % self.signal
97 elif self.exit_code is not None:
98 self.fail_reason = "exited with exit code %s" % self.exit_code
100 self.fail_reason = "unable to determine termination reason"
103 logging.debug("Command '%s' failed (%s); output: %s",
104 self.cmd, self.fail_reason, self.output)
106 def _GetOutput(self):
107 """Returns the combined stdout and stderr for easier usage.
110 return self.stdout + self.stderr
112 output = property(_GetOutput, None, None, "Return full output")
115 def RunCmd(cmd, env=None, output=None, cwd='/'):
116 """Execute a (shell) command.
118 The command should not read from its standard input, as it will be
121 @type cmd: string or list
122 @param cmd: Command to run
124 @param env: Additional environment
126 @param output: if desired, the output of the command can be
127 saved in a file instead of the RunResult instance; this
128 parameter denotes the file name (if not None)
130 @param cwd: if specified, will be used as the working
131 directory for the command; the default will be /
133 @return: RunResult instance
134 @raise erors.ProgrammerError: if we call this when forks are disabled
138 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
140 if isinstance(cmd, list):
141 cmd = [str(val) for val in cmd]
142 strcmd = " ".join(cmd)
147 logging.debug("RunCmd '%s'", strcmd)
149 cmd_env = os.environ.copy()
150 cmd_env["LC_ALL"] = "C"
155 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
157 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
167 return RunResult(exitcode, signal_, out, err, strcmd)
170 def _RunCmdPipe(cmd, env, via_shell, cwd):
171 """Run a command and return its output.
173 @type cmd: string or list
174 @param cmd: Command to run
176 @param env: The environment to use
177 @type via_shell: bool
178 @param via_shell: if we should run via the shell
180 @param cwd: the working directory for the program
182 @return: (out, err, status)
185 poller = select.poll()
186 child = subprocess.Popen(cmd, shell=via_shell,
187 stderr=subprocess.PIPE,
188 stdout=subprocess.PIPE,
189 stdin=subprocess.PIPE,
190 close_fds=True, env=env,
194 poller.register(child.stdout, select.POLLIN)
195 poller.register(child.stderr, select.POLLIN)
199 child.stdout.fileno(): (out, child.stdout),
200 child.stderr.fileno(): (err, child.stderr),
203 status = fcntl.fcntl(fd, fcntl.F_GETFL)
204 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
208 pollresult = poller.poll()
209 except EnvironmentError, eerr:
210 if eerr.errno == errno.EINTR:
213 except select.error, serr:
214 if serr[0] == errno.EINTR:
218 for fd, event in pollresult:
219 if event & select.POLLIN or event & select.POLLPRI:
220 data = fdmap[fd][1].read()
221 # no data from read signifies EOF (the same as POLLHUP)
223 poller.unregister(fd)
226 fdmap[fd][0].write(data)
227 if (event & select.POLLNVAL or event & select.POLLHUP or
228 event & select.POLLERR):
229 poller.unregister(fd)
235 status = child.wait()
236 return out, err, status
239 def _RunCmdFile(cmd, env, via_shell, output, cwd):
240 """Run a command and save its output to a file.
242 @type cmd: string or list
243 @param cmd: Command to run
245 @param env: The environment to use
246 @type via_shell: bool
247 @param via_shell: if we should run via the shell
249 @param output: the filename in which to save the output
251 @param cwd: the working directory for the program
253 @return: the exit status
256 fh = open(output, "a")
258 child = subprocess.Popen(cmd, shell=via_shell,
259 stderr=subprocess.STDOUT,
261 stdin=subprocess.PIPE,
262 close_fds=True, env=env,
266 status = child.wait()
272 def RemoveFile(filename):
273 """Remove a file ignoring some errors.
275 Remove a file, ignoring non-existing ones or directories. Other
279 @param filename: the file to be removed
285 if err.errno not in (errno.ENOENT, errno.EISDIR):
289 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
293 @param old: Original path
297 @param mkdir: Whether to create target directory if it doesn't exist
298 @type mkdir_mode: int
299 @param mkdir_mode: Mode for newly created directories
303 return os.rename(old, new)
305 # In at least one use case of this function, the job queue, directory
306 # creation is very rare. Checking for the directory before renaming is not
308 if mkdir and err.errno == errno.ENOENT:
309 # Create directory and try again
310 os.makedirs(os.path.dirname(new), mkdir_mode)
311 return os.rename(old, new)
315 def _FingerprintFile(filename):
316 """Compute the fingerprint of a file.
318 If the file does not exist, a None will be returned
322 @param filename: the filename to checksum
324 @return: the hex digest of the sha checksum of the contents
328 if not (os.path.exists(filename) and os.path.isfile(filename)):
341 return fp.hexdigest()
344 def FingerprintFiles(files):
345 """Compute fingerprints for a list of files.
348 @param files: the list of filename to fingerprint
350 @return: a dictionary filename: fingerprint, holding only
356 for filename in files:
357 cksum = _FingerprintFile(filename)
359 ret[filename] = cksum
364 def CheckDict(target, template, logname=None):
365 """Ensure a dictionary has a required set of keys.
367 For the given dictionaries I{target} and I{template}, ensure
368 I{target} has all the keys from I{template}. Missing keys are added
369 with values from template.
372 @param target: the dictionary to update
374 @param template: the dictionary holding the default values
375 @type logname: str or None
376 @param logname: if not None, causes the missing keys to be
377 logged with this name
384 target[k] = template[k]
386 if missing and logname:
387 logging.warning('%s missing keys %s', logname, ', '.join(missing))
390 def ForceDictType(target, key_types, allowed_values=None):
391 """Force the values of a dict to have certain types.
394 @param target: the dict to update
395 @type key_types: dict
396 @param key_types: dict mapping target dict keys to types
397 in constants.ENFORCEABLE_TYPES
398 @type allowed_values: list
399 @keyword allowed_values: list of specially allowed values
402 if allowed_values is None:
406 if key not in key_types:
407 msg = "Unknown key '%s'" % key
408 raise errors.TypeEnforcementError(msg)
410 if target[key] in allowed_values:
413 type = key_types[key]
414 if type not in constants.ENFORCEABLE_TYPES:
415 msg = "'%s' has non-enforceable type %s" % (key, type)
416 raise errors.ProgrammerError(msg)
418 if type == constants.VTYPE_STRING:
419 if not isinstance(target[key], basestring):
420 if isinstance(target[key], bool) and not target[key]:
423 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
424 raise errors.TypeEnforcementError(msg)
425 elif type == constants.VTYPE_BOOL:
426 if isinstance(target[key], basestring) and target[key]:
427 if target[key].lower() == constants.VALUE_FALSE:
429 elif target[key].lower() == constants.VALUE_TRUE:
432 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
433 raise errors.TypeEnforcementError(msg)
438 elif type == constants.VTYPE_SIZE:
440 target[key] = ParseUnit(target[key])
441 except errors.UnitParseError, err:
442 msg = "'%s' (value %s) is not a valid size. error: %s" % \
443 (key, target[key], err)
444 raise errors.TypeEnforcementError(msg)
445 elif type == constants.VTYPE_INT:
447 target[key] = int(target[key])
448 except (ValueError, TypeError):
449 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
450 raise errors.TypeEnforcementError(msg)
453 def IsProcessAlive(pid):
454 """Check if a given pid exists on the system.
456 @note: zombie status is not handled, so zombie processes
457 will be returned as alive
459 @param pid: the process ID to check
461 @return: True if the process exists
468 os.stat("/proc/%d/status" % pid)
470 except EnvironmentError, err:
471 if err.errno in (errno.ENOENT, errno.ENOTDIR):
476 def ReadPidFile(pidfile):
477 """Read a pid from a file.
479 @type pidfile: string
480 @param pidfile: path to the file containing the pid
482 @return: The process id, if the file exists and contains a valid PID,
487 pf = open(pidfile, 'r')
488 except EnvironmentError, err:
489 if err.errno != errno.ENOENT:
490 logging.exception("Can't read pid file?!")
495 except ValueError, err:
496 logging.info("Can't parse pid file contents", exc_info=True)
502 def MatchNameComponent(key, name_list):
503 """Try to match a name against a list.
505 This function will try to match a name like test1 against a list
506 like C{['test1.example.com', 'test2.example.com', ...]}. Against
507 this list, I{'test1'} as well as I{'test1.example'} will match, but
508 not I{'test1.ex'}. A multiple match will be considered as no match
509 at all (e.g. I{'test1'} against C{['test1.example.com',
510 'test1.example.org']}).
513 @param key: the name to be searched
514 @type name_list: list
515 @param name_list: the list of strings against which to search the key
518 @return: None if there is no match I{or} if there are multiple matches,
519 otherwise the element from the list which matches
522 mo = re.compile("^%s(\..*)?$" % re.escape(key))
523 names_filtered = [name for name in name_list if mo.match(name) is not None]
524 if len(names_filtered) != 1:
526 return names_filtered[0]
530 """Class implementing resolver and hostname functionality
533 def __init__(self, name=None):
534 """Initialize the host name object.
536 If the name argument is not passed, it will use this system's
541 name = self.SysName()
544 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
545 self.ip = self.ipaddrs[0]
548 """Returns the hostname without domain.
551 return self.name.split('.')[0]
555 """Return the current system's name.
557 This is simply a wrapper over C{socket.gethostname()}.
560 return socket.gethostname()
563 def LookupHostname(hostname):
567 @param hostname: hostname to look up
570 @return: a tuple (name, aliases, ipaddrs) as returned by
571 C{socket.gethostbyname_ex}
572 @raise errors.ResolverError: in case of errors in resolving
576 result = socket.gethostbyname_ex(hostname)
577 except socket.gaierror, err:
578 # hostname not found in DNS
579 raise errors.ResolverError(hostname, err.args[0], err.args[1])
584 def ListVolumeGroups():
585 """List volume groups and their size
589 Dictionary with keys volume name and values
590 the size of the volume
593 command = "vgs --noheadings --units m --nosuffix -o name,size"
594 result = RunCmd(command)
599 for line in result.stdout.splitlines():
601 name, size = line.split()
602 size = int(float(size))
603 except (IndexError, ValueError), err:
604 logging.error("Invalid output from vgs (%s): %s", err, line)
612 def BridgeExists(bridge):
613 """Check whether the given bridge exists in the system
616 @param bridge: the bridge name to check
618 @return: True if it does
621 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
624 def NiceSort(name_list):
625 """Sort a list of strings based on digit and non-digit groupings.
627 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
628 will sort the list in the logical order C{['a1', 'a2', 'a10',
631 The sort algorithm breaks each name in groups of either only-digits
632 or no-digits. Only the first eight such groups are considered, and
633 after that we just use what's left of the string.
635 @type name_list: list
636 @param name_list: the names to be sorted
638 @return: a copy of the name list sorted with our algorithm
641 _SORTER_BASE = "(\D+|\d+)"
642 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
643 _SORTER_BASE, _SORTER_BASE,
644 _SORTER_BASE, _SORTER_BASE,
645 _SORTER_BASE, _SORTER_BASE)
646 _SORTER_RE = re.compile(_SORTER_FULL)
647 _SORTER_NODIGIT = re.compile("^\D*$")
649 """Attempts to convert a variable to integer."""
650 if val is None or _SORTER_NODIGIT.match(val):
655 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
656 for name in name_list]
658 return [tup[1] for tup in to_sort]
661 def TryConvert(fn, val):
662 """Try to convert a value ignoring errors.
664 This function tries to apply function I{fn} to I{val}. If no
665 C{ValueError} or C{TypeError} exceptions are raised, it will return
666 the result, else it will return the original value. Any other
667 exceptions are propagated to the caller.
670 @param fn: function to apply to the value
671 @param val: the value to be converted
672 @return: The converted value if the conversion was successful,
673 otherwise the original value.
678 except (ValueError, TypeError), err:
684 """Verifies the syntax of an IPv4 address.
686 This function checks if the IPv4 address passes is valid or not based
687 on syntax (not IP range, class calculations, etc.).
690 @param ip: the address to be checked
691 @rtype: a regular expression match object
692 @return: a regular epression match object, or None if the
696 unit = "(0|[1-9]\d{0,2})"
697 #TODO: convert and return only boolean
698 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
701 def IsValidShellParam(word):
702 """Verifies is the given word is safe from the shell's p.o.v.
704 This means that we can pass this to a command via the shell and be
705 sure that it doesn't alter the command line and is passed as such to
708 Note that we are overly restrictive here, in order to be on the safe
712 @param word: the word to check
714 @return: True if the word is 'safe'
717 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
720 def BuildShellCmd(template, *args):
721 """Build a safe shell command line from the given arguments.
723 This function will check all arguments in the args list so that they
724 are valid shell parameters (i.e. they don't contain shell
725 metacharaters). If everything is ok, it will return the result of
729 @param template: the string holding the template for the
732 @return: the expanded command line
736 if not IsValidShellParam(word):
737 raise errors.ProgrammerError("Shell argument '%s' contains"
738 " invalid characters" % word)
739 return template % args
742 def FormatUnit(value, units):
743 """Formats an incoming number of MiB with the appropriate unit.
746 @param value: integer representing the value in MiB (1048576)
748 @param units: the type of formatting we should do:
749 - 'h' for automatic scaling
754 @return: the formatted value (with suffix)
757 if units not in ('m', 'g', 't', 'h'):
758 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
762 if units == 'm' or (units == 'h' and value < 1024):
765 return "%d%s" % (round(value, 0), suffix)
767 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
770 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
775 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
778 def ParseUnit(input_string):
779 """Tries to extract number and scale from the given string.
781 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
782 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
783 is always an int in MiB.
786 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
788 raise errors.UnitParseError("Invalid format")
790 value = float(m.groups()[0])
794 lcunit = unit.lower()
798 if lcunit in ('m', 'mb', 'mib'):
799 # Value already in MiB
802 elif lcunit in ('g', 'gb', 'gib'):
805 elif lcunit in ('t', 'tb', 'tib'):
809 raise errors.UnitParseError("Unknown unit: %s" % unit)
811 # Make sure we round up
812 if int(value) < value:
815 # Round up to the next multiple of 4
818 value += 4 - value % 4
823 def AddAuthorizedKey(file_name, key):
824 """Adds an SSH public key to an authorized_keys file.
827 @param file_name: path to authorized_keys file
829 @param key: string containing key
832 key_fields = key.split()
834 f = open(file_name, 'a+')
838 # Ignore whitespace changes
839 if line.split() == key_fields:
841 nl = line.endswith('\n')
845 f.write(key.rstrip('\r\n'))
852 def RemoveAuthorizedKey(file_name, key):
853 """Removes an SSH public key from an authorized_keys file.
856 @param file_name: path to authorized_keys file
858 @param key: string containing key
861 key_fields = key.split()
863 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
865 out = os.fdopen(fd, 'w')
867 f = open(file_name, 'r')
870 # Ignore whitespace changes while comparing lines
871 if line.split() != key_fields:
875 os.rename(tmpname, file_name)
885 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
886 """Sets the name of an IP address and hostname in /etc/hosts.
889 @param file_name: path to the file to modify (usually C{/etc/hosts})
891 @param ip: the IP address
893 @param hostname: the hostname to be added
895 @param aliases: the list of aliases to add for the hostname
898 # Ensure aliases are unique
899 aliases = UniqueSequence([hostname] + aliases)[1:]
901 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
903 out = os.fdopen(fd, 'w')
905 f = open(file_name, 'r')
908 fields = line.split()
909 if fields and not fields[0].startswith('#') and ip == fields[0]:
913 out.write("%s\t%s" % (ip, hostname))
915 out.write(" %s" % ' '.join(aliases))
920 os.rename(tmpname, file_name)
930 def AddHostToEtcHosts(hostname):
931 """Wrapper around SetEtcHostsEntry.
934 @param hostname: a hostname that will be resolved and added to
935 L{constants.ETC_HOSTS}
938 hi = HostInfo(name=hostname)
939 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
942 def RemoveEtcHostsEntry(file_name, hostname):
943 """Removes a hostname from /etc/hosts.
945 IP addresses without names are removed from the file.
948 @param file_name: path to the file to modify (usually C{/etc/hosts})
950 @param hostname: the hostname to be removed
953 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
955 out = os.fdopen(fd, 'w')
957 f = open(file_name, 'r')
960 fields = line.split()
961 if len(fields) > 1 and not fields[0].startswith('#'):
963 if hostname in names:
964 while hostname in names:
965 names.remove(hostname)
967 out.write("%s %s\n" % (fields[0], ' '.join(names)))
974 os.rename(tmpname, file_name)
984 def RemoveHostFromEtcHosts(hostname):
985 """Wrapper around RemoveEtcHostsEntry.
988 @param hostname: hostname that will be resolved and its
989 full and shot name will be removed from
990 L{constants.ETC_HOSTS}
993 hi = HostInfo(name=hostname)
994 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
995 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
998 def CreateBackup(file_name):
999 """Creates a backup of a file.
1001 @type file_name: str
1002 @param file_name: file to be backed up
1004 @return: the path to the newly created backup
1005 @raise errors.ProgrammerError: for invalid file names
1008 if not os.path.isfile(file_name):
1009 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1012 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1013 dir_name = os.path.dirname(file_name)
1015 fsrc = open(file_name, 'rb')
1017 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1018 fdst = os.fdopen(fd, 'wb')
1020 shutil.copyfileobj(fsrc, fdst)
1029 def ShellQuote(value):
1030 """Quotes shell argument according to POSIX.
1033 @param value: the argument to be quoted
1035 @return: the quoted value
1038 if _re_shell_unquoted.match(value):
1041 return "'%s'" % value.replace("'", "'\\''")
1044 def ShellQuoteArgs(args):
1045 """Quotes a list of shell arguments.
1048 @param args: list of arguments to be quoted
1050 @return: the quoted arguments concatenaned with spaces
1053 return ' '.join([ShellQuote(i) for i in args])
1056 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1057 """Simple ping implementation using TCP connect(2).
1059 Check if the given IP is reachable by doing attempting a TCP connect
1063 @param target: the IP or hostname to ping
1065 @param port: the port to connect to
1067 @param timeout: the timeout on the connection attemp
1068 @type live_port_needed: boolean
1069 @param live_port_needed: whether a closed port will cause the
1070 function to return failure, as if there was a timeout
1071 @type source: str or None
1072 @param source: if specified, will cause the connect to be made
1073 from this specific source address; failures to bind other
1074 than C{EADDRNOTAVAIL} will be ignored
1077 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1081 if source is not None:
1083 sock.bind((source, 0))
1084 except socket.error, (errcode, errstring):
1085 if errcode == errno.EADDRNOTAVAIL:
1088 sock.settimeout(timeout)
1091 sock.connect((target, port))
1094 except socket.timeout:
1096 except socket.error, (errcode, errstring):
1097 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1102 def OwnIpAddress(address):
1103 """Check if the current host has the the given IP address.
1105 Currently this is done by TCP-pinging the address from the loopback
1108 @type address: string
1109 @param address: the addres to check
1111 @return: True if we own the address
1114 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1115 source=constants.LOCALHOST_IP_ADDRESS)
1118 def ListVisibleFiles(path):
1119 """Returns a list of visible files in a directory.
1122 @param path: the directory to enumerate
1124 @return: the list of all files not starting with a dot
1127 files = [i for i in os.listdir(path) if not i.startswith(".")]
1132 def GetHomeDir(user, default=None):
1133 """Try to get the homedir of the given user.
1135 The user can be passed either as a string (denoting the name) or as
1136 an integer (denoting the user id). If the user is not found, the
1137 'default' argument is returned, which defaults to None.
1141 if isinstance(user, basestring):
1142 result = pwd.getpwnam(user)
1143 elif isinstance(user, (int, long)):
1144 result = pwd.getpwuid(user)
1146 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1150 return result.pw_dir
1154 """Returns a random UUID.
1156 @note: This is a Linux-specific method as it uses the /proc
1161 f = open("/proc/sys/kernel/random/uuid", "r")
1163 return f.read(128).rstrip("\n")
1168 def GenerateSecret():
1169 """Generates a random secret.
1171 This will generate a pseudo-random secret, and return its sha digest
1172 (so that it can be used where an ASCII string is needed).
1175 @return: a sha1 hexdigest of a block of 64 random bytes
1178 return sha.new(os.urandom(64)).hexdigest()
1181 def ReadFile(file_name, size=None):
1184 @type size: None or int
1185 @param size: Read at most size bytes
1187 @return: the (possibly partial) conent of the file
1190 f = open(file_name, "r")
1200 def WriteFile(file_name, fn=None, data=None,
1201 mode=None, uid=-1, gid=-1,
1202 atime=None, mtime=None, close=True,
1203 dry_run=False, backup=False,
1204 prewrite=None, postwrite=None):
1205 """(Over)write a file atomically.
1207 The file_name and either fn (a function taking one argument, the
1208 file descriptor, and which should write the data to it) or data (the
1209 contents of the file) must be passed. The other arguments are
1210 optional and allow setting the file mode, owner and group, and the
1211 mtime/atime of the file.
1213 If the function doesn't raise an exception, it has succeeded and the
1214 target file has the new contents. If the file has raised an
1215 exception, an existing target file should be unmodified and the
1216 temporary file should be removed.
1218 @type file_name: str
1219 @param file_name: the target filename
1221 @param fn: content writing function, called with
1222 file descriptor as parameter
1224 @param data: contents of the file
1226 @param mode: file mode
1228 @param uid: the owner of the file
1230 @param gid: the group of the file
1232 @param atime: a custom access time to be set on the file
1234 @param mtime: a custom modification time to be set on the file
1235 @type close: boolean
1236 @param close: whether to close file after writing it
1237 @type prewrite: callable
1238 @param prewrite: function to be called before writing content
1239 @type postwrite: callable
1240 @param postwrite: function to be called after writing content
1243 @return: None if the 'close' parameter evaluates to True,
1244 otherwise the file descriptor
1246 @raise errors.ProgrammerError: if an of the arguments are not valid
1249 if not os.path.isabs(file_name):
1250 raise errors.ProgrammerError("Path passed to WriteFile is not"
1251 " absolute: '%s'" % file_name)
1253 if [fn, data].count(None) != 1:
1254 raise errors.ProgrammerError("fn or data required")
1256 if [atime, mtime].count(None) == 1:
1257 raise errors.ProgrammerError("Both atime and mtime must be either"
1260 if backup and not dry_run and os.path.isfile(file_name):
1261 CreateBackup(file_name)
1263 dir_name, base_name = os.path.split(file_name)
1264 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1265 # here we need to make sure we remove the temp file, if any error
1266 # leaves it in place
1268 if uid != -1 or gid != -1:
1269 os.chown(new_name, uid, gid)
1271 os.chmod(new_name, mode)
1272 if callable(prewrite):
1274 if data is not None:
1278 if callable(postwrite):
1281 if atime is not None and mtime is not None:
1282 os.utime(new_name, (atime, mtime))
1284 os.rename(new_name, file_name)
1291 RemoveFile(new_name)
1296 def FirstFree(seq, base=0):
1297 """Returns the first non-existing integer from seq.
1299 The seq argument should be a sorted list of positive integers. The
1300 first time the index of an element is smaller than the element
1301 value, the index will be returned.
1303 The base argument is used to start at a different offset,
1304 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1306 Example: C{[0, 1, 3]} will return I{2}.
1309 @param seq: the sequence to be analyzed.
1311 @param base: use this value as the base index of the sequence
1313 @return: the first non-used index in the sequence
1316 for idx, elem in enumerate(seq):
1317 assert elem >= base, "Passed element is higher than base offset"
1318 if elem > idx + base:
1324 def all(seq, pred=bool):
1325 "Returns True if pred(x) is True for every element in the iterable"
1326 for elem in itertools.ifilterfalse(pred, seq):
1331 def any(seq, pred=bool):
1332 "Returns True if pred(x) is True for at least one element in the iterable"
1333 for elem in itertools.ifilter(pred, seq):
1338 def UniqueSequence(seq):
1339 """Returns a list with unique elements.
1341 Element order is preserved.
1344 @param seq: the sequence with the source elementes
1346 @return: list of unique elements from seq
1350 return [i for i in seq if i not in seen and not seen.add(i)]
1353 def IsValidMac(mac):
1354 """Predicate to check if a MAC address is valid.
1356 Checks wether the supplied MAC address is formally correct, only
1357 accepts colon separated format.
1360 @param mac: the MAC to be validated
1362 @return: True is the MAC seems valid
1365 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1366 return mac_check.match(mac) is not None
1369 def TestDelay(duration):
1370 """Sleep for a fixed amount of time.
1372 @type duration: float
1373 @param duration: the sleep duration
1375 @return: False for negative value, True otherwise
1380 time.sleep(duration)
1384 def _CloseFDNoErr(fd, retries=5):
1385 """Close a file descriptor ignoring errors.
1388 @param fd: the file descriptor
1390 @param retries: how many retries to make, in case we get any
1391 other error than EBADF
1396 except OSError, err:
1397 if err.errno != errno.EBADF:
1399 _CloseFDNoErr(fd, retries - 1)
1400 # else either it's closed already or we're out of retries, so we
1401 # ignore this and go on
1404 def CloseFDs(noclose_fds=None):
1405 """Close file descriptors.
1407 This closes all file descriptors above 2 (i.e. except
1410 @type noclose_fds: list or None
1411 @param noclose_fds: if given, it denotes a list of file descriptor
1412 that should not be closed
1415 # Default maximum for the number of available file descriptors.
1416 if 'SC_OPEN_MAX' in os.sysconf_names:
1418 MAXFD = os.sysconf('SC_OPEN_MAX')
1425 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1426 if (maxfd == resource.RLIM_INFINITY):
1429 # Iterate through and close all file descriptors (except the standard ones)
1430 for fd in range(3, maxfd):
1431 if noclose_fds and fd in noclose_fds:
1436 def Daemonize(logfile):
1437 """Daemonize the current process.
1439 This detaches the current process from the controlling terminal and
1440 runs it in the background as a daemon.
1443 @param logfile: the logfile to which we should redirect stdout/stderr
1445 @returns: the value zero
1453 if (pid == 0): # The first child.
1456 pid = os.fork() # Fork a second child.
1457 if (pid == 0): # The second child.
1461 # exit() or _exit()? See below.
1462 os._exit(0) # Exit parent (the first child) of the second child.
1464 os._exit(0) # Exit parent of the first child.
1468 i = os.open("/dev/null", os.O_RDONLY) # stdin
1469 assert i == 0, "Can't close/reopen stdin"
1470 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1471 assert i == 1, "Can't close/reopen stdout"
1472 # Duplicate standard output to standard error.
1477 def DaemonPidFileName(name):
1478 """Compute a ganeti pid file absolute path
1481 @param name: the daemon name
1483 @return: the full path to the pidfile corresponding to the given
1487 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1490 def WritePidFile(name):
1491 """Write the current process pidfile.
1493 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1496 @param name: the daemon name to use
1497 @raise errors.GenericError: if the pid file already exists and
1498 points to a live process
1502 pidfilename = DaemonPidFileName(name)
1503 if IsProcessAlive(ReadPidFile(pidfilename)):
1504 raise errors.GenericError("%s contains a live process" % pidfilename)
1506 WriteFile(pidfilename, data="%d\n" % pid)
1509 def RemovePidFile(name):
1510 """Remove the current process pidfile.
1512 Any errors are ignored.
1515 @param name: the daemon name used to derive the pidfile name
1519 pidfilename = DaemonPidFileName(name)
1520 # TODO: we could check here that the file contains our pid
1522 RemoveFile(pidfilename)
1527 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1529 """Kill a process given by its pid.
1532 @param pid: The PID to terminate.
1534 @param signal_: The signal to send, by default SIGTERM
1536 @param timeout: The timeout after which, if the process is still alive,
1537 a SIGKILL will be sent. If not positive, no such checking
1539 @type waitpid: boolean
1540 @param waitpid: If true, we should waitpid on this process after
1541 sending signals, since it's our own child and otherwise it
1542 would remain as zombie
1545 def _helper(pid, signal_, wait):
1546 """Simple helper to encapsulate the kill/waitpid sequence"""
1547 os.kill(pid, signal_)
1550 os.waitpid(pid, os.WNOHANG)
1555 # kill with pid=0 == suicide
1556 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1558 if not IsProcessAlive(pid):
1560 _helper(pid, signal_, waitpid)
1564 # Wait up to $timeout seconds
1565 end = time.time() + timeout
1567 while time.time() < end and IsProcessAlive(pid):
1569 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1575 # Make wait time longer for next try
1579 if IsProcessAlive(pid):
1580 # Kill process if it's still alive
1581 _helper(pid, signal.SIGKILL, waitpid)
1584 def FindFile(name, search_path, test=os.path.exists):
1585 """Look for a filesystem object in a given path.
1587 This is an abstract method to search for filesystem object (files,
1588 dirs) under a given search path.
1591 @param name: the name to look for
1592 @type search_path: str
1593 @param search_path: location to start at
1594 @type test: callable
1595 @param test: a function taking one argument that should return True
1596 if the a given object is valid; the default value is
1597 os.path.exists, causing only existing files to be returned
1599 @return: full path to the object if found, None otherwise
1602 for dir_name in search_path:
1603 item_name = os.path.sep.join([dir_name, name])
1609 def CheckVolumeGroupSize(vglist, vgname, minsize):
1610 """Checks if the volume group list is valid.
1612 The function will check if a given volume group is in the list of
1613 volume groups and has a minimum size.
1616 @param vglist: dictionary of volume group names and their size
1618 @param vgname: the volume group we should check
1620 @param minsize: the minimum size we accept
1622 @return: None for success, otherwise the error message
1625 vgsize = vglist.get(vgname, None)
1627 return "volume group '%s' missing" % vgname
1628 elif vgsize < minsize:
1629 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1630 (vgname, minsize, vgsize))
1634 def SplitTime(value):
1635 """Splits time as floating point number into a tuple.
1637 @param value: Time in seconds
1638 @type value: int or float
1639 @return: Tuple containing (seconds, microseconds)
1642 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1644 assert 0 <= seconds, \
1645 "Seconds must be larger than or equal to 0, but are %s" % seconds
1646 assert 0 <= microseconds <= 999999, \
1647 "Microseconds must be 0-999999, but are %s" % microseconds
1649 return (int(seconds), int(microseconds))
1652 def MergeTime(timetuple):
1653 """Merges a tuple into time as a floating point number.
1655 @param timetuple: Time as tuple, (seconds, microseconds)
1656 @type timetuple: tuple
1657 @return: Time as a floating point number expressed in seconds
1660 (seconds, microseconds) = timetuple
1662 assert 0 <= seconds, \
1663 "Seconds must be larger than or equal to 0, but are %s" % seconds
1664 assert 0 <= microseconds <= 999999, \
1665 "Microseconds must be 0-999999, but are %s" % microseconds
1667 return float(seconds) + (float(microseconds) * 0.000001)
1670 def GetNodeDaemonPort():
1671 """Get the node daemon port for this cluster.
1673 Note that this routine does not read a ganeti-specific file, but
1674 instead uses C{socket.getservbyname} to allow pre-customization of
1675 this parameter outside of Ganeti.
1681 port = socket.getservbyname("ganeti-noded", "tcp")
1682 except socket.error:
1683 port = constants.DEFAULT_NODED_PORT
1688 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1689 multithreaded=False):
1690 """Configures the logging module.
1693 @param logfile: the filename to which we should log
1694 @type debug: boolean
1695 @param debug: whether to enable debug messages too or
1696 only those at C{INFO} and above level
1697 @type stderr_logging: boolean
1698 @param stderr_logging: whether we should also log to the standard error
1700 @param program: the name under which we should log messages
1701 @type multithreaded: boolean
1702 @param multithreaded: if True, will add the thread name to the log file
1703 @raise EnvironmentError: if we can't open the log file and
1704 stderr logging is disabled
1707 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1709 fmt += "/%(threadName)s"
1711 fmt += " %(module)s:%(lineno)s"
1712 fmt += " %(levelname)s %(message)s"
1713 formatter = logging.Formatter(fmt)
1715 root_logger = logging.getLogger("")
1716 root_logger.setLevel(logging.NOTSET)
1718 # Remove all previously setup handlers
1719 for handler in root_logger.handlers:
1721 root_logger.removeHandler(handler)
1724 stderr_handler = logging.StreamHandler()
1725 stderr_handler.setFormatter(formatter)
1727 stderr_handler.setLevel(logging.NOTSET)
1729 stderr_handler.setLevel(logging.CRITICAL)
1730 root_logger.addHandler(stderr_handler)
1732 # this can fail, if the logging directories are not setup or we have
1733 # a permisssion problem; in this case, it's best to log but ignore
1734 # the error if stderr_logging is True, and if false we re-raise the
1735 # exception since otherwise we could run but without any logs at all
1737 logfile_handler = logging.FileHandler(logfile)
1738 logfile_handler.setFormatter(formatter)
1740 logfile_handler.setLevel(logging.DEBUG)
1742 logfile_handler.setLevel(logging.INFO)
1743 root_logger.addHandler(logfile_handler)
1744 except EnvironmentError:
1746 logging.exception("Failed to enable logging to file '%s'", logfile)
1748 # we need to re-raise the exception
1752 def TailFile(fname, lines=20):
1753 """Return the last lines from a file.
1755 @note: this function will only read and parse the last 4KB of
1756 the file; if the lines are very long, it could be that less
1757 than the requested number of lines are returned
1759 @param fname: the file name
1761 @param lines: the (maximum) number of lines to return
1764 fd = open(fname, "r")
1768 pos = max(0, pos-4096)
1770 raw_data = fd.read()
1774 rows = raw_data.splitlines()
1775 return rows[-lines:]
1778 def SafeEncode(text):
1779 """Return a 'safe' version of a source string.
1781 This function mangles the input string and returns a version that
1782 should be safe to disply/encode as ASCII. To this end, we first
1783 convert it to ASCII using the 'backslashreplace' encoding which
1784 should get rid of any non-ASCII chars, and then we again encode it
1785 via 'string_escape' which converts '\n' into '\\n' so that log
1786 messages remain one-line.
1788 @type text: str or unicode
1789 @param text: input data
1791 @return: a safe version of text
1794 text = text.encode('ascii', 'backslashreplace')
1795 text = text.encode('string_escape')
1799 def LockedMethod(fn):
1800 """Synchronized object access decorator.
1802 This decorator is intended to protect access to an object using the
1803 object's own lock which is hardcoded to '_lock'.
1806 def _LockDebug(*args, **kwargs):
1808 logging.debug(*args, **kwargs)
1810 def wrapper(self, *args, **kwargs):
1811 assert hasattr(self, '_lock')
1813 _LockDebug("Waiting for %s", lock)
1816 _LockDebug("Acquired %s", lock)
1817 result = fn(self, *args, **kwargs)
1819 _LockDebug("Releasing %s", lock)
1821 _LockDebug("Released %s", lock)
1827 """Locks a file using POSIX locks.
1830 @param fd: the file descriptor we need to lock
1834 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1835 except IOError, err:
1836 if err.errno == errno.EAGAIN:
1837 raise errors.LockError("File already locked")
1841 class FileLock(object):
1842 """Utility class for file locks.
1845 def __init__(self, filename):
1846 """Constructor for FileLock.
1848 This will open the file denoted by the I{filename} argument.
1851 @param filename: path to the file to be locked
1854 self.filename = filename
1855 self.fd = open(self.filename, "w")
1861 """Close the file and release the lock.
1868 def _flock(self, flag, blocking, timeout, errmsg):
1869 """Wrapper for fcntl.flock.
1872 @param flag: operation flag
1873 @type blocking: bool
1874 @param blocking: whether the operation should be done in blocking mode.
1875 @type timeout: None or float
1876 @param timeout: for how long the operation should be retried (implies
1878 @type errmsg: string
1879 @param errmsg: error message in case operation fails.
1882 assert self.fd, "Lock was closed"
1883 assert timeout is None or timeout >= 0, \
1884 "If specified, timeout must be positive"
1886 if timeout is not None:
1887 flag |= fcntl.LOCK_NB
1888 timeout_end = time.time() + timeout
1890 # Blocking doesn't have effect with timeout
1892 flag |= fcntl.LOCK_NB
1898 fcntl.flock(self.fd, flag)
1900 except IOError, err:
1901 if err.errno in (errno.EAGAIN, ):
1902 if timeout_end is not None and time.time() < timeout_end:
1903 # Wait before trying again
1904 time.sleep(max(0.1, min(1.0, timeout)))
1906 raise errors.LockError(errmsg)
1908 logging.exception("fcntl.flock failed")
1911 def Exclusive(self, blocking=False, timeout=None):
1912 """Locks the file in exclusive mode.
1914 @type blocking: boolean
1915 @param blocking: whether to block and wait until we
1916 can lock the file or return immediately
1917 @type timeout: int or None
1918 @param timeout: if not None, the duration to wait for the lock
1922 self._flock(fcntl.LOCK_EX, blocking, timeout,
1923 "Failed to lock %s in exclusive mode" % self.filename)
1925 def Shared(self, blocking=False, timeout=None):
1926 """Locks the file in shared mode.
1928 @type blocking: boolean
1929 @param blocking: whether to block and wait until we
1930 can lock the file or return immediately
1931 @type timeout: int or None
1932 @param timeout: if not None, the duration to wait for the lock
1936 self._flock(fcntl.LOCK_SH, blocking, timeout,
1937 "Failed to lock %s in shared mode" % self.filename)
1939 def Unlock(self, blocking=True, timeout=None):
1940 """Unlocks the file.
1942 According to C{flock(2)}, unlocking can also be a nonblocking
1945 To make a non-blocking request, include LOCK_NB with any of the above
1948 @type blocking: boolean
1949 @param blocking: whether to block and wait until we
1950 can lock the file or return immediately
1951 @type timeout: int or None
1952 @param timeout: if not None, the duration to wait for the lock
1956 self._flock(fcntl.LOCK_UN, blocking, timeout,
1957 "Failed to unlock %s" % self.filename)
1960 class SignalHandler(object):
1961 """Generic signal handler class.
1963 It automatically restores the original handler when deconstructed or
1964 when L{Reset} is called. You can either pass your own handler
1965 function in or query the L{called} attribute to detect whether the
1969 @ivar signum: the signals we handle
1970 @type called: boolean
1971 @ivar called: tracks whether any of the signals have been raised
1974 def __init__(self, signum):
1975 """Constructs a new SignalHandler instance.
1977 @type signum: int or list of ints
1978 @param signum: Single signal number or set of signal numbers
1981 if isinstance(signum, (int, long)):
1982 self.signum = set([signum])
1984 self.signum = set(signum)
1990 for signum in self.signum:
1992 prev_handler = signal.signal(signum, self._HandleSignal)
1994 self._previous[signum] = prev_handler
1996 # Restore previous handler
1997 signal.signal(signum, prev_handler)
2000 # Reset all handlers
2002 # Here we have a race condition: a handler may have already been called,
2003 # but there's not much we can do about it at this point.
2010 """Restore previous handler.
2012 This will reset all the signals to their previous handlers.
2015 for signum, prev_handler in self._previous.items():
2016 signal.signal(signum, prev_handler)
2017 # If successful, remove from dict
2018 del self._previous[signum]
2021 """Unsets the L{called} flag.
2023 This function can be used in case a signal may arrive several times.
2028 def _HandleSignal(self, signum, frame):
2029 """Actual signal handling function.
2032 # This is not nice and not absolutely atomic, but it appears to be the only
2033 # solution in Python -- there are no atomic types.
2037 class FieldSet(object):
2038 """A simple field set.
2040 Among the features are:
2041 - checking if a string is among a list of static string or regex objects
2042 - checking if a whole list of string matches
2043 - returning the matching groups from a regex match
2045 Internally, all fields are held as regular expression objects.
2048 def __init__(self, *items):
2049 self.items = [re.compile("^%s$" % value) for value in items]
2051 def Extend(self, other_set):
2052 """Extend the field set with the items from another one"""
2053 self.items.extend(other_set.items)
2055 def Matches(self, field):
2056 """Checks if a field matches the current set
2059 @param field: the string to match
2060 @return: either False or a regular expression match object
2063 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2067 def NonMatching(self, items):
2068 """Returns the list of fields not matching the current set
2071 @param items: the list of fields to check
2073 @return: list of non-matching fields
2076 return [val for val in items if not self.Matches(val)]