4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
48 from cStringIO import StringIO
50 from ganeti import errors
51 from ganeti import constants
55 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60 #: when set to True, L{RunCmd} is disabled
64 class RunResult(object):
65 """Holds the result of running external programs.
68 @ivar exit_code: the exit code of the program, or None (if the program
70 @type signal: int or None
71 @ivar signal: the signal that caused the program to finish, or None
72 (if the program wasn't terminated by a signal)
74 @ivar stdout: the standard output of the program
76 @ivar stderr: the standard error of the program
78 @ivar failed: True in case the program was
79 terminated by a signal or exited with a non-zero exit code
80 @ivar fail_reason: a string detailing the termination reason
83 __slots__ = ["exit_code", "signal", "stdout", "stderr",
84 "failed", "fail_reason", "cmd"]
87 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
89 self.exit_code = exit_code
93 self.failed = (signal_ is not None or exit_code != 0)
95 if self.signal is not None:
96 self.fail_reason = "terminated by signal %s" % self.signal
97 elif self.exit_code is not None:
98 self.fail_reason = "exited with exit code %s" % self.exit_code
100 self.fail_reason = "unable to determine termination reason"
103 logging.debug("Command '%s' failed (%s); output: %s",
104 self.cmd, self.fail_reason, self.output)
106 def _GetOutput(self):
107 """Returns the combined stdout and stderr for easier usage.
110 return self.stdout + self.stderr
112 output = property(_GetOutput, None, None, "Return full output")
115 def RunCmd(cmd, env=None, output=None, cwd='/'):
116 """Execute a (shell) command.
118 The command should not read from its standard input, as it will be
121 @type cmd: string or list
122 @param cmd: Command to run
124 @param env: Additional environment
126 @param output: if desired, the output of the command can be
127 saved in a file instead of the RunResult instance; this
128 parameter denotes the file name (if not None)
130 @param cwd: if specified, will be used as the working
131 directory for the command; the default will be /
133 @return: RunResult instance
134 @raise erors.ProgrammerError: if we call this when forks are disabled
138 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
140 if isinstance(cmd, list):
141 cmd = [str(val) for val in cmd]
142 strcmd = " ".join(cmd)
147 logging.debug("RunCmd '%s'", strcmd)
149 cmd_env = os.environ.copy()
150 cmd_env["LC_ALL"] = "C"
155 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
157 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
167 return RunResult(exitcode, signal_, out, err, strcmd)
170 def _RunCmdPipe(cmd, env, via_shell, cwd):
171 """Run a command and return its output.
173 @type cmd: string or list
174 @param cmd: Command to run
176 @param env: The environment to use
177 @type via_shell: bool
178 @param via_shell: if we should run via the shell
180 @param cwd: the working directory for the program
182 @return: (out, err, status)
185 poller = select.poll()
186 child = subprocess.Popen(cmd, shell=via_shell,
187 stderr=subprocess.PIPE,
188 stdout=subprocess.PIPE,
189 stdin=subprocess.PIPE,
190 close_fds=True, env=env,
194 poller.register(child.stdout, select.POLLIN)
195 poller.register(child.stderr, select.POLLIN)
199 child.stdout.fileno(): (out, child.stdout),
200 child.stderr.fileno(): (err, child.stderr),
203 status = fcntl.fcntl(fd, fcntl.F_GETFL)
204 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
208 pollresult = poller.poll()
209 except EnvironmentError, eerr:
210 if eerr.errno == errno.EINTR:
213 except select.error, serr:
214 if serr[0] == errno.EINTR:
218 for fd, event in pollresult:
219 if event & select.POLLIN or event & select.POLLPRI:
220 data = fdmap[fd][1].read()
221 # no data from read signifies EOF (the same as POLLHUP)
223 poller.unregister(fd)
226 fdmap[fd][0].write(data)
227 if (event & select.POLLNVAL or event & select.POLLHUP or
228 event & select.POLLERR):
229 poller.unregister(fd)
235 status = child.wait()
236 return out, err, status
239 def _RunCmdFile(cmd, env, via_shell, output, cwd):
240 """Run a command and save its output to a file.
242 @type cmd: string or list
243 @param cmd: Command to run
245 @param env: The environment to use
246 @type via_shell: bool
247 @param via_shell: if we should run via the shell
249 @param output: the filename in which to save the output
251 @param cwd: the working directory for the program
253 @return: the exit status
256 fh = open(output, "a")
258 child = subprocess.Popen(cmd, shell=via_shell,
259 stderr=subprocess.STDOUT,
261 stdin=subprocess.PIPE,
262 close_fds=True, env=env,
266 status = child.wait()
272 def RemoveFile(filename):
273 """Remove a file ignoring some errors.
275 Remove a file, ignoring non-existing ones or directories. Other
279 @param filename: the file to be removed
285 if err.errno not in (errno.ENOENT, errno.EISDIR):
289 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
293 @param old: Original path
297 @param mkdir: Whether to create target directory if it doesn't exist
298 @type mkdir_mode: int
299 @param mkdir_mode: Mode for newly created directories
303 return os.rename(old, new)
305 # In at least one use case of this function, the job queue, directory
306 # creation is very rare. Checking for the directory before renaming is not
308 if mkdir and err.errno == errno.ENOENT:
309 # Create directory and try again
310 os.makedirs(os.path.dirname(new), mkdir_mode)
311 return os.rename(old, new)
315 def _FingerprintFile(filename):
316 """Compute the fingerprint of a file.
318 If the file does not exist, a None will be returned
322 @param filename: the filename to checksum
324 @return: the hex digest of the sha checksum of the contents
328 if not (os.path.exists(filename) and os.path.isfile(filename)):
341 return fp.hexdigest()
344 def FingerprintFiles(files):
345 """Compute fingerprints for a list of files.
348 @param files: the list of filename to fingerprint
350 @return: a dictionary filename: fingerprint, holding only
356 for filename in files:
357 cksum = _FingerprintFile(filename)
359 ret[filename] = cksum
364 def CheckDict(target, template, logname=None):
365 """Ensure a dictionary has a required set of keys.
367 For the given dictionaries I{target} and I{template}, ensure
368 I{target} has all the keys from I{template}. Missing keys are added
369 with values from template.
372 @param target: the dictionary to update
374 @param template: the dictionary holding the default values
375 @type logname: str or None
376 @param logname: if not None, causes the missing keys to be
377 logged with this name
384 target[k] = template[k]
386 if missing and logname:
387 logging.warning('%s missing keys %s', logname, ', '.join(missing))
390 def ForceDictType(target, key_types, allowed_values=None):
391 """Force the values of a dict to have certain types.
394 @param target: the dict to update
395 @type key_types: dict
396 @param key_types: dict mapping target dict keys to types
397 in constants.ENFORCEABLE_TYPES
398 @type allowed_values: list
399 @keyword allowed_values: list of specially allowed values
402 if allowed_values is None:
406 if key not in key_types:
407 msg = "Unknown key '%s'" % key
408 raise errors.TypeEnforcementError(msg)
410 if target[key] in allowed_values:
413 type = key_types[key]
414 if type not in constants.ENFORCEABLE_TYPES:
415 msg = "'%s' has non-enforceable type %s" % (key, type)
416 raise errors.ProgrammerError(msg)
418 if type == constants.VTYPE_STRING:
419 if not isinstance(target[key], basestring):
420 if isinstance(target[key], bool) and not target[key]:
423 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
424 raise errors.TypeEnforcementError(msg)
425 elif type == constants.VTYPE_BOOL:
426 if isinstance(target[key], basestring) and target[key]:
427 if target[key].lower() == constants.VALUE_FALSE:
429 elif target[key].lower() == constants.VALUE_TRUE:
432 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
433 raise errors.TypeEnforcementError(msg)
438 elif type == constants.VTYPE_SIZE:
440 target[key] = ParseUnit(target[key])
441 except errors.UnitParseError, err:
442 msg = "'%s' (value %s) is not a valid size. error: %s" % \
443 (key, target[key], err)
444 raise errors.TypeEnforcementError(msg)
445 elif type == constants.VTYPE_INT:
447 target[key] = int(target[key])
448 except (ValueError, TypeError):
449 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
450 raise errors.TypeEnforcementError(msg)
453 def IsProcessAlive(pid):
454 """Check if a given pid exists on the system.
456 @note: zombie status is not handled, so zombie processes
457 will be returned as alive
459 @param pid: the process ID to check
461 @return: True if the process exists
468 os.stat("/proc/%d/status" % pid)
470 except EnvironmentError, err:
471 if err.errno in (errno.ENOENT, errno.ENOTDIR):
476 def ReadPidFile(pidfile):
477 """Read a pid from a file.
479 @type pidfile: string
480 @param pidfile: path to the file containing the pid
482 @return: The process id, if the file exists and contains a valid PID,
487 pf = open(pidfile, 'r')
488 except EnvironmentError, err:
489 if err.errno != errno.ENOENT:
490 logging.exception("Can't read pid file?!")
495 except ValueError, err:
496 logging.info("Can't parse pid file contents", exc_info=True)
502 def MatchNameComponent(key, name_list):
503 """Try to match a name against a list.
505 This function will try to match a name like test1 against a list
506 like C{['test1.example.com', 'test2.example.com', ...]}. Against
507 this list, I{'test1'} as well as I{'test1.example'} will match, but
508 not I{'test1.ex'}. A multiple match will be considered as no match
509 at all (e.g. I{'test1'} against C{['test1.example.com',
510 'test1.example.org']}).
513 @param key: the name to be searched
514 @type name_list: list
515 @param name_list: the list of strings against which to search the key
518 @return: None if there is no match I{or} if there are multiple matches,
519 otherwise the element from the list which matches
522 mo = re.compile("^%s(\..*)?$" % re.escape(key))
523 names_filtered = [name for name in name_list if mo.match(name) is not None]
524 if len(names_filtered) != 1:
526 return names_filtered[0]
530 """Class implementing resolver and hostname functionality
533 def __init__(self, name=None):
534 """Initialize the host name object.
536 If the name argument is not passed, it will use this system's
541 name = self.SysName()
544 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
545 self.ip = self.ipaddrs[0]
548 """Returns the hostname without domain.
551 return self.name.split('.')[0]
555 """Return the current system's name.
557 This is simply a wrapper over C{socket.gethostname()}.
560 return socket.gethostname()
563 def LookupHostname(hostname):
567 @param hostname: hostname to look up
570 @return: a tuple (name, aliases, ipaddrs) as returned by
571 C{socket.gethostbyname_ex}
572 @raise errors.ResolverError: in case of errors in resolving
576 result = socket.gethostbyname_ex(hostname)
577 except socket.gaierror, err:
578 # hostname not found in DNS
579 raise errors.ResolverError(hostname, err.args[0], err.args[1])
584 def ListVolumeGroups():
585 """List volume groups and their size
589 Dictionary with keys volume name and values
590 the size of the volume
593 command = "vgs --noheadings --units m --nosuffix -o name,size"
594 result = RunCmd(command)
599 for line in result.stdout.splitlines():
601 name, size = line.split()
602 size = int(float(size))
603 except (IndexError, ValueError), err:
604 logging.error("Invalid output from vgs (%s): %s", err, line)
612 def BridgeExists(bridge):
613 """Check whether the given bridge exists in the system
616 @param bridge: the bridge name to check
618 @return: True if it does
621 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
624 def NiceSort(name_list):
625 """Sort a list of strings based on digit and non-digit groupings.
627 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
628 will sort the list in the logical order C{['a1', 'a2', 'a10',
631 The sort algorithm breaks each name in groups of either only-digits
632 or no-digits. Only the first eight such groups are considered, and
633 after that we just use what's left of the string.
635 @type name_list: list
636 @param name_list: the names to be sorted
638 @return: a copy of the name list sorted with our algorithm
641 _SORTER_BASE = "(\D+|\d+)"
642 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
643 _SORTER_BASE, _SORTER_BASE,
644 _SORTER_BASE, _SORTER_BASE,
645 _SORTER_BASE, _SORTER_BASE)
646 _SORTER_RE = re.compile(_SORTER_FULL)
647 _SORTER_NODIGIT = re.compile("^\D*$")
649 """Attempts to convert a variable to integer."""
650 if val is None or _SORTER_NODIGIT.match(val):
655 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
656 for name in name_list]
658 return [tup[1] for tup in to_sort]
661 def TryConvert(fn, val):
662 """Try to convert a value ignoring errors.
664 This function tries to apply function I{fn} to I{val}. If no
665 C{ValueError} or C{TypeError} exceptions are raised, it will return
666 the result, else it will return the original value. Any other
667 exceptions are propagated to the caller.
670 @param fn: function to apply to the value
671 @param val: the value to be converted
672 @return: The converted value if the conversion was successful,
673 otherwise the original value.
678 except (ValueError, TypeError), err:
684 """Verifies the syntax of an IPv4 address.
686 This function checks if the IPv4 address passes is valid or not based
687 on syntax (not IP range, class calculations, etc.).
690 @param ip: the address to be checked
691 @rtype: a regular expression match object
692 @return: a regular epression match object, or None if the
696 unit = "(0|[1-9]\d{0,2})"
697 #TODO: convert and return only boolean
698 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
701 def IsValidShellParam(word):
702 """Verifies is the given word is safe from the shell's p.o.v.
704 This means that we can pass this to a command via the shell and be
705 sure that it doesn't alter the command line and is passed as such to
708 Note that we are overly restrictive here, in order to be on the safe
712 @param word: the word to check
714 @return: True if the word is 'safe'
717 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
720 def BuildShellCmd(template, *args):
721 """Build a safe shell command line from the given arguments.
723 This function will check all arguments in the args list so that they
724 are valid shell parameters (i.e. they don't contain shell
725 metacharaters). If everything is ok, it will return the result of
729 @param template: the string holding the template for the
732 @return: the expanded command line
736 if not IsValidShellParam(word):
737 raise errors.ProgrammerError("Shell argument '%s' contains"
738 " invalid characters" % word)
739 return template % args
742 def FormatUnit(value, units):
743 """Formats an incoming number of MiB with the appropriate unit.
746 @param value: integer representing the value in MiB (1048576)
748 @param units: the type of formatting we should do:
749 - 'h' for automatic scaling
754 @return: the formatted value (with suffix)
757 if units not in ('m', 'g', 't', 'h'):
758 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
762 if units == 'm' or (units == 'h' and value < 1024):
765 return "%d%s" % (round(value, 0), suffix)
767 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
770 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
775 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
778 def ParseUnit(input_string):
779 """Tries to extract number and scale from the given string.
781 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
782 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
783 is always an int in MiB.
786 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
788 raise errors.UnitParseError("Invalid format")
790 value = float(m.groups()[0])
794 lcunit = unit.lower()
798 if lcunit in ('m', 'mb', 'mib'):
799 # Value already in MiB
802 elif lcunit in ('g', 'gb', 'gib'):
805 elif lcunit in ('t', 'tb', 'tib'):
809 raise errors.UnitParseError("Unknown unit: %s" % unit)
811 # Make sure we round up
812 if int(value) < value:
815 # Round up to the next multiple of 4
818 value += 4 - value % 4
823 def AddAuthorizedKey(file_name, key):
824 """Adds an SSH public key to an authorized_keys file.
827 @param file_name: path to authorized_keys file
829 @param key: string containing key
832 key_fields = key.split()
834 f = open(file_name, 'a+')
838 # Ignore whitespace changes
839 if line.split() == key_fields:
841 nl = line.endswith('\n')
845 f.write(key.rstrip('\r\n'))
852 def RemoveAuthorizedKey(file_name, key):
853 """Removes an SSH public key from an authorized_keys file.
856 @param file_name: path to authorized_keys file
858 @param key: string containing key
861 key_fields = key.split()
863 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
865 out = os.fdopen(fd, 'w')
867 f = open(file_name, 'r')
870 # Ignore whitespace changes while comparing lines
871 if line.split() != key_fields:
875 os.rename(tmpname, file_name)
885 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
886 """Sets the name of an IP address and hostname in /etc/hosts.
889 @param file_name: path to the file to modify (usually C{/etc/hosts})
891 @param ip: the IP address
893 @param hostname: the hostname to be added
895 @param aliases: the list of aliases to add for the hostname
898 # FIXME: use WriteFile + fn rather than duplicating its efforts
899 # Ensure aliases are unique
900 aliases = UniqueSequence([hostname] + aliases)[1:]
902 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
904 out = os.fdopen(fd, 'w')
906 f = open(file_name, 'r')
909 fields = line.split()
910 if fields and not fields[0].startswith('#') and ip == fields[0]:
914 out.write("%s\t%s" % (ip, hostname))
916 out.write(" %s" % ' '.join(aliases))
921 os.chmod(tmpname, 0644)
922 os.rename(tmpname, file_name)
932 def AddHostToEtcHosts(hostname):
933 """Wrapper around SetEtcHostsEntry.
936 @param hostname: a hostname that will be resolved and added to
937 L{constants.ETC_HOSTS}
940 hi = HostInfo(name=hostname)
941 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
944 def RemoveEtcHostsEntry(file_name, hostname):
945 """Removes a hostname from /etc/hosts.
947 IP addresses without names are removed from the file.
950 @param file_name: path to the file to modify (usually C{/etc/hosts})
952 @param hostname: the hostname to be removed
955 # FIXME: use WriteFile + fn rather than duplicating its efforts
956 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
958 out = os.fdopen(fd, 'w')
960 f = open(file_name, 'r')
963 fields = line.split()
964 if len(fields) > 1 and not fields[0].startswith('#'):
966 if hostname in names:
967 while hostname in names:
968 names.remove(hostname)
970 out.write("%s %s\n" % (fields[0], ' '.join(names)))
977 os.chmod(tmpname, 0644)
978 os.rename(tmpname, file_name)
988 def RemoveHostFromEtcHosts(hostname):
989 """Wrapper around RemoveEtcHostsEntry.
992 @param hostname: hostname that will be resolved and its
993 full and shot name will be removed from
994 L{constants.ETC_HOSTS}
997 hi = HostInfo(name=hostname)
998 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
999 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1002 def CreateBackup(file_name):
1003 """Creates a backup of a file.
1005 @type file_name: str
1006 @param file_name: file to be backed up
1008 @return: the path to the newly created backup
1009 @raise errors.ProgrammerError: for invalid file names
1012 if not os.path.isfile(file_name):
1013 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1016 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1017 dir_name = os.path.dirname(file_name)
1019 fsrc = open(file_name, 'rb')
1021 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1022 fdst = os.fdopen(fd, 'wb')
1024 shutil.copyfileobj(fsrc, fdst)
1033 def ShellQuote(value):
1034 """Quotes shell argument according to POSIX.
1037 @param value: the argument to be quoted
1039 @return: the quoted value
1042 if _re_shell_unquoted.match(value):
1045 return "'%s'" % value.replace("'", "'\\''")
1048 def ShellQuoteArgs(args):
1049 """Quotes a list of shell arguments.
1052 @param args: list of arguments to be quoted
1054 @return: the quoted arguments concatenaned with spaces
1057 return ' '.join([ShellQuote(i) for i in args])
1060 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1061 """Simple ping implementation using TCP connect(2).
1063 Check if the given IP is reachable by doing attempting a TCP connect
1067 @param target: the IP or hostname to ping
1069 @param port: the port to connect to
1071 @param timeout: the timeout on the connection attemp
1072 @type live_port_needed: boolean
1073 @param live_port_needed: whether a closed port will cause the
1074 function to return failure, as if there was a timeout
1075 @type source: str or None
1076 @param source: if specified, will cause the connect to be made
1077 from this specific source address; failures to bind other
1078 than C{EADDRNOTAVAIL} will be ignored
1081 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1085 if source is not None:
1087 sock.bind((source, 0))
1088 except socket.error, (errcode, errstring):
1089 if errcode == errno.EADDRNOTAVAIL:
1092 sock.settimeout(timeout)
1095 sock.connect((target, port))
1098 except socket.timeout:
1100 except socket.error, (errcode, errstring):
1101 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1106 def OwnIpAddress(address):
1107 """Check if the current host has the the given IP address.
1109 Currently this is done by TCP-pinging the address from the loopback
1112 @type address: string
1113 @param address: the addres to check
1115 @return: True if we own the address
1118 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1119 source=constants.LOCALHOST_IP_ADDRESS)
1122 def ListVisibleFiles(path):
1123 """Returns a list of visible files in a directory.
1126 @param path: the directory to enumerate
1128 @return: the list of all files not starting with a dot
1131 files = [i for i in os.listdir(path) if not i.startswith(".")]
1136 def GetHomeDir(user, default=None):
1137 """Try to get the homedir of the given user.
1139 The user can be passed either as a string (denoting the name) or as
1140 an integer (denoting the user id). If the user is not found, the
1141 'default' argument is returned, which defaults to None.
1145 if isinstance(user, basestring):
1146 result = pwd.getpwnam(user)
1147 elif isinstance(user, (int, long)):
1148 result = pwd.getpwuid(user)
1150 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1154 return result.pw_dir
1158 """Returns a random UUID.
1160 @note: This is a Linux-specific method as it uses the /proc
1165 f = open("/proc/sys/kernel/random/uuid", "r")
1167 return f.read(128).rstrip("\n")
1172 def GenerateSecret():
1173 """Generates a random secret.
1175 This will generate a pseudo-random secret, and return its sha digest
1176 (so that it can be used where an ASCII string is needed).
1179 @return: a sha1 hexdigest of a block of 64 random bytes
1182 return sha.new(os.urandom(64)).hexdigest()
1185 def ReadFile(file_name, size=None):
1188 @type size: None or int
1189 @param size: Read at most size bytes
1191 @return: the (possibly partial) conent of the file
1194 f = open(file_name, "r")
1204 def WriteFile(file_name, fn=None, data=None,
1205 mode=None, uid=-1, gid=-1,
1206 atime=None, mtime=None, close=True,
1207 dry_run=False, backup=False,
1208 prewrite=None, postwrite=None):
1209 """(Over)write a file atomically.
1211 The file_name and either fn (a function taking one argument, the
1212 file descriptor, and which should write the data to it) or data (the
1213 contents of the file) must be passed. The other arguments are
1214 optional and allow setting the file mode, owner and group, and the
1215 mtime/atime of the file.
1217 If the function doesn't raise an exception, it has succeeded and the
1218 target file has the new contents. If the function has raised an
1219 exception, an existing target file should be unmodified and the
1220 temporary file should be removed.
1222 @type file_name: str
1223 @param file_name: the target filename
1225 @param fn: content writing function, called with
1226 file descriptor as parameter
1228 @param data: contents of the file
1230 @param mode: file mode
1232 @param uid: the owner of the file
1234 @param gid: the group of the file
1236 @param atime: a custom access time to be set on the file
1238 @param mtime: a custom modification time to be set on the file
1239 @type close: boolean
1240 @param close: whether to close file after writing it
1241 @type prewrite: callable
1242 @param prewrite: function to be called before writing content
1243 @type postwrite: callable
1244 @param postwrite: function to be called after writing content
1247 @return: None if the 'close' parameter evaluates to True,
1248 otherwise the file descriptor
1250 @raise errors.ProgrammerError: if any of the arguments are not valid
1253 if not os.path.isabs(file_name):
1254 raise errors.ProgrammerError("Path passed to WriteFile is not"
1255 " absolute: '%s'" % file_name)
1257 if [fn, data].count(None) != 1:
1258 raise errors.ProgrammerError("fn or data required")
1260 if [atime, mtime].count(None) == 1:
1261 raise errors.ProgrammerError("Both atime and mtime must be either"
1264 if backup and not dry_run and os.path.isfile(file_name):
1265 CreateBackup(file_name)
1267 dir_name, base_name = os.path.split(file_name)
1268 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1269 # here we need to make sure we remove the temp file, if any error
1270 # leaves it in place
1272 if uid != -1 or gid != -1:
1273 os.chown(new_name, uid, gid)
1275 os.chmod(new_name, mode)
1276 if callable(prewrite):
1278 if data is not None:
1282 if callable(postwrite):
1285 if atime is not None and mtime is not None:
1286 os.utime(new_name, (atime, mtime))
1288 os.rename(new_name, file_name)
1295 RemoveFile(new_name)
1300 def FirstFree(seq, base=0):
1301 """Returns the first non-existing integer from seq.
1303 The seq argument should be a sorted list of positive integers. The
1304 first time the index of an element is smaller than the element
1305 value, the index will be returned.
1307 The base argument is used to start at a different offset,
1308 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1310 Example: C{[0, 1, 3]} will return I{2}.
1313 @param seq: the sequence to be analyzed.
1315 @param base: use this value as the base index of the sequence
1317 @return: the first non-used index in the sequence
1320 for idx, elem in enumerate(seq):
1321 assert elem >= base, "Passed element is higher than base offset"
1322 if elem > idx + base:
1328 def all(seq, pred=bool):
1329 "Returns True if pred(x) is True for every element in the iterable"
1330 for elem in itertools.ifilterfalse(pred, seq):
1335 def any(seq, pred=bool):
1336 "Returns True if pred(x) is True for at least one element in the iterable"
1337 for elem in itertools.ifilter(pred, seq):
1342 def UniqueSequence(seq):
1343 """Returns a list with unique elements.
1345 Element order is preserved.
1348 @param seq: the sequence with the source elementes
1350 @return: list of unique elements from seq
1354 return [i for i in seq if i not in seen and not seen.add(i)]
1357 def IsValidMac(mac):
1358 """Predicate to check if a MAC address is valid.
1360 Checks wether the supplied MAC address is formally correct, only
1361 accepts colon separated format.
1364 @param mac: the MAC to be validated
1366 @return: True is the MAC seems valid
1369 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1370 return mac_check.match(mac) is not None
1373 def TestDelay(duration):
1374 """Sleep for a fixed amount of time.
1376 @type duration: float
1377 @param duration: the sleep duration
1379 @return: False for negative value, True otherwise
1384 time.sleep(duration)
1388 def _CloseFDNoErr(fd, retries=5):
1389 """Close a file descriptor ignoring errors.
1392 @param fd: the file descriptor
1394 @param retries: how many retries to make, in case we get any
1395 other error than EBADF
1400 except OSError, err:
1401 if err.errno != errno.EBADF:
1403 _CloseFDNoErr(fd, retries - 1)
1404 # else either it's closed already or we're out of retries, so we
1405 # ignore this and go on
1408 def CloseFDs(noclose_fds=None):
1409 """Close file descriptors.
1411 This closes all file descriptors above 2 (i.e. except
1414 @type noclose_fds: list or None
1415 @param noclose_fds: if given, it denotes a list of file descriptor
1416 that should not be closed
1419 # Default maximum for the number of available file descriptors.
1420 if 'SC_OPEN_MAX' in os.sysconf_names:
1422 MAXFD = os.sysconf('SC_OPEN_MAX')
1429 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1430 if (maxfd == resource.RLIM_INFINITY):
1433 # Iterate through and close all file descriptors (except the standard ones)
1434 for fd in range(3, maxfd):
1435 if noclose_fds and fd in noclose_fds:
1440 def Daemonize(logfile):
1441 """Daemonize the current process.
1443 This detaches the current process from the controlling terminal and
1444 runs it in the background as a daemon.
1447 @param logfile: the logfile to which we should redirect stdout/stderr
1449 @returns: the value zero
1457 if (pid == 0): # The first child.
1460 pid = os.fork() # Fork a second child.
1461 if (pid == 0): # The second child.
1465 # exit() or _exit()? See below.
1466 os._exit(0) # Exit parent (the first child) of the second child.
1468 os._exit(0) # Exit parent of the first child.
1472 i = os.open("/dev/null", os.O_RDONLY) # stdin
1473 assert i == 0, "Can't close/reopen stdin"
1474 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1475 assert i == 1, "Can't close/reopen stdout"
1476 # Duplicate standard output to standard error.
1481 def DaemonPidFileName(name):
1482 """Compute a ganeti pid file absolute path
1485 @param name: the daemon name
1487 @return: the full path to the pidfile corresponding to the given
1491 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1494 def WritePidFile(name):
1495 """Write the current process pidfile.
1497 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1500 @param name: the daemon name to use
1501 @raise errors.GenericError: if the pid file already exists and
1502 points to a live process
1506 pidfilename = DaemonPidFileName(name)
1507 if IsProcessAlive(ReadPidFile(pidfilename)):
1508 raise errors.GenericError("%s contains a live process" % pidfilename)
1510 WriteFile(pidfilename, data="%d\n" % pid)
1513 def RemovePidFile(name):
1514 """Remove the current process pidfile.
1516 Any errors are ignored.
1519 @param name: the daemon name used to derive the pidfile name
1523 pidfilename = DaemonPidFileName(name)
1524 # TODO: we could check here that the file contains our pid
1526 RemoveFile(pidfilename)
1531 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1533 """Kill a process given by its pid.
1536 @param pid: The PID to terminate.
1538 @param signal_: The signal to send, by default SIGTERM
1540 @param timeout: The timeout after which, if the process is still alive,
1541 a SIGKILL will be sent. If not positive, no such checking
1543 @type waitpid: boolean
1544 @param waitpid: If true, we should waitpid on this process after
1545 sending signals, since it's our own child and otherwise it
1546 would remain as zombie
1549 def _helper(pid, signal_, wait):
1550 """Simple helper to encapsulate the kill/waitpid sequence"""
1551 os.kill(pid, signal_)
1554 os.waitpid(pid, os.WNOHANG)
1559 # kill with pid=0 == suicide
1560 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1562 if not IsProcessAlive(pid):
1564 _helper(pid, signal_, waitpid)
1568 # Wait up to $timeout seconds
1569 end = time.time() + timeout
1571 while time.time() < end and IsProcessAlive(pid):
1573 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1579 # Make wait time longer for next try
1583 if IsProcessAlive(pid):
1584 # Kill process if it's still alive
1585 _helper(pid, signal.SIGKILL, waitpid)
1588 def FindFile(name, search_path, test=os.path.exists):
1589 """Look for a filesystem object in a given path.
1591 This is an abstract method to search for filesystem object (files,
1592 dirs) under a given search path.
1595 @param name: the name to look for
1596 @type search_path: str
1597 @param search_path: location to start at
1598 @type test: callable
1599 @param test: a function taking one argument that should return True
1600 if the a given object is valid; the default value is
1601 os.path.exists, causing only existing files to be returned
1603 @return: full path to the object if found, None otherwise
1606 for dir_name in search_path:
1607 item_name = os.path.sep.join([dir_name, name])
1613 def CheckVolumeGroupSize(vglist, vgname, minsize):
1614 """Checks if the volume group list is valid.
1616 The function will check if a given volume group is in the list of
1617 volume groups and has a minimum size.
1620 @param vglist: dictionary of volume group names and their size
1622 @param vgname: the volume group we should check
1624 @param minsize: the minimum size we accept
1626 @return: None for success, otherwise the error message
1629 vgsize = vglist.get(vgname, None)
1631 return "volume group '%s' missing" % vgname
1632 elif vgsize < minsize:
1633 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1634 (vgname, minsize, vgsize))
1638 def SplitTime(value):
1639 """Splits time as floating point number into a tuple.
1641 @param value: Time in seconds
1642 @type value: int or float
1643 @return: Tuple containing (seconds, microseconds)
1646 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1648 assert 0 <= seconds, \
1649 "Seconds must be larger than or equal to 0, but are %s" % seconds
1650 assert 0 <= microseconds <= 999999, \
1651 "Microseconds must be 0-999999, but are %s" % microseconds
1653 return (int(seconds), int(microseconds))
1656 def MergeTime(timetuple):
1657 """Merges a tuple into time as a floating point number.
1659 @param timetuple: Time as tuple, (seconds, microseconds)
1660 @type timetuple: tuple
1661 @return: Time as a floating point number expressed in seconds
1664 (seconds, microseconds) = timetuple
1666 assert 0 <= seconds, \
1667 "Seconds must be larger than or equal to 0, but are %s" % seconds
1668 assert 0 <= microseconds <= 999999, \
1669 "Microseconds must be 0-999999, but are %s" % microseconds
1671 return float(seconds) + (float(microseconds) * 0.000001)
1674 def GetNodeDaemonPort():
1675 """Get the node daemon port for this cluster.
1677 Note that this routine does not read a ganeti-specific file, but
1678 instead uses C{socket.getservbyname} to allow pre-customization of
1679 this parameter outside of Ganeti.
1685 port = socket.getservbyname("ganeti-noded", "tcp")
1686 except socket.error:
1687 port = constants.DEFAULT_NODED_PORT
1692 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1693 multithreaded=False):
1694 """Configures the logging module.
1697 @param logfile: the filename to which we should log
1698 @type debug: boolean
1699 @param debug: whether to enable debug messages too or
1700 only those at C{INFO} and above level
1701 @type stderr_logging: boolean
1702 @param stderr_logging: whether we should also log to the standard error
1704 @param program: the name under which we should log messages
1705 @type multithreaded: boolean
1706 @param multithreaded: if True, will add the thread name to the log file
1707 @raise EnvironmentError: if we can't open the log file and
1708 stderr logging is disabled
1711 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1713 fmt += "/%(threadName)s"
1715 fmt += " %(module)s:%(lineno)s"
1716 fmt += " %(levelname)s %(message)s"
1717 formatter = logging.Formatter(fmt)
1719 root_logger = logging.getLogger("")
1720 root_logger.setLevel(logging.NOTSET)
1722 # Remove all previously setup handlers
1723 for handler in root_logger.handlers:
1725 root_logger.removeHandler(handler)
1728 stderr_handler = logging.StreamHandler()
1729 stderr_handler.setFormatter(formatter)
1731 stderr_handler.setLevel(logging.NOTSET)
1733 stderr_handler.setLevel(logging.CRITICAL)
1734 root_logger.addHandler(stderr_handler)
1736 # this can fail, if the logging directories are not setup or we have
1737 # a permisssion problem; in this case, it's best to log but ignore
1738 # the error if stderr_logging is True, and if false we re-raise the
1739 # exception since otherwise we could run but without any logs at all
1741 logfile_handler = logging.FileHandler(logfile)
1742 logfile_handler.setFormatter(formatter)
1744 logfile_handler.setLevel(logging.DEBUG)
1746 logfile_handler.setLevel(logging.INFO)
1747 root_logger.addHandler(logfile_handler)
1748 except EnvironmentError:
1750 logging.exception("Failed to enable logging to file '%s'", logfile)
1752 # we need to re-raise the exception
1756 def TailFile(fname, lines=20):
1757 """Return the last lines from a file.
1759 @note: this function will only read and parse the last 4KB of
1760 the file; if the lines are very long, it could be that less
1761 than the requested number of lines are returned
1763 @param fname: the file name
1765 @param lines: the (maximum) number of lines to return
1768 fd = open(fname, "r")
1772 pos = max(0, pos-4096)
1774 raw_data = fd.read()
1778 rows = raw_data.splitlines()
1779 return rows[-lines:]
1782 def SafeEncode(text):
1783 """Return a 'safe' version of a source string.
1785 This function mangles the input string and returns a version that
1786 should be safe to disply/encode as ASCII. To this end, we first
1787 convert it to ASCII using the 'backslashreplace' encoding which
1788 should get rid of any non-ASCII chars, and then we again encode it
1789 via 'string_escape' which converts '\n' into '\\n' so that log
1790 messages remain one-line.
1792 @type text: str or unicode
1793 @param text: input data
1795 @return: a safe version of text
1798 text = text.encode('ascii', 'backslashreplace')
1799 text = text.encode('string_escape')
1803 def LockedMethod(fn):
1804 """Synchronized object access decorator.
1806 This decorator is intended to protect access to an object using the
1807 object's own lock which is hardcoded to '_lock'.
1810 def _LockDebug(*args, **kwargs):
1812 logging.debug(*args, **kwargs)
1814 def wrapper(self, *args, **kwargs):
1815 assert hasattr(self, '_lock')
1817 _LockDebug("Waiting for %s", lock)
1820 _LockDebug("Acquired %s", lock)
1821 result = fn(self, *args, **kwargs)
1823 _LockDebug("Releasing %s", lock)
1825 _LockDebug("Released %s", lock)
1831 """Locks a file using POSIX locks.
1834 @param fd: the file descriptor we need to lock
1838 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1839 except IOError, err:
1840 if err.errno == errno.EAGAIN:
1841 raise errors.LockError("File already locked")
1845 class FileLock(object):
1846 """Utility class for file locks.
1849 def __init__(self, filename):
1850 """Constructor for FileLock.
1852 This will open the file denoted by the I{filename} argument.
1855 @param filename: path to the file to be locked
1858 self.filename = filename
1859 self.fd = open(self.filename, "w")
1865 """Close the file and release the lock.
1872 def _flock(self, flag, blocking, timeout, errmsg):
1873 """Wrapper for fcntl.flock.
1876 @param flag: operation flag
1877 @type blocking: bool
1878 @param blocking: whether the operation should be done in blocking mode.
1879 @type timeout: None or float
1880 @param timeout: for how long the operation should be retried (implies
1882 @type errmsg: string
1883 @param errmsg: error message in case operation fails.
1886 assert self.fd, "Lock was closed"
1887 assert timeout is None or timeout >= 0, \
1888 "If specified, timeout must be positive"
1890 if timeout is not None:
1891 flag |= fcntl.LOCK_NB
1892 timeout_end = time.time() + timeout
1894 # Blocking doesn't have effect with timeout
1896 flag |= fcntl.LOCK_NB
1902 fcntl.flock(self.fd, flag)
1904 except IOError, err:
1905 if err.errno in (errno.EAGAIN, ):
1906 if timeout_end is not None and time.time() < timeout_end:
1907 # Wait before trying again
1908 time.sleep(max(0.1, min(1.0, timeout)))
1910 raise errors.LockError(errmsg)
1912 logging.exception("fcntl.flock failed")
1915 def Exclusive(self, blocking=False, timeout=None):
1916 """Locks the file in exclusive mode.
1918 @type blocking: boolean
1919 @param blocking: whether to block and wait until we
1920 can lock the file or return immediately
1921 @type timeout: int or None
1922 @param timeout: if not None, the duration to wait for the lock
1926 self._flock(fcntl.LOCK_EX, blocking, timeout,
1927 "Failed to lock %s in exclusive mode" % self.filename)
1929 def Shared(self, blocking=False, timeout=None):
1930 """Locks the file in shared mode.
1932 @type blocking: boolean
1933 @param blocking: whether to block and wait until we
1934 can lock the file or return immediately
1935 @type timeout: int or None
1936 @param timeout: if not None, the duration to wait for the lock
1940 self._flock(fcntl.LOCK_SH, blocking, timeout,
1941 "Failed to lock %s in shared mode" % self.filename)
1943 def Unlock(self, blocking=True, timeout=None):
1944 """Unlocks the file.
1946 According to C{flock(2)}, unlocking can also be a nonblocking
1949 To make a non-blocking request, include LOCK_NB with any of the above
1952 @type blocking: boolean
1953 @param blocking: whether to block and wait until we
1954 can lock the file or return immediately
1955 @type timeout: int or None
1956 @param timeout: if not None, the duration to wait for the lock
1960 self._flock(fcntl.LOCK_UN, blocking, timeout,
1961 "Failed to unlock %s" % self.filename)
1964 class SignalHandler(object):
1965 """Generic signal handler class.
1967 It automatically restores the original handler when deconstructed or
1968 when L{Reset} is called. You can either pass your own handler
1969 function in or query the L{called} attribute to detect whether the
1973 @ivar signum: the signals we handle
1974 @type called: boolean
1975 @ivar called: tracks whether any of the signals have been raised
1978 def __init__(self, signum):
1979 """Constructs a new SignalHandler instance.
1981 @type signum: int or list of ints
1982 @param signum: Single signal number or set of signal numbers
1985 if isinstance(signum, (int, long)):
1986 self.signum = set([signum])
1988 self.signum = set(signum)
1994 for signum in self.signum:
1996 prev_handler = signal.signal(signum, self._HandleSignal)
1998 self._previous[signum] = prev_handler
2000 # Restore previous handler
2001 signal.signal(signum, prev_handler)
2004 # Reset all handlers
2006 # Here we have a race condition: a handler may have already been called,
2007 # but there's not much we can do about it at this point.
2014 """Restore previous handler.
2016 This will reset all the signals to their previous handlers.
2019 for signum, prev_handler in self._previous.items():
2020 signal.signal(signum, prev_handler)
2021 # If successful, remove from dict
2022 del self._previous[signum]
2025 """Unsets the L{called} flag.
2027 This function can be used in case a signal may arrive several times.
2032 def _HandleSignal(self, signum, frame):
2033 """Actual signal handling function.
2036 # This is not nice and not absolutely atomic, but it appears to be the only
2037 # solution in Python -- there are no atomic types.
2041 class FieldSet(object):
2042 """A simple field set.
2044 Among the features are:
2045 - checking if a string is among a list of static string or regex objects
2046 - checking if a whole list of string matches
2047 - returning the matching groups from a regex match
2049 Internally, all fields are held as regular expression objects.
2052 def __init__(self, *items):
2053 self.items = [re.compile("^%s$" % value) for value in items]
2055 def Extend(self, other_set):
2056 """Extend the field set with the items from another one"""
2057 self.items.extend(other_set.items)
2059 def Matches(self, field):
2060 """Checks if a field matches the current set
2063 @param field: the string to match
2064 @return: either False or a regular expression match object
2067 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2071 def NonMatching(self, items):
2072 """Returns the list of fields not matching the current set
2075 @param items: the list of fields to check
2077 @return: list of non-matching fields
2080 return [val for val in items if not self.Matches(val)]