4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
48 from cStringIO import StringIO
50 from ganeti import errors
51 from ganeti import constants
55 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60 #: when set to True, L{RunCmd} is disabled
64 class RunResult(object):
65 """Holds the result of running external programs.
68 @ivar exit_code: the exit code of the program, or None (if the program
70 @type signal: int or None
71 @ivar signal: the signal that caused the program to finish, or None
72 (if the program wasn't terminated by a signal)
74 @ivar stdout: the standard output of the program
76 @ivar stderr: the standard error of the program
78 @ivar failed: True in case the program was
79 terminated by a signal or exited with a non-zero exit code
80 @ivar fail_reason: a string detailing the termination reason
83 __slots__ = ["exit_code", "signal", "stdout", "stderr",
84 "failed", "fail_reason", "cmd"]
87 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
89 self.exit_code = exit_code
93 self.failed = (signal_ is not None or exit_code != 0)
95 if self.signal is not None:
96 self.fail_reason = "terminated by signal %s" % self.signal
97 elif self.exit_code is not None:
98 self.fail_reason = "exited with exit code %s" % self.exit_code
100 self.fail_reason = "unable to determine termination reason"
103 logging.debug("Command '%s' failed (%s); output: %s",
104 self.cmd, self.fail_reason, self.output)
106 def _GetOutput(self):
107 """Returns the combined stdout and stderr for easier usage.
110 return self.stdout + self.stderr
112 output = property(_GetOutput, None, None, "Return full output")
115 def RunCmd(cmd, env=None, output=None, cwd='/'):
116 """Execute a (shell) command.
118 The command should not read from its standard input, as it will be
121 @type cmd: string or list
122 @param cmd: Command to run
124 @param env: Additional environment
126 @param output: if desired, the output of the command can be
127 saved in a file instead of the RunResult instance; this
128 parameter denotes the file name (if not None)
130 @param cwd: if specified, will be used as the working
131 directory for the command; the default will be /
133 @return: RunResult instance
134 @raise erors.ProgrammerError: if we call this when forks are disabled
138 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
140 if isinstance(cmd, list):
141 cmd = [str(val) for val in cmd]
142 strcmd = " ".join(cmd)
147 logging.debug("RunCmd '%s'", strcmd)
149 cmd_env = os.environ.copy()
150 cmd_env["LC_ALL"] = "C"
155 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
157 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
167 return RunResult(exitcode, signal_, out, err, strcmd)
170 def _RunCmdPipe(cmd, env, via_shell, cwd):
171 """Run a command and return its output.
173 @type cmd: string or list
174 @param cmd: Command to run
176 @param env: The environment to use
177 @type via_shell: bool
178 @param via_shell: if we should run via the shell
180 @param cwd: the working directory for the program
182 @return: (out, err, status)
185 poller = select.poll()
186 child = subprocess.Popen(cmd, shell=via_shell,
187 stderr=subprocess.PIPE,
188 stdout=subprocess.PIPE,
189 stdin=subprocess.PIPE,
190 close_fds=True, env=env,
194 poller.register(child.stdout, select.POLLIN)
195 poller.register(child.stderr, select.POLLIN)
199 child.stdout.fileno(): (out, child.stdout),
200 child.stderr.fileno(): (err, child.stderr),
203 status = fcntl.fcntl(fd, fcntl.F_GETFL)
204 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
208 pollresult = poller.poll()
209 except EnvironmentError, eerr:
210 if eerr.errno == errno.EINTR:
213 except select.error, serr:
214 if serr[0] == errno.EINTR:
218 for fd, event in pollresult:
219 if event & select.POLLIN or event & select.POLLPRI:
220 data = fdmap[fd][1].read()
221 # no data from read signifies EOF (the same as POLLHUP)
223 poller.unregister(fd)
226 fdmap[fd][0].write(data)
227 if (event & select.POLLNVAL or event & select.POLLHUP or
228 event & select.POLLERR):
229 poller.unregister(fd)
235 status = child.wait()
236 return out, err, status
239 def _RunCmdFile(cmd, env, via_shell, output, cwd):
240 """Run a command and save its output to a file.
242 @type cmd: string or list
243 @param cmd: Command to run
245 @param env: The environment to use
246 @type via_shell: bool
247 @param via_shell: if we should run via the shell
249 @param output: the filename in which to save the output
251 @param cwd: the working directory for the program
253 @return: the exit status
256 fh = open(output, "a")
258 child = subprocess.Popen(cmd, shell=via_shell,
259 stderr=subprocess.STDOUT,
261 stdin=subprocess.PIPE,
262 close_fds=True, env=env,
266 status = child.wait()
272 def RemoveFile(filename):
273 """Remove a file ignoring some errors.
275 Remove a file, ignoring non-existing ones or directories. Other
279 @param filename: the file to be removed
285 if err.errno not in (errno.ENOENT, errno.EISDIR):
289 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
293 @param old: Original path
297 @param mkdir: Whether to create target directory if it doesn't exist
298 @type mkdir_mode: int
299 @param mkdir_mode: Mode for newly created directories
303 return os.rename(old, new)
305 # In at least one use case of this function, the job queue, directory
306 # creation is very rare. Checking for the directory before renaming is not
308 if mkdir and err.errno == errno.ENOENT:
309 # Create directory and try again
310 os.makedirs(os.path.dirname(new), mkdir_mode)
311 return os.rename(old, new)
315 def _FingerprintFile(filename):
316 """Compute the fingerprint of a file.
318 If the file does not exist, a None will be returned
322 @param filename: the filename to checksum
324 @return: the hex digest of the sha checksum of the contents
328 if not (os.path.exists(filename) and os.path.isfile(filename)):
341 return fp.hexdigest()
344 def FingerprintFiles(files):
345 """Compute fingerprints for a list of files.
348 @param files: the list of filename to fingerprint
350 @return: a dictionary filename: fingerprint, holding only
356 for filename in files:
357 cksum = _FingerprintFile(filename)
359 ret[filename] = cksum
364 def CheckDict(target, template, logname=None):
365 """Ensure a dictionary has a required set of keys.
367 For the given dictionaries I{target} and I{template}, ensure
368 I{target} has all the keys from I{template}. Missing keys are added
369 with values from template.
372 @param target: the dictionary to update
374 @param template: the dictionary holding the default values
375 @type logname: str or None
376 @param logname: if not None, causes the missing keys to be
377 logged with this name
384 target[k] = template[k]
386 if missing and logname:
387 logging.warning('%s missing keys %s', logname, ', '.join(missing))
390 def ForceDictType(target, key_types, allowed_values=None):
391 """Force the values of a dict to have certain types.
394 @param target: the dict to update
395 @type key_types: dict
396 @param key_types: dict mapping target dict keys to types
397 in constants.ENFORCEABLE_TYPES
398 @type allowed_values: list
399 @keyword allowed_values: list of specially allowed values
402 if allowed_values is None:
406 if key not in key_types:
407 msg = "Unknown key '%s'" % key
408 raise errors.TypeEnforcementError(msg)
410 if target[key] in allowed_values:
413 type = key_types[key]
414 if type not in constants.ENFORCEABLE_TYPES:
415 msg = "'%s' has non-enforceable type %s" % (key, type)
416 raise errors.ProgrammerError(msg)
418 if type == constants.VTYPE_STRING:
419 if not isinstance(target[key], basestring):
420 if isinstance(target[key], bool) and not target[key]:
423 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
424 raise errors.TypeEnforcementError(msg)
425 elif type == constants.VTYPE_BOOL:
426 if isinstance(target[key], basestring) and target[key]:
427 if target[key].lower() == constants.VALUE_FALSE:
429 elif target[key].lower() == constants.VALUE_TRUE:
432 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
433 raise errors.TypeEnforcementError(msg)
438 elif type == constants.VTYPE_SIZE:
440 target[key] = ParseUnit(target[key])
441 except errors.UnitParseError, err:
442 msg = "'%s' (value %s) is not a valid size. error: %s" % \
443 (key, target[key], err)
444 raise errors.TypeEnforcementError(msg)
445 elif type == constants.VTYPE_INT:
447 target[key] = int(target[key])
448 except (ValueError, TypeError):
449 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
450 raise errors.TypeEnforcementError(msg)
453 def IsProcessAlive(pid):
454 """Check if a given pid exists on the system.
456 @note: zombie status is not handled, so zombie processes
457 will be returned as alive
459 @param pid: the process ID to check
461 @return: True if the process exists
468 os.stat("/proc/%d/status" % pid)
470 except EnvironmentError, err:
471 if err.errno in (errno.ENOENT, errno.ENOTDIR):
476 def ReadPidFile(pidfile):
477 """Read a pid from a file.
479 @type pidfile: string
480 @param pidfile: path to the file containing the pid
482 @return: The process id, if the file exists and contains a valid PID,
487 pf = open(pidfile, 'r')
488 except EnvironmentError, err:
489 if err.errno != errno.ENOENT:
490 logging.exception("Can't read pid file?!")
495 except ValueError, err:
496 logging.info("Can't parse pid file contents", exc_info=True)
502 def MatchNameComponent(key, name_list):
503 """Try to match a name against a list.
505 This function will try to match a name like test1 against a list
506 like C{['test1.example.com', 'test2.example.com', ...]}. Against
507 this list, I{'test1'} as well as I{'test1.example'} will match, but
508 not I{'test1.ex'}. A multiple match will be considered as no match
509 at all (e.g. I{'test1'} against C{['test1.example.com',
510 'test1.example.org']}).
513 @param key: the name to be searched
514 @type name_list: list
515 @param name_list: the list of strings against which to search the key
518 @return: None if there is no match I{or} if there are multiple matches,
519 otherwise the element from the list which matches
522 mo = re.compile("^%s(\..*)?$" % re.escape(key))
523 names_filtered = [name for name in name_list if mo.match(name) is not None]
524 if len(names_filtered) != 1:
526 return names_filtered[0]
530 """Class implementing resolver and hostname functionality
533 def __init__(self, name=None):
534 """Initialize the host name object.
536 If the name argument is not passed, it will use this system's
541 name = self.SysName()
544 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
545 self.ip = self.ipaddrs[0]
548 """Returns the hostname without domain.
551 return self.name.split('.')[0]
555 """Return the current system's name.
557 This is simply a wrapper over C{socket.gethostname()}.
560 return socket.gethostname()
563 def LookupHostname(hostname):
567 @param hostname: hostname to look up
570 @return: a tuple (name, aliases, ipaddrs) as returned by
571 C{socket.gethostbyname_ex}
572 @raise errors.ResolverError: in case of errors in resolving
576 result = socket.gethostbyname_ex(hostname)
577 except socket.gaierror, err:
578 # hostname not found in DNS
579 raise errors.ResolverError(hostname, err.args[0], err.args[1])
584 def ListVolumeGroups():
585 """List volume groups and their size
589 Dictionary with keys volume name and values
590 the size of the volume
593 command = "vgs --noheadings --units m --nosuffix -o name,size"
594 result = RunCmd(command)
599 for line in result.stdout.splitlines():
601 name, size = line.split()
602 size = int(float(size))
603 except (IndexError, ValueError), err:
604 logging.error("Invalid output from vgs (%s): %s", err, line)
612 def BridgeExists(bridge):
613 """Check whether the given bridge exists in the system
616 @param bridge: the bridge name to check
618 @return: True if it does
621 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
624 def NiceSort(name_list):
625 """Sort a list of strings based on digit and non-digit groupings.
627 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
628 will sort the list in the logical order C{['a1', 'a2', 'a10',
631 The sort algorithm breaks each name in groups of either only-digits
632 or no-digits. Only the first eight such groups are considered, and
633 after that we just use what's left of the string.
635 @type name_list: list
636 @param name_list: the names to be sorted
638 @return: a copy of the name list sorted with our algorithm
641 _SORTER_BASE = "(\D+|\d+)"
642 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
643 _SORTER_BASE, _SORTER_BASE,
644 _SORTER_BASE, _SORTER_BASE,
645 _SORTER_BASE, _SORTER_BASE)
646 _SORTER_RE = re.compile(_SORTER_FULL)
647 _SORTER_NODIGIT = re.compile("^\D*$")
649 """Attempts to convert a variable to integer."""
650 if val is None or _SORTER_NODIGIT.match(val):
655 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
656 for name in name_list]
658 return [tup[1] for tup in to_sort]
661 def TryConvert(fn, val):
662 """Try to convert a value ignoring errors.
664 This function tries to apply function I{fn} to I{val}. If no
665 C{ValueError} or C{TypeError} exceptions are raised, it will return
666 the result, else it will return the original value. Any other
667 exceptions are propagated to the caller.
670 @param fn: function to apply to the value
671 @param val: the value to be converted
672 @return: The converted value if the conversion was successful,
673 otherwise the original value.
678 except (ValueError, TypeError), err:
684 """Verifies the syntax of an IPv4 address.
686 This function checks if the IPv4 address passes is valid or not based
687 on syntax (not IP range, class calculations, etc.).
690 @param ip: the address to be checked
691 @rtype: a regular expression match object
692 @return: a regular epression match object, or None if the
696 unit = "(0|[1-9]\d{0,2})"
697 #TODO: convert and return only boolean
698 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
701 def IsValidShellParam(word):
702 """Verifies is the given word is safe from the shell's p.o.v.
704 This means that we can pass this to a command via the shell and be
705 sure that it doesn't alter the command line and is passed as such to
708 Note that we are overly restrictive here, in order to be on the safe
712 @param word: the word to check
714 @return: True if the word is 'safe'
717 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
720 def BuildShellCmd(template, *args):
721 """Build a safe shell command line from the given arguments.
723 This function will check all arguments in the args list so that they
724 are valid shell parameters (i.e. they don't contain shell
725 metacharaters). If everything is ok, it will return the result of
729 @param template: the string holding the template for the
732 @return: the expanded command line
736 if not IsValidShellParam(word):
737 raise errors.ProgrammerError("Shell argument '%s' contains"
738 " invalid characters" % word)
739 return template % args
742 def FormatUnit(value, units):
743 """Formats an incoming number of MiB with the appropriate unit.
746 @param value: integer representing the value in MiB (1048576)
748 @param units: the type of formatting we should do:
749 - 'h' for automatic scaling
754 @return: the formatted value (with suffix)
757 if units not in ('m', 'g', 't', 'h'):
758 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
762 if units == 'm' or (units == 'h' and value < 1024):
765 return "%d%s" % (round(value, 0), suffix)
767 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
770 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
775 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
778 def ParseUnit(input_string):
779 """Tries to extract number and scale from the given string.
781 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
782 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
783 is always an int in MiB.
786 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
788 raise errors.UnitParseError("Invalid format")
790 value = float(m.groups()[0])
794 lcunit = unit.lower()
798 if lcunit in ('m', 'mb', 'mib'):
799 # Value already in MiB
802 elif lcunit in ('g', 'gb', 'gib'):
805 elif lcunit in ('t', 'tb', 'tib'):
809 raise errors.UnitParseError("Unknown unit: %s" % unit)
811 # Make sure we round up
812 if int(value) < value:
815 # Round up to the next multiple of 4
818 value += 4 - value % 4
823 def AddAuthorizedKey(file_name, key):
824 """Adds an SSH public key to an authorized_keys file.
827 @param file_name: path to authorized_keys file
829 @param key: string containing key
832 key_fields = key.split()
834 f = open(file_name, 'a+')
838 # Ignore whitespace changes
839 if line.split() == key_fields:
841 nl = line.endswith('\n')
845 f.write(key.rstrip('\r\n'))
852 def RemoveAuthorizedKey(file_name, key):
853 """Removes an SSH public key from an authorized_keys file.
856 @param file_name: path to authorized_keys file
858 @param key: string containing key
861 key_fields = key.split()
863 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
865 out = os.fdopen(fd, 'w')
867 f = open(file_name, 'r')
870 # Ignore whitespace changes while comparing lines
871 if line.split() != key_fields:
875 os.rename(tmpname, file_name)
885 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
886 """Sets the name of an IP address and hostname in /etc/hosts.
889 @param file_name: path to the file to modify (usually C{/etc/hosts})
891 @param ip: the IP address
893 @param hostname: the hostname to be added
895 @param aliases: the list of aliases to add for the hostname
898 # FIXME: use WriteFile + fn rather than duplicating its efforts
899 # Ensure aliases are unique
900 aliases = UniqueSequence([hostname] + aliases)[1:]
902 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
904 out = os.fdopen(fd, 'w')
906 f = open(file_name, 'r')
909 fields = line.split()
910 if fields and not fields[0].startswith('#') and ip == fields[0]:
914 out.write("%s\t%s" % (ip, hostname))
916 out.write(" %s" % ' '.join(aliases))
921 os.chmod(tmpname, 0644)
922 os.rename(tmpname, file_name)
932 def AddHostToEtcHosts(hostname):
933 """Wrapper around SetEtcHostsEntry.
936 @param hostname: a hostname that will be resolved and added to
937 L{constants.ETC_HOSTS}
940 hi = HostInfo(name=hostname)
941 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
944 def RemoveEtcHostsEntry(file_name, hostname):
945 """Removes a hostname from /etc/hosts.
947 IP addresses without names are removed from the file.
950 @param file_name: path to the file to modify (usually C{/etc/hosts})
952 @param hostname: the hostname to be removed
955 # FIXME: use WriteFile + fn rather than duplicating its efforts
956 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
958 out = os.fdopen(fd, 'w')
960 f = open(file_name, 'r')
963 fields = line.split()
964 if len(fields) > 1 and not fields[0].startswith('#'):
966 if hostname in names:
967 while hostname in names:
968 names.remove(hostname)
970 out.write("%s %s\n" % (fields[0], ' '.join(names)))
977 os.chmod(tmpname, 0644)
978 os.rename(tmpname, file_name)
988 def RemoveHostFromEtcHosts(hostname):
989 """Wrapper around RemoveEtcHostsEntry.
992 @param hostname: hostname that will be resolved and its
993 full and shot name will be removed from
994 L{constants.ETC_HOSTS}
997 hi = HostInfo(name=hostname)
998 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
999 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1002 def CreateBackup(file_name):
1003 """Creates a backup of a file.
1005 @type file_name: str
1006 @param file_name: file to be backed up
1008 @return: the path to the newly created backup
1009 @raise errors.ProgrammerError: for invalid file names
1012 if not os.path.isfile(file_name):
1013 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1016 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1017 dir_name = os.path.dirname(file_name)
1019 fsrc = open(file_name, 'rb')
1021 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1022 fdst = os.fdopen(fd, 'wb')
1024 shutil.copyfileobj(fsrc, fdst)
1033 def ShellQuote(value):
1034 """Quotes shell argument according to POSIX.
1037 @param value: the argument to be quoted
1039 @return: the quoted value
1042 if _re_shell_unquoted.match(value):
1045 return "'%s'" % value.replace("'", "'\\''")
1048 def ShellQuoteArgs(args):
1049 """Quotes a list of shell arguments.
1052 @param args: list of arguments to be quoted
1054 @return: the quoted arguments concatenaned with spaces
1057 return ' '.join([ShellQuote(i) for i in args])
1060 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1061 """Simple ping implementation using TCP connect(2).
1063 Check if the given IP is reachable by doing attempting a TCP connect
1067 @param target: the IP or hostname to ping
1069 @param port: the port to connect to
1071 @param timeout: the timeout on the connection attemp
1072 @type live_port_needed: boolean
1073 @param live_port_needed: whether a closed port will cause the
1074 function to return failure, as if there was a timeout
1075 @type source: str or None
1076 @param source: if specified, will cause the connect to be made
1077 from this specific source address; failures to bind other
1078 than C{EADDRNOTAVAIL} will be ignored
1081 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1085 if source is not None:
1087 sock.bind((source, 0))
1088 except socket.error, (errcode, errstring):
1089 if errcode == errno.EADDRNOTAVAIL:
1092 sock.settimeout(timeout)
1095 sock.connect((target, port))
1098 except socket.timeout:
1100 except socket.error, (errcode, errstring):
1101 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1106 def OwnIpAddress(address):
1107 """Check if the current host has the the given IP address.
1109 Currently this is done by TCP-pinging the address from the loopback
1112 @type address: string
1113 @param address: the addres to check
1115 @return: True if we own the address
1118 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1119 source=constants.LOCALHOST_IP_ADDRESS)
1122 def ListVisibleFiles(path):
1123 """Returns a list of visible files in a directory.
1126 @param path: the directory to enumerate
1128 @return: the list of all files not starting with a dot
1131 files = [i for i in os.listdir(path) if not i.startswith(".")]
1136 def GetHomeDir(user, default=None):
1137 """Try to get the homedir of the given user.
1139 The user can be passed either as a string (denoting the name) or as
1140 an integer (denoting the user id). If the user is not found, the
1141 'default' argument is returned, which defaults to None.
1145 if isinstance(user, basestring):
1146 result = pwd.getpwnam(user)
1147 elif isinstance(user, (int, long)):
1148 result = pwd.getpwuid(user)
1150 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1154 return result.pw_dir
1158 """Returns a random UUID.
1160 @note: This is a Linux-specific method as it uses the /proc
1165 f = open("/proc/sys/kernel/random/uuid", "r")
1167 return f.read(128).rstrip("\n")
1172 def GenerateSecret():
1173 """Generates a random secret.
1175 This will generate a pseudo-random secret, and return its sha digest
1176 (so that it can be used where an ASCII string is needed).
1179 @return: a sha1 hexdigest of a block of 64 random bytes
1182 return sha.new(os.urandom(64)).hexdigest()
1185 def EnsureDirs(dirs):
1186 """Make required directories, if they don't exist.
1188 @param dirs: list of tuples (dir_name, dir_mode)
1189 @type dirs: list of (string, integer)
1192 for dir_name, dir_mode in dirs:
1194 os.mkdir(dir_name, dir_mode)
1195 except EnvironmentError, err:
1196 if err.errno != errno.EEXIST:
1197 raise errors.GenericError("Cannot create needed directory"
1198 " '%s': %s" % (dir_name, err))
1199 if not os.path.isdir(dir_name):
1200 raise errors.GenericError("%s is not a directory" % dir_name)
1203 def ReadFile(file_name, size=None):
1206 @type size: None or int
1207 @param size: Read at most size bytes
1209 @return: the (possibly partial) conent of the file
1212 f = open(file_name, "r")
1222 def WriteFile(file_name, fn=None, data=None,
1223 mode=None, uid=-1, gid=-1,
1224 atime=None, mtime=None, close=True,
1225 dry_run=False, backup=False,
1226 prewrite=None, postwrite=None):
1227 """(Over)write a file atomically.
1229 The file_name and either fn (a function taking one argument, the
1230 file descriptor, and which should write the data to it) or data (the
1231 contents of the file) must be passed. The other arguments are
1232 optional and allow setting the file mode, owner and group, and the
1233 mtime/atime of the file.
1235 If the function doesn't raise an exception, it has succeeded and the
1236 target file has the new contents. If the function has raised an
1237 exception, an existing target file should be unmodified and the
1238 temporary file should be removed.
1240 @type file_name: str
1241 @param file_name: the target filename
1243 @param fn: content writing function, called with
1244 file descriptor as parameter
1246 @param data: contents of the file
1248 @param mode: file mode
1250 @param uid: the owner of the file
1252 @param gid: the group of the file
1254 @param atime: a custom access time to be set on the file
1256 @param mtime: a custom modification time to be set on the file
1257 @type close: boolean
1258 @param close: whether to close file after writing it
1259 @type prewrite: callable
1260 @param prewrite: function to be called before writing content
1261 @type postwrite: callable
1262 @param postwrite: function to be called after writing content
1265 @return: None if the 'close' parameter evaluates to True,
1266 otherwise the file descriptor
1268 @raise errors.ProgrammerError: if any of the arguments are not valid
1271 if not os.path.isabs(file_name):
1272 raise errors.ProgrammerError("Path passed to WriteFile is not"
1273 " absolute: '%s'" % file_name)
1275 if [fn, data].count(None) != 1:
1276 raise errors.ProgrammerError("fn or data required")
1278 if [atime, mtime].count(None) == 1:
1279 raise errors.ProgrammerError("Both atime and mtime must be either"
1282 if backup and not dry_run and os.path.isfile(file_name):
1283 CreateBackup(file_name)
1285 dir_name, base_name = os.path.split(file_name)
1286 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1288 # here we need to make sure we remove the temp file, if any error
1289 # leaves it in place
1291 if uid != -1 or gid != -1:
1292 os.chown(new_name, uid, gid)
1294 os.chmod(new_name, mode)
1295 if callable(prewrite):
1297 if data is not None:
1301 if callable(postwrite):
1304 if atime is not None and mtime is not None:
1305 os.utime(new_name, (atime, mtime))
1307 os.rename(new_name, file_name)
1316 RemoveFile(new_name)
1321 def FirstFree(seq, base=0):
1322 """Returns the first non-existing integer from seq.
1324 The seq argument should be a sorted list of positive integers. The
1325 first time the index of an element is smaller than the element
1326 value, the index will be returned.
1328 The base argument is used to start at a different offset,
1329 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1331 Example: C{[0, 1, 3]} will return I{2}.
1334 @param seq: the sequence to be analyzed.
1336 @param base: use this value as the base index of the sequence
1338 @return: the first non-used index in the sequence
1341 for idx, elem in enumerate(seq):
1342 assert elem >= base, "Passed element is higher than base offset"
1343 if elem > idx + base:
1349 def all(seq, pred=bool):
1350 "Returns True if pred(x) is True for every element in the iterable"
1351 for elem in itertools.ifilterfalse(pred, seq):
1356 def any(seq, pred=bool):
1357 "Returns True if pred(x) is True for at least one element in the iterable"
1358 for elem in itertools.ifilter(pred, seq):
1363 def UniqueSequence(seq):
1364 """Returns a list with unique elements.
1366 Element order is preserved.
1369 @param seq: the sequence with the source elementes
1371 @return: list of unique elements from seq
1375 return [i for i in seq if i not in seen and not seen.add(i)]
1378 def IsValidMac(mac):
1379 """Predicate to check if a MAC address is valid.
1381 Checks wether the supplied MAC address is formally correct, only
1382 accepts colon separated format.
1385 @param mac: the MAC to be validated
1387 @return: True is the MAC seems valid
1390 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1391 return mac_check.match(mac) is not None
1394 def TestDelay(duration):
1395 """Sleep for a fixed amount of time.
1397 @type duration: float
1398 @param duration: the sleep duration
1400 @return: False for negative value, True otherwise
1405 time.sleep(duration)
1409 def _CloseFDNoErr(fd, retries=5):
1410 """Close a file descriptor ignoring errors.
1413 @param fd: the file descriptor
1415 @param retries: how many retries to make, in case we get any
1416 other error than EBADF
1421 except OSError, err:
1422 if err.errno != errno.EBADF:
1424 _CloseFDNoErr(fd, retries - 1)
1425 # else either it's closed already or we're out of retries, so we
1426 # ignore this and go on
1429 def CloseFDs(noclose_fds=None):
1430 """Close file descriptors.
1432 This closes all file descriptors above 2 (i.e. except
1435 @type noclose_fds: list or None
1436 @param noclose_fds: if given, it denotes a list of file descriptor
1437 that should not be closed
1440 # Default maximum for the number of available file descriptors.
1441 if 'SC_OPEN_MAX' in os.sysconf_names:
1443 MAXFD = os.sysconf('SC_OPEN_MAX')
1450 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1451 if (maxfd == resource.RLIM_INFINITY):
1454 # Iterate through and close all file descriptors (except the standard ones)
1455 for fd in range(3, maxfd):
1456 if noclose_fds and fd in noclose_fds:
1461 def Daemonize(logfile):
1462 """Daemonize the current process.
1464 This detaches the current process from the controlling terminal and
1465 runs it in the background as a daemon.
1468 @param logfile: the logfile to which we should redirect stdout/stderr
1470 @return: the value zero
1478 if (pid == 0): # The first child.
1481 pid = os.fork() # Fork a second child.
1482 if (pid == 0): # The second child.
1486 # exit() or _exit()? See below.
1487 os._exit(0) # Exit parent (the first child) of the second child.
1489 os._exit(0) # Exit parent of the first child.
1493 i = os.open("/dev/null", os.O_RDONLY) # stdin
1494 assert i == 0, "Can't close/reopen stdin"
1495 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1496 assert i == 1, "Can't close/reopen stdout"
1497 # Duplicate standard output to standard error.
1502 def DaemonPidFileName(name):
1503 """Compute a ganeti pid file absolute path
1506 @param name: the daemon name
1508 @return: the full path to the pidfile corresponding to the given
1512 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1515 def WritePidFile(name):
1516 """Write the current process pidfile.
1518 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1521 @param name: the daemon name to use
1522 @raise errors.GenericError: if the pid file already exists and
1523 points to a live process
1527 pidfilename = DaemonPidFileName(name)
1528 if IsProcessAlive(ReadPidFile(pidfilename)):
1529 raise errors.GenericError("%s contains a live process" % pidfilename)
1531 WriteFile(pidfilename, data="%d\n" % pid)
1534 def RemovePidFile(name):
1535 """Remove the current process pidfile.
1537 Any errors are ignored.
1540 @param name: the daemon name used to derive the pidfile name
1544 pidfilename = DaemonPidFileName(name)
1545 # TODO: we could check here that the file contains our pid
1547 RemoveFile(pidfilename)
1552 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1554 """Kill a process given by its pid.
1557 @param pid: The PID to terminate.
1559 @param signal_: The signal to send, by default SIGTERM
1561 @param timeout: The timeout after which, if the process is still alive,
1562 a SIGKILL will be sent. If not positive, no such checking
1564 @type waitpid: boolean
1565 @param waitpid: If true, we should waitpid on this process after
1566 sending signals, since it's our own child and otherwise it
1567 would remain as zombie
1570 def _helper(pid, signal_, wait):
1571 """Simple helper to encapsulate the kill/waitpid sequence"""
1572 os.kill(pid, signal_)
1575 os.waitpid(pid, os.WNOHANG)
1580 # kill with pid=0 == suicide
1581 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1583 if not IsProcessAlive(pid):
1585 _helper(pid, signal_, waitpid)
1589 # Wait up to $timeout seconds
1590 end = time.time() + timeout
1592 while time.time() < end and IsProcessAlive(pid):
1594 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1600 # Make wait time longer for next try
1604 if IsProcessAlive(pid):
1605 # Kill process if it's still alive
1606 _helper(pid, signal.SIGKILL, waitpid)
1609 def FindFile(name, search_path, test=os.path.exists):
1610 """Look for a filesystem object in a given path.
1612 This is an abstract method to search for filesystem object (files,
1613 dirs) under a given search path.
1616 @param name: the name to look for
1617 @type search_path: str
1618 @param search_path: location to start at
1619 @type test: callable
1620 @param test: a function taking one argument that should return True
1621 if the a given object is valid; the default value is
1622 os.path.exists, causing only existing files to be returned
1624 @return: full path to the object if found, None otherwise
1627 for dir_name in search_path:
1628 item_name = os.path.sep.join([dir_name, name])
1634 def CheckVolumeGroupSize(vglist, vgname, minsize):
1635 """Checks if the volume group list is valid.
1637 The function will check if a given volume group is in the list of
1638 volume groups and has a minimum size.
1641 @param vglist: dictionary of volume group names and their size
1643 @param vgname: the volume group we should check
1645 @param minsize: the minimum size we accept
1647 @return: None for success, otherwise the error message
1650 vgsize = vglist.get(vgname, None)
1652 return "volume group '%s' missing" % vgname
1653 elif vgsize < minsize:
1654 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1655 (vgname, minsize, vgsize))
1659 def SplitTime(value):
1660 """Splits time as floating point number into a tuple.
1662 @param value: Time in seconds
1663 @type value: int or float
1664 @return: Tuple containing (seconds, microseconds)
1667 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1669 assert 0 <= seconds, \
1670 "Seconds must be larger than or equal to 0, but are %s" % seconds
1671 assert 0 <= microseconds <= 999999, \
1672 "Microseconds must be 0-999999, but are %s" % microseconds
1674 return (int(seconds), int(microseconds))
1677 def MergeTime(timetuple):
1678 """Merges a tuple into time as a floating point number.
1680 @param timetuple: Time as tuple, (seconds, microseconds)
1681 @type timetuple: tuple
1682 @return: Time as a floating point number expressed in seconds
1685 (seconds, microseconds) = timetuple
1687 assert 0 <= seconds, \
1688 "Seconds must be larger than or equal to 0, but are %s" % seconds
1689 assert 0 <= microseconds <= 999999, \
1690 "Microseconds must be 0-999999, but are %s" % microseconds
1692 return float(seconds) + (float(microseconds) * 0.000001)
1695 def GetNodeDaemonPort():
1696 """Get the node daemon port for this cluster.
1698 Note that this routine does not read a ganeti-specific file, but
1699 instead uses C{socket.getservbyname} to allow pre-customization of
1700 this parameter outside of Ganeti.
1706 port = socket.getservbyname("ganeti-noded", "tcp")
1707 except socket.error:
1708 port = constants.DEFAULT_NODED_PORT
1713 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1714 multithreaded=False):
1715 """Configures the logging module.
1718 @param logfile: the filename to which we should log
1719 @type debug: boolean
1720 @param debug: whether to enable debug messages too or
1721 only those at C{INFO} and above level
1722 @type stderr_logging: boolean
1723 @param stderr_logging: whether we should also log to the standard error
1725 @param program: the name under which we should log messages
1726 @type multithreaded: boolean
1727 @param multithreaded: if True, will add the thread name to the log file
1728 @raise EnvironmentError: if we can't open the log file and
1729 stderr logging is disabled
1732 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1734 fmt += "/%(threadName)s"
1736 fmt += " %(module)s:%(lineno)s"
1737 fmt += " %(levelname)s %(message)s"
1738 formatter = logging.Formatter(fmt)
1740 root_logger = logging.getLogger("")
1741 root_logger.setLevel(logging.NOTSET)
1743 # Remove all previously setup handlers
1744 for handler in root_logger.handlers:
1746 root_logger.removeHandler(handler)
1749 stderr_handler = logging.StreamHandler()
1750 stderr_handler.setFormatter(formatter)
1752 stderr_handler.setLevel(logging.NOTSET)
1754 stderr_handler.setLevel(logging.CRITICAL)
1755 root_logger.addHandler(stderr_handler)
1757 # this can fail, if the logging directories are not setup or we have
1758 # a permisssion problem; in this case, it's best to log but ignore
1759 # the error if stderr_logging is True, and if false we re-raise the
1760 # exception since otherwise we could run but without any logs at all
1762 logfile_handler = logging.FileHandler(logfile)
1763 logfile_handler.setFormatter(formatter)
1765 logfile_handler.setLevel(logging.DEBUG)
1767 logfile_handler.setLevel(logging.INFO)
1768 root_logger.addHandler(logfile_handler)
1769 except EnvironmentError:
1771 logging.exception("Failed to enable logging to file '%s'", logfile)
1773 # we need to re-raise the exception
1777 def TailFile(fname, lines=20):
1778 """Return the last lines from a file.
1780 @note: this function will only read and parse the last 4KB of
1781 the file; if the lines are very long, it could be that less
1782 than the requested number of lines are returned
1784 @param fname: the file name
1786 @param lines: the (maximum) number of lines to return
1789 fd = open(fname, "r")
1793 pos = max(0, pos-4096)
1795 raw_data = fd.read()
1799 rows = raw_data.splitlines()
1800 return rows[-lines:]
1803 def SafeEncode(text):
1804 """Return a 'safe' version of a source string.
1806 This function mangles the input string and returns a version that
1807 should be safe to disply/encode as ASCII. To this end, we first
1808 convert it to ASCII using the 'backslashreplace' encoding which
1809 should get rid of any non-ASCII chars, and then we again encode it
1810 via 'string_escape' which converts '\n' into '\\n' so that log
1811 messages remain one-line.
1813 @type text: str or unicode
1814 @param text: input data
1816 @return: a safe version of text
1819 text = text.encode('ascii', 'backslashreplace')
1820 text = text.encode('string_escape')
1824 def LockedMethod(fn):
1825 """Synchronized object access decorator.
1827 This decorator is intended to protect access to an object using the
1828 object's own lock which is hardcoded to '_lock'.
1831 def _LockDebug(*args, **kwargs):
1833 logging.debug(*args, **kwargs)
1835 def wrapper(self, *args, **kwargs):
1836 assert hasattr(self, '_lock')
1838 _LockDebug("Waiting for %s", lock)
1841 _LockDebug("Acquired %s", lock)
1842 result = fn(self, *args, **kwargs)
1844 _LockDebug("Releasing %s", lock)
1846 _LockDebug("Released %s", lock)
1852 """Locks a file using POSIX locks.
1855 @param fd: the file descriptor we need to lock
1859 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1860 except IOError, err:
1861 if err.errno == errno.EAGAIN:
1862 raise errors.LockError("File already locked")
1866 class FileLock(object):
1867 """Utility class for file locks.
1870 def __init__(self, filename):
1871 """Constructor for FileLock.
1873 This will open the file denoted by the I{filename} argument.
1876 @param filename: path to the file to be locked
1879 self.filename = filename
1880 self.fd = open(self.filename, "w")
1886 """Close the file and release the lock.
1893 def _flock(self, flag, blocking, timeout, errmsg):
1894 """Wrapper for fcntl.flock.
1897 @param flag: operation flag
1898 @type blocking: bool
1899 @param blocking: whether the operation should be done in blocking mode.
1900 @type timeout: None or float
1901 @param timeout: for how long the operation should be retried (implies
1903 @type errmsg: string
1904 @param errmsg: error message in case operation fails.
1907 assert self.fd, "Lock was closed"
1908 assert timeout is None or timeout >= 0, \
1909 "If specified, timeout must be positive"
1911 if timeout is not None:
1912 flag |= fcntl.LOCK_NB
1913 timeout_end = time.time() + timeout
1915 # Blocking doesn't have effect with timeout
1917 flag |= fcntl.LOCK_NB
1923 fcntl.flock(self.fd, flag)
1925 except IOError, err:
1926 if err.errno in (errno.EAGAIN, ):
1927 if timeout_end is not None and time.time() < timeout_end:
1928 # Wait before trying again
1929 time.sleep(max(0.1, min(1.0, timeout)))
1931 raise errors.LockError(errmsg)
1933 logging.exception("fcntl.flock failed")
1936 def Exclusive(self, blocking=False, timeout=None):
1937 """Locks the file in exclusive mode.
1939 @type blocking: boolean
1940 @param blocking: whether to block and wait until we
1941 can lock the file or return immediately
1942 @type timeout: int or None
1943 @param timeout: if not None, the duration to wait for the lock
1947 self._flock(fcntl.LOCK_EX, blocking, timeout,
1948 "Failed to lock %s in exclusive mode" % self.filename)
1950 def Shared(self, blocking=False, timeout=None):
1951 """Locks the file in shared mode.
1953 @type blocking: boolean
1954 @param blocking: whether to block and wait until we
1955 can lock the file or return immediately
1956 @type timeout: int or None
1957 @param timeout: if not None, the duration to wait for the lock
1961 self._flock(fcntl.LOCK_SH, blocking, timeout,
1962 "Failed to lock %s in shared mode" % self.filename)
1964 def Unlock(self, blocking=True, timeout=None):
1965 """Unlocks the file.
1967 According to C{flock(2)}, unlocking can also be a nonblocking
1970 To make a non-blocking request, include LOCK_NB with any of the above
1973 @type blocking: boolean
1974 @param blocking: whether to block and wait until we
1975 can lock the file or return immediately
1976 @type timeout: int or None
1977 @param timeout: if not None, the duration to wait for the lock
1981 self._flock(fcntl.LOCK_UN, blocking, timeout,
1982 "Failed to unlock %s" % self.filename)
1985 class SignalHandler(object):
1986 """Generic signal handler class.
1988 It automatically restores the original handler when deconstructed or
1989 when L{Reset} is called. You can either pass your own handler
1990 function in or query the L{called} attribute to detect whether the
1994 @ivar signum: the signals we handle
1995 @type called: boolean
1996 @ivar called: tracks whether any of the signals have been raised
1999 def __init__(self, signum):
2000 """Constructs a new SignalHandler instance.
2002 @type signum: int or list of ints
2003 @param signum: Single signal number or set of signal numbers
2006 if isinstance(signum, (int, long)):
2007 self.signum = set([signum])
2009 self.signum = set(signum)
2015 for signum in self.signum:
2017 prev_handler = signal.signal(signum, self._HandleSignal)
2019 self._previous[signum] = prev_handler
2021 # Restore previous handler
2022 signal.signal(signum, prev_handler)
2025 # Reset all handlers
2027 # Here we have a race condition: a handler may have already been called,
2028 # but there's not much we can do about it at this point.
2035 """Restore previous handler.
2037 This will reset all the signals to their previous handlers.
2040 for signum, prev_handler in self._previous.items():
2041 signal.signal(signum, prev_handler)
2042 # If successful, remove from dict
2043 del self._previous[signum]
2046 """Unsets the L{called} flag.
2048 This function can be used in case a signal may arrive several times.
2053 def _HandleSignal(self, signum, frame):
2054 """Actual signal handling function.
2057 # This is not nice and not absolutely atomic, but it appears to be the only
2058 # solution in Python -- there are no atomic types.
2062 class FieldSet(object):
2063 """A simple field set.
2065 Among the features are:
2066 - checking if a string is among a list of static string or regex objects
2067 - checking if a whole list of string matches
2068 - returning the matching groups from a regex match
2070 Internally, all fields are held as regular expression objects.
2073 def __init__(self, *items):
2074 self.items = [re.compile("^%s$" % value) for value in items]
2076 def Extend(self, other_set):
2077 """Extend the field set with the items from another one"""
2078 self.items.extend(other_set.items)
2080 def Matches(self, field):
2081 """Checks if a field matches the current set
2084 @param field: the string to match
2085 @return: either False or a regular expression match object
2088 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2092 def NonMatching(self, items):
2093 """Returns the list of fields not matching the current set
2096 @param items: the list of fields to check
2098 @return: list of non-matching fields
2101 return [val for val in items if not self.Matches(val)]