4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
47 from cStringIO import StringIO
50 from hashlib import sha1
55 from ganeti import errors
56 from ganeti import constants
60 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
65 #: when set to True, L{RunCmd} is disabled
69 class RunResult(object):
70 """Holds the result of running external programs.
73 @ivar exit_code: the exit code of the program, or None (if the program
75 @type signal: int or None
76 @ivar signal: the signal that caused the program to finish, or None
77 (if the program wasn't terminated by a signal)
79 @ivar stdout: the standard output of the program
81 @ivar stderr: the standard error of the program
83 @ivar failed: True in case the program was
84 terminated by a signal or exited with a non-zero exit code
85 @ivar fail_reason: a string detailing the termination reason
88 __slots__ = ["exit_code", "signal", "stdout", "stderr",
89 "failed", "fail_reason", "cmd"]
92 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
94 self.exit_code = exit_code
98 self.failed = (signal_ is not None or exit_code != 0)
100 if self.signal is not None:
101 self.fail_reason = "terminated by signal %s" % self.signal
102 elif self.exit_code is not None:
103 self.fail_reason = "exited with exit code %s" % self.exit_code
105 self.fail_reason = "unable to determine termination reason"
108 logging.debug("Command '%s' failed (%s); output: %s",
109 self.cmd, self.fail_reason, self.output)
111 def _GetOutput(self):
112 """Returns the combined stdout and stderr for easier usage.
115 return self.stdout + self.stderr
117 output = property(_GetOutput, None, None, "Return full output")
120 def RunCmd(cmd, env=None, output=None, cwd='/'):
121 """Execute a (shell) command.
123 The command should not read from its standard input, as it will be
126 @type cmd: string or list
127 @param cmd: Command to run
129 @param env: Additional environment
131 @param output: if desired, the output of the command can be
132 saved in a file instead of the RunResult instance; this
133 parameter denotes the file name (if not None)
135 @param cwd: if specified, will be used as the working
136 directory for the command; the default will be /
138 @return: RunResult instance
139 @raise erors.ProgrammerError: if we call this when forks are disabled
143 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
145 if isinstance(cmd, list):
146 cmd = [str(val) for val in cmd]
147 strcmd = " ".join(cmd)
152 logging.debug("RunCmd '%s'", strcmd)
154 cmd_env = os.environ.copy()
155 cmd_env["LC_ALL"] = "C"
161 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
163 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
166 if err.errno == errno.ENOENT:
167 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
179 return RunResult(exitcode, signal_, out, err, strcmd)
182 def _RunCmdPipe(cmd, env, via_shell, cwd):
183 """Run a command and return its output.
185 @type cmd: string or list
186 @param cmd: Command to run
188 @param env: The environment to use
189 @type via_shell: bool
190 @param via_shell: if we should run via the shell
192 @param cwd: the working directory for the program
194 @return: (out, err, status)
197 poller = select.poll()
198 child = subprocess.Popen(cmd, shell=via_shell,
199 stderr=subprocess.PIPE,
200 stdout=subprocess.PIPE,
201 stdin=subprocess.PIPE,
202 close_fds=True, env=env,
206 poller.register(child.stdout, select.POLLIN)
207 poller.register(child.stderr, select.POLLIN)
211 child.stdout.fileno(): (out, child.stdout),
212 child.stderr.fileno(): (err, child.stderr),
215 status = fcntl.fcntl(fd, fcntl.F_GETFL)
216 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
220 pollresult = poller.poll()
221 except EnvironmentError, eerr:
222 if eerr.errno == errno.EINTR:
225 except select.error, serr:
226 if serr[0] == errno.EINTR:
230 for fd, event in pollresult:
231 if event & select.POLLIN or event & select.POLLPRI:
232 data = fdmap[fd][1].read()
233 # no data from read signifies EOF (the same as POLLHUP)
235 poller.unregister(fd)
238 fdmap[fd][0].write(data)
239 if (event & select.POLLNVAL or event & select.POLLHUP or
240 event & select.POLLERR):
241 poller.unregister(fd)
247 status = child.wait()
248 return out, err, status
251 def _RunCmdFile(cmd, env, via_shell, output, cwd):
252 """Run a command and save its output to a file.
254 @type cmd: string or list
255 @param cmd: Command to run
257 @param env: The environment to use
258 @type via_shell: bool
259 @param via_shell: if we should run via the shell
261 @param output: the filename in which to save the output
263 @param cwd: the working directory for the program
265 @return: the exit status
268 fh = open(output, "a")
270 child = subprocess.Popen(cmd, shell=via_shell,
271 stderr=subprocess.STDOUT,
273 stdin=subprocess.PIPE,
274 close_fds=True, env=env,
278 status = child.wait()
284 def RemoveFile(filename):
285 """Remove a file ignoring some errors.
287 Remove a file, ignoring non-existing ones or directories. Other
291 @param filename: the file to be removed
297 if err.errno not in (errno.ENOENT, errno.EISDIR):
301 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
305 @param old: Original path
309 @param mkdir: Whether to create target directory if it doesn't exist
310 @type mkdir_mode: int
311 @param mkdir_mode: Mode for newly created directories
315 return os.rename(old, new)
317 # In at least one use case of this function, the job queue, directory
318 # creation is very rare. Checking for the directory before renaming is not
320 if mkdir and err.errno == errno.ENOENT:
321 # Create directory and try again
322 os.makedirs(os.path.dirname(new), mkdir_mode)
323 return os.rename(old, new)
327 def _FingerprintFile(filename):
328 """Compute the fingerprint of a file.
330 If the file does not exist, a None will be returned
334 @param filename: the filename to checksum
336 @return: the hex digest of the sha checksum of the contents
340 if not (os.path.exists(filename) and os.path.isfile(filename)):
353 return fp.hexdigest()
356 def FingerprintFiles(files):
357 """Compute fingerprints for a list of files.
360 @param files: the list of filename to fingerprint
362 @return: a dictionary filename: fingerprint, holding only
368 for filename in files:
369 cksum = _FingerprintFile(filename)
371 ret[filename] = cksum
376 def ForceDictType(target, key_types, allowed_values=None):
377 """Force the values of a dict to have certain types.
380 @param target: the dict to update
381 @type key_types: dict
382 @param key_types: dict mapping target dict keys to types
383 in constants.ENFORCEABLE_TYPES
384 @type allowed_values: list
385 @keyword allowed_values: list of specially allowed values
388 if allowed_values is None:
392 if key not in key_types:
393 msg = "Unknown key '%s'" % key
394 raise errors.TypeEnforcementError(msg)
396 if target[key] in allowed_values:
399 type = key_types[key]
400 if type not in constants.ENFORCEABLE_TYPES:
401 msg = "'%s' has non-enforceable type %s" % (key, type)
402 raise errors.ProgrammerError(msg)
404 if type == constants.VTYPE_STRING:
405 if not isinstance(target[key], basestring):
406 if isinstance(target[key], bool) and not target[key]:
409 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
410 raise errors.TypeEnforcementError(msg)
411 elif type == constants.VTYPE_BOOL:
412 if isinstance(target[key], basestring) and target[key]:
413 if target[key].lower() == constants.VALUE_FALSE:
415 elif target[key].lower() == constants.VALUE_TRUE:
418 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
419 raise errors.TypeEnforcementError(msg)
424 elif type == constants.VTYPE_SIZE:
426 target[key] = ParseUnit(target[key])
427 except errors.UnitParseError, err:
428 msg = "'%s' (value %s) is not a valid size. error: %s" % \
429 (key, target[key], err)
430 raise errors.TypeEnforcementError(msg)
431 elif type == constants.VTYPE_INT:
433 target[key] = int(target[key])
434 except (ValueError, TypeError):
435 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
436 raise errors.TypeEnforcementError(msg)
439 def IsProcessAlive(pid):
440 """Check if a given pid exists on the system.
442 @note: zombie status is not handled, so zombie processes
443 will be returned as alive
445 @param pid: the process ID to check
447 @return: True if the process exists
454 os.stat("/proc/%d/status" % pid)
456 except EnvironmentError, err:
457 if err.errno in (errno.ENOENT, errno.ENOTDIR):
462 def ReadPidFile(pidfile):
463 """Read a pid from a file.
465 @type pidfile: string
466 @param pidfile: path to the file containing the pid
468 @return: The process id, if the file exists and contains a valid PID,
473 pf = open(pidfile, 'r')
474 except EnvironmentError, err:
475 if err.errno != errno.ENOENT:
476 logging.exception("Can't read pid file?!")
481 except ValueError, err:
482 logging.info("Can't parse pid file contents", exc_info=True)
488 def MatchNameComponent(key, name_list):
489 """Try to match a name against a list.
491 This function will try to match a name like test1 against a list
492 like C{['test1.example.com', 'test2.example.com', ...]}. Against
493 this list, I{'test1'} as well as I{'test1.example'} will match, but
494 not I{'test1.ex'}. A multiple match will be considered as no match
495 at all (e.g. I{'test1'} against C{['test1.example.com',
496 'test1.example.org']}).
499 @param key: the name to be searched
500 @type name_list: list
501 @param name_list: the list of strings against which to search the key
504 @return: None if there is no match I{or} if there are multiple matches,
505 otherwise the element from the list which matches
508 mo = re.compile("^%s(\..*)?$" % re.escape(key))
509 names_filtered = [name for name in name_list if mo.match(name) is not None]
510 if len(names_filtered) != 1:
512 return names_filtered[0]
516 """Class implementing resolver and hostname functionality
519 def __init__(self, name=None):
520 """Initialize the host name object.
522 If the name argument is not passed, it will use this system's
527 name = self.SysName()
530 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
531 self.ip = self.ipaddrs[0]
534 """Returns the hostname without domain.
537 return self.name.split('.')[0]
541 """Return the current system's name.
543 This is simply a wrapper over C{socket.gethostname()}.
546 return socket.gethostname()
549 def LookupHostname(hostname):
553 @param hostname: hostname to look up
556 @return: a tuple (name, aliases, ipaddrs) as returned by
557 C{socket.gethostbyname_ex}
558 @raise errors.ResolverError: in case of errors in resolving
562 result = socket.gethostbyname_ex(hostname)
563 except socket.gaierror, err:
564 # hostname not found in DNS
565 raise errors.ResolverError(hostname, err.args[0], err.args[1])
570 def ListVolumeGroups():
571 """List volume groups and their size
575 Dictionary with keys volume name and values
576 the size of the volume
579 command = "vgs --noheadings --units m --nosuffix -o name,size"
580 result = RunCmd(command)
585 for line in result.stdout.splitlines():
587 name, size = line.split()
588 size = int(float(size))
589 except (IndexError, ValueError), err:
590 logging.error("Invalid output from vgs (%s): %s", err, line)
598 def BridgeExists(bridge):
599 """Check whether the given bridge exists in the system
602 @param bridge: the bridge name to check
604 @return: True if it does
607 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
610 def NiceSort(name_list):
611 """Sort a list of strings based on digit and non-digit groupings.
613 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
614 will sort the list in the logical order C{['a1', 'a2', 'a10',
617 The sort algorithm breaks each name in groups of either only-digits
618 or no-digits. Only the first eight such groups are considered, and
619 after that we just use what's left of the string.
621 @type name_list: list
622 @param name_list: the names to be sorted
624 @return: a copy of the name list sorted with our algorithm
627 _SORTER_BASE = "(\D+|\d+)"
628 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
629 _SORTER_BASE, _SORTER_BASE,
630 _SORTER_BASE, _SORTER_BASE,
631 _SORTER_BASE, _SORTER_BASE)
632 _SORTER_RE = re.compile(_SORTER_FULL)
633 _SORTER_NODIGIT = re.compile("^\D*$")
635 """Attempts to convert a variable to integer."""
636 if val is None or _SORTER_NODIGIT.match(val):
641 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
642 for name in name_list]
644 return [tup[1] for tup in to_sort]
647 def TryConvert(fn, val):
648 """Try to convert a value ignoring errors.
650 This function tries to apply function I{fn} to I{val}. If no
651 C{ValueError} or C{TypeError} exceptions are raised, it will return
652 the result, else it will return the original value. Any other
653 exceptions are propagated to the caller.
656 @param fn: function to apply to the value
657 @param val: the value to be converted
658 @return: The converted value if the conversion was successful,
659 otherwise the original value.
664 except (ValueError, TypeError), err:
670 """Verifies the syntax of an IPv4 address.
672 This function checks if the IPv4 address passes is valid or not based
673 on syntax (not IP range, class calculations, etc.).
676 @param ip: the address to be checked
677 @rtype: a regular expression match object
678 @return: a regular epression match object, or None if the
682 unit = "(0|[1-9]\d{0,2})"
683 #TODO: convert and return only boolean
684 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
687 def IsValidShellParam(word):
688 """Verifies is the given word is safe from the shell's p.o.v.
690 This means that we can pass this to a command via the shell and be
691 sure that it doesn't alter the command line and is passed as such to
694 Note that we are overly restrictive here, in order to be on the safe
698 @param word: the word to check
700 @return: True if the word is 'safe'
703 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
706 def BuildShellCmd(template, *args):
707 """Build a safe shell command line from the given arguments.
709 This function will check all arguments in the args list so that they
710 are valid shell parameters (i.e. they don't contain shell
711 metacharaters). If everything is ok, it will return the result of
715 @param template: the string holding the template for the
718 @return: the expanded command line
722 if not IsValidShellParam(word):
723 raise errors.ProgrammerError("Shell argument '%s' contains"
724 " invalid characters" % word)
725 return template % args
728 def FormatUnit(value, units):
729 """Formats an incoming number of MiB with the appropriate unit.
732 @param value: integer representing the value in MiB (1048576)
734 @param units: the type of formatting we should do:
735 - 'h' for automatic scaling
740 @return: the formatted value (with suffix)
743 if units not in ('m', 'g', 't', 'h'):
744 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
748 if units == 'm' or (units == 'h' and value < 1024):
751 return "%d%s" % (round(value, 0), suffix)
753 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
756 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
761 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
764 def ParseUnit(input_string):
765 """Tries to extract number and scale from the given string.
767 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
768 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
769 is always an int in MiB.
772 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
774 raise errors.UnitParseError("Invalid format")
776 value = float(m.groups()[0])
780 lcunit = unit.lower()
784 if lcunit in ('m', 'mb', 'mib'):
785 # Value already in MiB
788 elif lcunit in ('g', 'gb', 'gib'):
791 elif lcunit in ('t', 'tb', 'tib'):
795 raise errors.UnitParseError("Unknown unit: %s" % unit)
797 # Make sure we round up
798 if int(value) < value:
801 # Round up to the next multiple of 4
804 value += 4 - value % 4
809 def AddAuthorizedKey(file_name, key):
810 """Adds an SSH public key to an authorized_keys file.
813 @param file_name: path to authorized_keys file
815 @param key: string containing key
818 key_fields = key.split()
820 f = open(file_name, 'a+')
824 # Ignore whitespace changes
825 if line.split() == key_fields:
827 nl = line.endswith('\n')
831 f.write(key.rstrip('\r\n'))
838 def RemoveAuthorizedKey(file_name, key):
839 """Removes an SSH public key from an authorized_keys file.
842 @param file_name: path to authorized_keys file
844 @param key: string containing key
847 key_fields = key.split()
849 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
851 out = os.fdopen(fd, 'w')
853 f = open(file_name, 'r')
856 # Ignore whitespace changes while comparing lines
857 if line.split() != key_fields:
861 os.rename(tmpname, file_name)
871 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
872 """Sets the name of an IP address and hostname in /etc/hosts.
875 @param file_name: path to the file to modify (usually C{/etc/hosts})
877 @param ip: the IP address
879 @param hostname: the hostname to be added
881 @param aliases: the list of aliases to add for the hostname
884 # FIXME: use WriteFile + fn rather than duplicating its efforts
885 # Ensure aliases are unique
886 aliases = UniqueSequence([hostname] + aliases)[1:]
888 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
890 out = os.fdopen(fd, 'w')
892 f = open(file_name, 'r')
895 fields = line.split()
896 if fields and not fields[0].startswith('#') and ip == fields[0]:
900 out.write("%s\t%s" % (ip, hostname))
902 out.write(" %s" % ' '.join(aliases))
907 os.chmod(tmpname, 0644)
908 os.rename(tmpname, file_name)
918 def AddHostToEtcHosts(hostname):
919 """Wrapper around SetEtcHostsEntry.
922 @param hostname: a hostname that will be resolved and added to
923 L{constants.ETC_HOSTS}
926 hi = HostInfo(name=hostname)
927 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
930 def RemoveEtcHostsEntry(file_name, hostname):
931 """Removes a hostname from /etc/hosts.
933 IP addresses without names are removed from the file.
936 @param file_name: path to the file to modify (usually C{/etc/hosts})
938 @param hostname: the hostname to be removed
941 # FIXME: use WriteFile + fn rather than duplicating its efforts
942 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
944 out = os.fdopen(fd, 'w')
946 f = open(file_name, 'r')
949 fields = line.split()
950 if len(fields) > 1 and not fields[0].startswith('#'):
952 if hostname in names:
953 while hostname in names:
954 names.remove(hostname)
956 out.write("%s %s\n" % (fields[0], ' '.join(names)))
963 os.chmod(tmpname, 0644)
964 os.rename(tmpname, file_name)
974 def RemoveHostFromEtcHosts(hostname):
975 """Wrapper around RemoveEtcHostsEntry.
978 @param hostname: hostname that will be resolved and its
979 full and shot name will be removed from
980 L{constants.ETC_HOSTS}
983 hi = HostInfo(name=hostname)
984 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
985 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
988 def CreateBackup(file_name):
989 """Creates a backup of a file.
992 @param file_name: file to be backed up
994 @return: the path to the newly created backup
995 @raise errors.ProgrammerError: for invalid file names
998 if not os.path.isfile(file_name):
999 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1002 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1003 dir_name = os.path.dirname(file_name)
1005 fsrc = open(file_name, 'rb')
1007 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1008 fdst = os.fdopen(fd, 'wb')
1010 shutil.copyfileobj(fsrc, fdst)
1019 def ShellQuote(value):
1020 """Quotes shell argument according to POSIX.
1023 @param value: the argument to be quoted
1025 @return: the quoted value
1028 if _re_shell_unquoted.match(value):
1031 return "'%s'" % value.replace("'", "'\\''")
1034 def ShellQuoteArgs(args):
1035 """Quotes a list of shell arguments.
1038 @param args: list of arguments to be quoted
1040 @return: the quoted arguments concatenaned with spaces
1043 return ' '.join([ShellQuote(i) for i in args])
1046 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1047 """Simple ping implementation using TCP connect(2).
1049 Check if the given IP is reachable by doing attempting a TCP connect
1053 @param target: the IP or hostname to ping
1055 @param port: the port to connect to
1057 @param timeout: the timeout on the connection attemp
1058 @type live_port_needed: boolean
1059 @param live_port_needed: whether a closed port will cause the
1060 function to return failure, as if there was a timeout
1061 @type source: str or None
1062 @param source: if specified, will cause the connect to be made
1063 from this specific source address; failures to bind other
1064 than C{EADDRNOTAVAIL} will be ignored
1067 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1071 if source is not None:
1073 sock.bind((source, 0))
1074 except socket.error, (errcode, errstring):
1075 if errcode == errno.EADDRNOTAVAIL:
1078 sock.settimeout(timeout)
1081 sock.connect((target, port))
1084 except socket.timeout:
1086 except socket.error, (errcode, errstring):
1087 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1092 def OwnIpAddress(address):
1093 """Check if the current host has the the given IP address.
1095 Currently this is done by TCP-pinging the address from the loopback
1098 @type address: string
1099 @param address: the addres to check
1101 @return: True if we own the address
1104 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1105 source=constants.LOCALHOST_IP_ADDRESS)
1108 def ListVisibleFiles(path):
1109 """Returns a list of visible files in a directory.
1112 @param path: the directory to enumerate
1114 @return: the list of all files not starting with a dot
1117 files = [i for i in os.listdir(path) if not i.startswith(".")]
1122 def GetHomeDir(user, default=None):
1123 """Try to get the homedir of the given user.
1125 The user can be passed either as a string (denoting the name) or as
1126 an integer (denoting the user id). If the user is not found, the
1127 'default' argument is returned, which defaults to None.
1131 if isinstance(user, basestring):
1132 result = pwd.getpwnam(user)
1133 elif isinstance(user, (int, long)):
1134 result = pwd.getpwuid(user)
1136 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1140 return result.pw_dir
1144 """Returns a random UUID.
1146 @note: This is a Linux-specific method as it uses the /proc
1151 f = open("/proc/sys/kernel/random/uuid", "r")
1153 return f.read(128).rstrip("\n")
1158 def GenerateSecret():
1159 """Generates a random secret.
1161 This will generate a pseudo-random secret, and return its sha digest
1162 (so that it can be used where an ASCII string is needed).
1165 @return: a sha1 hexdigest of a block of 64 random bytes
1168 return sha1(os.urandom(64)).hexdigest()
1171 def EnsureDirs(dirs):
1172 """Make required directories, if they don't exist.
1174 @param dirs: list of tuples (dir_name, dir_mode)
1175 @type dirs: list of (string, integer)
1178 for dir_name, dir_mode in dirs:
1180 os.mkdir(dir_name, dir_mode)
1181 except EnvironmentError, err:
1182 if err.errno != errno.EEXIST:
1183 raise errors.GenericError("Cannot create needed directory"
1184 " '%s': %s" % (dir_name, err))
1185 if not os.path.isdir(dir_name):
1186 raise errors.GenericError("%s is not a directory" % dir_name)
1189 def ReadFile(file_name, size=None):
1192 @type size: None or int
1193 @param size: Read at most size bytes
1195 @return: the (possibly partial) conent of the file
1198 f = open(file_name, "r")
1208 def WriteFile(file_name, fn=None, data=None,
1209 mode=None, uid=-1, gid=-1,
1210 atime=None, mtime=None, close=True,
1211 dry_run=False, backup=False,
1212 prewrite=None, postwrite=None):
1213 """(Over)write a file atomically.
1215 The file_name and either fn (a function taking one argument, the
1216 file descriptor, and which should write the data to it) or data (the
1217 contents of the file) must be passed. The other arguments are
1218 optional and allow setting the file mode, owner and group, and the
1219 mtime/atime of the file.
1221 If the function doesn't raise an exception, it has succeeded and the
1222 target file has the new contents. If the function has raised an
1223 exception, an existing target file should be unmodified and the
1224 temporary file should be removed.
1226 @type file_name: str
1227 @param file_name: the target filename
1229 @param fn: content writing function, called with
1230 file descriptor as parameter
1232 @param data: contents of the file
1234 @param mode: file mode
1236 @param uid: the owner of the file
1238 @param gid: the group of the file
1240 @param atime: a custom access time to be set on the file
1242 @param mtime: a custom modification time to be set on the file
1243 @type close: boolean
1244 @param close: whether to close file after writing it
1245 @type prewrite: callable
1246 @param prewrite: function to be called before writing content
1247 @type postwrite: callable
1248 @param postwrite: function to be called after writing content
1251 @return: None if the 'close' parameter evaluates to True,
1252 otherwise the file descriptor
1254 @raise errors.ProgrammerError: if any of the arguments are not valid
1257 if not os.path.isabs(file_name):
1258 raise errors.ProgrammerError("Path passed to WriteFile is not"
1259 " absolute: '%s'" % file_name)
1261 if [fn, data].count(None) != 1:
1262 raise errors.ProgrammerError("fn or data required")
1264 if [atime, mtime].count(None) == 1:
1265 raise errors.ProgrammerError("Both atime and mtime must be either"
1268 if backup and not dry_run and os.path.isfile(file_name):
1269 CreateBackup(file_name)
1271 dir_name, base_name = os.path.split(file_name)
1272 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1274 # here we need to make sure we remove the temp file, if any error
1275 # leaves it in place
1277 if uid != -1 or gid != -1:
1278 os.chown(new_name, uid, gid)
1280 os.chmod(new_name, mode)
1281 if callable(prewrite):
1283 if data is not None:
1287 if callable(postwrite):
1290 if atime is not None and mtime is not None:
1291 os.utime(new_name, (atime, mtime))
1293 os.rename(new_name, file_name)
1302 RemoveFile(new_name)
1307 def FirstFree(seq, base=0):
1308 """Returns the first non-existing integer from seq.
1310 The seq argument should be a sorted list of positive integers. The
1311 first time the index of an element is smaller than the element
1312 value, the index will be returned.
1314 The base argument is used to start at a different offset,
1315 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1317 Example: C{[0, 1, 3]} will return I{2}.
1320 @param seq: the sequence to be analyzed.
1322 @param base: use this value as the base index of the sequence
1324 @return: the first non-used index in the sequence
1327 for idx, elem in enumerate(seq):
1328 assert elem >= base, "Passed element is higher than base offset"
1329 if elem > idx + base:
1335 def all(seq, pred=bool):
1336 "Returns True if pred(x) is True for every element in the iterable"
1337 for elem in itertools.ifilterfalse(pred, seq):
1342 def any(seq, pred=bool):
1343 "Returns True if pred(x) is True for at least one element in the iterable"
1344 for elem in itertools.ifilter(pred, seq):
1349 def UniqueSequence(seq):
1350 """Returns a list with unique elements.
1352 Element order is preserved.
1355 @param seq: the sequence with the source elementes
1357 @return: list of unique elements from seq
1361 return [i for i in seq if i not in seen and not seen.add(i)]
1364 def IsValidMac(mac):
1365 """Predicate to check if a MAC address is valid.
1367 Checks wether the supplied MAC address is formally correct, only
1368 accepts colon separated format.
1371 @param mac: the MAC to be validated
1373 @return: True is the MAC seems valid
1376 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1377 return mac_check.match(mac) is not None
1380 def TestDelay(duration):
1381 """Sleep for a fixed amount of time.
1383 @type duration: float
1384 @param duration: the sleep duration
1386 @return: False for negative value, True otherwise
1391 time.sleep(duration)
1395 def _CloseFDNoErr(fd, retries=5):
1396 """Close a file descriptor ignoring errors.
1399 @param fd: the file descriptor
1401 @param retries: how many retries to make, in case we get any
1402 other error than EBADF
1407 except OSError, err:
1408 if err.errno != errno.EBADF:
1410 _CloseFDNoErr(fd, retries - 1)
1411 # else either it's closed already or we're out of retries, so we
1412 # ignore this and go on
1415 def CloseFDs(noclose_fds=None):
1416 """Close file descriptors.
1418 This closes all file descriptors above 2 (i.e. except
1421 @type noclose_fds: list or None
1422 @param noclose_fds: if given, it denotes a list of file descriptor
1423 that should not be closed
1426 # Default maximum for the number of available file descriptors.
1427 if 'SC_OPEN_MAX' in os.sysconf_names:
1429 MAXFD = os.sysconf('SC_OPEN_MAX')
1436 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1437 if (maxfd == resource.RLIM_INFINITY):
1440 # Iterate through and close all file descriptors (except the standard ones)
1441 for fd in range(3, maxfd):
1442 if noclose_fds and fd in noclose_fds:
1447 def Daemonize(logfile):
1448 """Daemonize the current process.
1450 This detaches the current process from the controlling terminal and
1451 runs it in the background as a daemon.
1454 @param logfile: the logfile to which we should redirect stdout/stderr
1456 @return: the value zero
1464 if (pid == 0): # The first child.
1467 pid = os.fork() # Fork a second child.
1468 if (pid == 0): # The second child.
1472 # exit() or _exit()? See below.
1473 os._exit(0) # Exit parent (the first child) of the second child.
1475 os._exit(0) # Exit parent of the first child.
1479 i = os.open("/dev/null", os.O_RDONLY) # stdin
1480 assert i == 0, "Can't close/reopen stdin"
1481 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1482 assert i == 1, "Can't close/reopen stdout"
1483 # Duplicate standard output to standard error.
1488 def DaemonPidFileName(name):
1489 """Compute a ganeti pid file absolute path
1492 @param name: the daemon name
1494 @return: the full path to the pidfile corresponding to the given
1498 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1501 def WritePidFile(name):
1502 """Write the current process pidfile.
1504 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1507 @param name: the daemon name to use
1508 @raise errors.GenericError: if the pid file already exists and
1509 points to a live process
1513 pidfilename = DaemonPidFileName(name)
1514 if IsProcessAlive(ReadPidFile(pidfilename)):
1515 raise errors.GenericError("%s contains a live process" % pidfilename)
1517 WriteFile(pidfilename, data="%d\n" % pid)
1520 def RemovePidFile(name):
1521 """Remove the current process pidfile.
1523 Any errors are ignored.
1526 @param name: the daemon name used to derive the pidfile name
1530 pidfilename = DaemonPidFileName(name)
1531 # TODO: we could check here that the file contains our pid
1533 RemoveFile(pidfilename)
1538 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1540 """Kill a process given by its pid.
1543 @param pid: The PID to terminate.
1545 @param signal_: The signal to send, by default SIGTERM
1547 @param timeout: The timeout after which, if the process is still alive,
1548 a SIGKILL will be sent. If not positive, no such checking
1550 @type waitpid: boolean
1551 @param waitpid: If true, we should waitpid on this process after
1552 sending signals, since it's our own child and otherwise it
1553 would remain as zombie
1556 def _helper(pid, signal_, wait):
1557 """Simple helper to encapsulate the kill/waitpid sequence"""
1558 os.kill(pid, signal_)
1561 os.waitpid(pid, os.WNOHANG)
1566 # kill with pid=0 == suicide
1567 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1569 if not IsProcessAlive(pid):
1571 _helper(pid, signal_, waitpid)
1575 # Wait up to $timeout seconds
1576 end = time.time() + timeout
1578 while time.time() < end and IsProcessAlive(pid):
1580 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1586 # Make wait time longer for next try
1590 if IsProcessAlive(pid):
1591 # Kill process if it's still alive
1592 _helper(pid, signal.SIGKILL, waitpid)
1595 def FindFile(name, search_path, test=os.path.exists):
1596 """Look for a filesystem object in a given path.
1598 This is an abstract method to search for filesystem object (files,
1599 dirs) under a given search path.
1602 @param name: the name to look for
1603 @type search_path: str
1604 @param search_path: location to start at
1605 @type test: callable
1606 @param test: a function taking one argument that should return True
1607 if the a given object is valid; the default value is
1608 os.path.exists, causing only existing files to be returned
1610 @return: full path to the object if found, None otherwise
1613 for dir_name in search_path:
1614 item_name = os.path.sep.join([dir_name, name])
1620 def CheckVolumeGroupSize(vglist, vgname, minsize):
1621 """Checks if the volume group list is valid.
1623 The function will check if a given volume group is in the list of
1624 volume groups and has a minimum size.
1627 @param vglist: dictionary of volume group names and their size
1629 @param vgname: the volume group we should check
1631 @param minsize: the minimum size we accept
1633 @return: None for success, otherwise the error message
1636 vgsize = vglist.get(vgname, None)
1638 return "volume group '%s' missing" % vgname
1639 elif vgsize < minsize:
1640 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1641 (vgname, minsize, vgsize))
1645 def SplitTime(value):
1646 """Splits time as floating point number into a tuple.
1648 @param value: Time in seconds
1649 @type value: int or float
1650 @return: Tuple containing (seconds, microseconds)
1653 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1655 assert 0 <= seconds, \
1656 "Seconds must be larger than or equal to 0, but are %s" % seconds
1657 assert 0 <= microseconds <= 999999, \
1658 "Microseconds must be 0-999999, but are %s" % microseconds
1660 return (int(seconds), int(microseconds))
1663 def MergeTime(timetuple):
1664 """Merges a tuple into time as a floating point number.
1666 @param timetuple: Time as tuple, (seconds, microseconds)
1667 @type timetuple: tuple
1668 @return: Time as a floating point number expressed in seconds
1671 (seconds, microseconds) = timetuple
1673 assert 0 <= seconds, \
1674 "Seconds must be larger than or equal to 0, but are %s" % seconds
1675 assert 0 <= microseconds <= 999999, \
1676 "Microseconds must be 0-999999, but are %s" % microseconds
1678 return float(seconds) + (float(microseconds) * 0.000001)
1681 def GetNodeDaemonPort():
1682 """Get the node daemon port for this cluster.
1684 Note that this routine does not read a ganeti-specific file, but
1685 instead uses C{socket.getservbyname} to allow pre-customization of
1686 this parameter outside of Ganeti.
1692 port = socket.getservbyname("ganeti-noded", "tcp")
1693 except socket.error:
1694 port = constants.DEFAULT_NODED_PORT
1699 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1700 multithreaded=False):
1701 """Configures the logging module.
1704 @param logfile: the filename to which we should log
1705 @type debug: boolean
1706 @param debug: whether to enable debug messages too or
1707 only those at C{INFO} and above level
1708 @type stderr_logging: boolean
1709 @param stderr_logging: whether we should also log to the standard error
1711 @param program: the name under which we should log messages
1712 @type multithreaded: boolean
1713 @param multithreaded: if True, will add the thread name to the log file
1714 @raise EnvironmentError: if we can't open the log file and
1715 stderr logging is disabled
1718 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1720 fmt += "/%(threadName)s"
1722 fmt += " %(module)s:%(lineno)s"
1723 fmt += " %(levelname)s %(message)s"
1724 formatter = logging.Formatter(fmt)
1726 root_logger = logging.getLogger("")
1727 root_logger.setLevel(logging.NOTSET)
1729 # Remove all previously setup handlers
1730 for handler in root_logger.handlers:
1732 root_logger.removeHandler(handler)
1735 stderr_handler = logging.StreamHandler()
1736 stderr_handler.setFormatter(formatter)
1738 stderr_handler.setLevel(logging.NOTSET)
1740 stderr_handler.setLevel(logging.CRITICAL)
1741 root_logger.addHandler(stderr_handler)
1743 # this can fail, if the logging directories are not setup or we have
1744 # a permisssion problem; in this case, it's best to log but ignore
1745 # the error if stderr_logging is True, and if false we re-raise the
1746 # exception since otherwise we could run but without any logs at all
1748 logfile_handler = logging.FileHandler(logfile)
1749 logfile_handler.setFormatter(formatter)
1751 logfile_handler.setLevel(logging.DEBUG)
1753 logfile_handler.setLevel(logging.INFO)
1754 root_logger.addHandler(logfile_handler)
1755 except EnvironmentError:
1757 logging.exception("Failed to enable logging to file '%s'", logfile)
1759 # we need to re-raise the exception
1762 def IsNormAbsPath(path):
1763 """Check whether a path is absolute and also normalized
1765 This avoids things like /dir/../../other/path to be valid.
1768 return os.path.normpath(path) == path and os.path.isabs(path)
1770 def TailFile(fname, lines=20):
1771 """Return the last lines from a file.
1773 @note: this function will only read and parse the last 4KB of
1774 the file; if the lines are very long, it could be that less
1775 than the requested number of lines are returned
1777 @param fname: the file name
1779 @param lines: the (maximum) number of lines to return
1782 fd = open(fname, "r")
1786 pos = max(0, pos-4096)
1788 raw_data = fd.read()
1792 rows = raw_data.splitlines()
1793 return rows[-lines:]
1796 def SafeEncode(text):
1797 """Return a 'safe' version of a source string.
1799 This function mangles the input string and returns a version that
1800 should be safe to disply/encode as ASCII. To this end, we first
1801 convert it to ASCII using the 'backslashreplace' encoding which
1802 should get rid of any non-ASCII chars, and then we again encode it
1803 via 'string_escape' which converts '\n' into '\\n' so that log
1804 messages remain one-line.
1806 @type text: str or unicode
1807 @param text: input data
1809 @return: a safe version of text
1812 text = text.encode('ascii', 'backslashreplace')
1813 text = text.encode('string_escape')
1817 def CommaJoin(names):
1818 """Nicely join a set of identifiers.
1820 @param names: set, list or tuple
1821 @return: a string with the formatted results
1824 return ", ".join(["'%s'" % val for val in names])
1827 def LockedMethod(fn):
1828 """Synchronized object access decorator.
1830 This decorator is intended to protect access to an object using the
1831 object's own lock which is hardcoded to '_lock'.
1834 def _LockDebug(*args, **kwargs):
1836 logging.debug(*args, **kwargs)
1838 def wrapper(self, *args, **kwargs):
1839 assert hasattr(self, '_lock')
1841 _LockDebug("Waiting for %s", lock)
1844 _LockDebug("Acquired %s", lock)
1845 result = fn(self, *args, **kwargs)
1847 _LockDebug("Releasing %s", lock)
1849 _LockDebug("Released %s", lock)
1855 """Locks a file using POSIX locks.
1858 @param fd: the file descriptor we need to lock
1862 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1863 except IOError, err:
1864 if err.errno == errno.EAGAIN:
1865 raise errors.LockError("File already locked")
1869 class FileLock(object):
1870 """Utility class for file locks.
1873 def __init__(self, filename):
1874 """Constructor for FileLock.
1876 This will open the file denoted by the I{filename} argument.
1879 @param filename: path to the file to be locked
1882 self.filename = filename
1883 self.fd = open(self.filename, "w")
1889 """Close the file and release the lock.
1896 def _flock(self, flag, blocking, timeout, errmsg):
1897 """Wrapper for fcntl.flock.
1900 @param flag: operation flag
1901 @type blocking: bool
1902 @param blocking: whether the operation should be done in blocking mode.
1903 @type timeout: None or float
1904 @param timeout: for how long the operation should be retried (implies
1906 @type errmsg: string
1907 @param errmsg: error message in case operation fails.
1910 assert self.fd, "Lock was closed"
1911 assert timeout is None or timeout >= 0, \
1912 "If specified, timeout must be positive"
1914 if timeout is not None:
1915 flag |= fcntl.LOCK_NB
1916 timeout_end = time.time() + timeout
1918 # Blocking doesn't have effect with timeout
1920 flag |= fcntl.LOCK_NB
1926 fcntl.flock(self.fd, flag)
1928 except IOError, err:
1929 if err.errno in (errno.EAGAIN, ):
1930 if timeout_end is not None and time.time() < timeout_end:
1931 # Wait before trying again
1932 time.sleep(max(0.1, min(1.0, timeout)))
1934 raise errors.LockError(errmsg)
1936 logging.exception("fcntl.flock failed")
1939 def Exclusive(self, blocking=False, timeout=None):
1940 """Locks the file in exclusive mode.
1942 @type blocking: boolean
1943 @param blocking: whether to block and wait until we
1944 can lock the file or return immediately
1945 @type timeout: int or None
1946 @param timeout: if not None, the duration to wait for the lock
1950 self._flock(fcntl.LOCK_EX, blocking, timeout,
1951 "Failed to lock %s in exclusive mode" % self.filename)
1953 def Shared(self, blocking=False, timeout=None):
1954 """Locks the file in shared mode.
1956 @type blocking: boolean
1957 @param blocking: whether to block and wait until we
1958 can lock the file or return immediately
1959 @type timeout: int or None
1960 @param timeout: if not None, the duration to wait for the lock
1964 self._flock(fcntl.LOCK_SH, blocking, timeout,
1965 "Failed to lock %s in shared mode" % self.filename)
1967 def Unlock(self, blocking=True, timeout=None):
1968 """Unlocks the file.
1970 According to C{flock(2)}, unlocking can also be a nonblocking
1973 To make a non-blocking request, include LOCK_NB with any of the above
1976 @type blocking: boolean
1977 @param blocking: whether to block and wait until we
1978 can lock the file or return immediately
1979 @type timeout: int or None
1980 @param timeout: if not None, the duration to wait for the lock
1984 self._flock(fcntl.LOCK_UN, blocking, timeout,
1985 "Failed to unlock %s" % self.filename)
1988 class SignalHandler(object):
1989 """Generic signal handler class.
1991 It automatically restores the original handler when deconstructed or
1992 when L{Reset} is called. You can either pass your own handler
1993 function in or query the L{called} attribute to detect whether the
1997 @ivar signum: the signals we handle
1998 @type called: boolean
1999 @ivar called: tracks whether any of the signals have been raised
2002 def __init__(self, signum):
2003 """Constructs a new SignalHandler instance.
2005 @type signum: int or list of ints
2006 @param signum: Single signal number or set of signal numbers
2009 if isinstance(signum, (int, long)):
2010 self.signum = set([signum])
2012 self.signum = set(signum)
2018 for signum in self.signum:
2020 prev_handler = signal.signal(signum, self._HandleSignal)
2022 self._previous[signum] = prev_handler
2024 # Restore previous handler
2025 signal.signal(signum, prev_handler)
2028 # Reset all handlers
2030 # Here we have a race condition: a handler may have already been called,
2031 # but there's not much we can do about it at this point.
2038 """Restore previous handler.
2040 This will reset all the signals to their previous handlers.
2043 for signum, prev_handler in self._previous.items():
2044 signal.signal(signum, prev_handler)
2045 # If successful, remove from dict
2046 del self._previous[signum]
2049 """Unsets the L{called} flag.
2051 This function can be used in case a signal may arrive several times.
2056 def _HandleSignal(self, signum, frame):
2057 """Actual signal handling function.
2060 # This is not nice and not absolutely atomic, but it appears to be the only
2061 # solution in Python -- there are no atomic types.
2065 class FieldSet(object):
2066 """A simple field set.
2068 Among the features are:
2069 - checking if a string is among a list of static string or regex objects
2070 - checking if a whole list of string matches
2071 - returning the matching groups from a regex match
2073 Internally, all fields are held as regular expression objects.
2076 def __init__(self, *items):
2077 self.items = [re.compile("^%s$" % value) for value in items]
2079 def Extend(self, other_set):
2080 """Extend the field set with the items from another one"""
2081 self.items.extend(other_set.items)
2083 def Matches(self, field):
2084 """Checks if a field matches the current set
2087 @param field: the string to match
2088 @return: either False or a regular expression match object
2091 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2095 def NonMatching(self, items):
2096 """Returns the list of fields not matching the current set
2099 @param items: the list of fields to check
2101 @return: list of non-matching fields
2104 return [val for val in items if not self.Matches(val)]