4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
48 from cStringIO import StringIO
50 from ganeti import errors
51 from ganeti import constants
55 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60 #: when set to True, L{RunCmd} is disabled
64 class RunResult(object):
65 """Holds the result of running external programs.
68 @ivar exit_code: the exit code of the program, or None (if the program
70 @type signal: int or None
71 @ivar signal: the signal that caused the program to finish, or None
72 (if the program wasn't terminated by a signal)
74 @ivar stdout: the standard output of the program
76 @ivar stderr: the standard error of the program
78 @ivar failed: True in case the program was
79 terminated by a signal or exited with a non-zero exit code
80 @ivar fail_reason: a string detailing the termination reason
83 __slots__ = ["exit_code", "signal", "stdout", "stderr",
84 "failed", "fail_reason", "cmd"]
87 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
89 self.exit_code = exit_code
93 self.failed = (signal_ is not None or exit_code != 0)
95 if self.signal is not None:
96 self.fail_reason = "terminated by signal %s" % self.signal
97 elif self.exit_code is not None:
98 self.fail_reason = "exited with exit code %s" % self.exit_code
100 self.fail_reason = "unable to determine termination reason"
103 logging.debug("Command '%s' failed (%s); output: %s",
104 self.cmd, self.fail_reason, self.output)
106 def _GetOutput(self):
107 """Returns the combined stdout and stderr for easier usage.
110 return self.stdout + self.stderr
112 output = property(_GetOutput, None, None, "Return full output")
115 def RunCmd(cmd, env=None, output=None, cwd='/'):
116 """Execute a (shell) command.
118 The command should not read from its standard input, as it will be
121 @type cmd: string or list
122 @param cmd: Command to run
124 @param env: Additional environment
126 @param output: if desired, the output of the command can be
127 saved in a file instead of the RunResult instance; this
128 parameter denotes the file name (if not None)
130 @param cwd: if specified, will be used as the working
131 directory for the command; the default will be /
133 @return: RunResult instance
134 @raise erors.ProgrammerError: if we call this when forks are disabled
138 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
140 if isinstance(cmd, list):
141 cmd = [str(val) for val in cmd]
142 strcmd = " ".join(cmd)
147 logging.debug("RunCmd '%s'", strcmd)
149 cmd_env = os.environ.copy()
150 cmd_env["LC_ALL"] = "C"
155 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
157 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
167 return RunResult(exitcode, signal_, out, err, strcmd)
169 def _RunCmdPipe(cmd, env, via_shell, cwd):
170 """Run a command and return its output.
172 @type cmd: string or list
173 @param cmd: Command to run
175 @param env: The environment to use
176 @type via_shell: bool
177 @param via_shell: if we should run via the shell
179 @param cwd: the working directory for the program
181 @return: (out, err, status)
184 poller = select.poll()
185 child = subprocess.Popen(cmd, shell=via_shell,
186 stderr=subprocess.PIPE,
187 stdout=subprocess.PIPE,
188 stdin=subprocess.PIPE,
189 close_fds=True, env=env,
193 poller.register(child.stdout, select.POLLIN)
194 poller.register(child.stderr, select.POLLIN)
198 child.stdout.fileno(): (out, child.stdout),
199 child.stderr.fileno(): (err, child.stderr),
202 status = fcntl.fcntl(fd, fcntl.F_GETFL)
203 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
206 for fd, event in poller.poll():
207 if event & select.POLLIN or event & select.POLLPRI:
208 data = fdmap[fd][1].read()
209 # no data from read signifies EOF (the same as POLLHUP)
211 poller.unregister(fd)
214 fdmap[fd][0].write(data)
215 if (event & select.POLLNVAL or event & select.POLLHUP or
216 event & select.POLLERR):
217 poller.unregister(fd)
223 status = child.wait()
224 return out, err, status
227 def _RunCmdFile(cmd, env, via_shell, output, cwd):
228 """Run a command and save its output to a file.
230 @type cmd: string or list
231 @param cmd: Command to run
233 @param env: The environment to use
234 @type via_shell: bool
235 @param via_shell: if we should run via the shell
237 @param output: the filename in which to save the output
239 @param cwd: the working directory for the program
241 @return: the exit status
244 fh = open(output, "a")
246 child = subprocess.Popen(cmd, shell=via_shell,
247 stderr=subprocess.STDOUT,
249 stdin=subprocess.PIPE,
250 close_fds=True, env=env,
254 status = child.wait()
260 def RemoveFile(filename):
261 """Remove a file ignoring some errors.
263 Remove a file, ignoring non-existing ones or directories. Other
267 @param filename: the file to be removed
273 if err.errno not in (errno.ENOENT, errno.EISDIR):
277 def _FingerprintFile(filename):
278 """Compute the fingerprint of a file.
280 If the file does not exist, a None will be returned
284 @param filename: the filename to checksum
286 @return: the hex digest of the sha checksum of the contents
290 if not (os.path.exists(filename) and os.path.isfile(filename)):
303 return fp.hexdigest()
306 def FingerprintFiles(files):
307 """Compute fingerprints for a list of files.
310 @param files: the list of filename to fingerprint
312 @return: a dictionary filename: fingerprint, holding only
318 for filename in files:
319 cksum = _FingerprintFile(filename)
321 ret[filename] = cksum
326 def CheckDict(target, template, logname=None):
327 """Ensure a dictionary has a required set of keys.
329 For the given dictionaries I{target} and I{template}, ensure
330 I{target} has all the keys from I{template}. Missing keys are added
331 with values from template.
334 @param target: the dictionary to update
336 @param template: the dictionary holding the default values
337 @type logname: str or None
338 @param logname: if not None, causes the missing keys to be
339 logged with this name
346 target[k] = template[k]
348 if missing and logname:
349 logging.warning('%s missing keys %s', logname, ', '.join(missing))
352 def IsProcessAlive(pid):
353 """Check if a given pid exists on the system.
355 @note: zombie status is not handled, so zombie processes
356 will be returned as alive
358 @param pid: the process ID to check
360 @return: True if the process exists
367 os.stat("/proc/%d/status" % pid)
369 except EnvironmentError, err:
370 if err.errno in (errno.ENOENT, errno.ENOTDIR):
375 def ReadPidFile(pidfile):
376 """Read a pid from a file.
378 @type pidfile: string
379 @param pidfile: path to the file containing the pid
381 @return: The process id, if the file exista and contains a valid PID,
386 pf = open(pidfile, 'r')
387 except EnvironmentError, err:
388 if err.errno != errno.ENOENT:
389 logging.exception("Can't read pid file?!")
394 except ValueError, err:
395 logging.info("Can't parse pid file contents", exc_info=True)
401 def MatchNameComponent(key, name_list):
402 """Try to match a name against a list.
404 This function will try to match a name like test1 against a list
405 like C{['test1.example.com', 'test2.example.com', ...]}. Against
406 this list, I{'test1'} as well as I{'test1.example'} will match, but
407 not I{'test1.ex'}. A multiple match will be considered as no match
408 at all (e.g. I{'test1'} against C{['test1.example.com',
409 'test1.example.org']}).
412 @param key: the name to be searched
413 @type name_list: list
414 @param name_list: the list of strings against which to search the key
417 @return: None if there is no match I{or} if there are multiple matches,
418 otherwise the element from the list which matches
421 mo = re.compile("^%s(\..*)?$" % re.escape(key))
422 names_filtered = [name for name in name_list if mo.match(name) is not None]
423 if len(names_filtered) != 1:
425 return names_filtered[0]
429 """Class implementing resolver and hostname functionality
432 def __init__(self, name=None):
433 """Initialize the host name object.
435 If the name argument is not passed, it will use this system's
440 name = self.SysName()
443 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
444 self.ip = self.ipaddrs[0]
447 """Returns the hostname without domain.
450 return self.name.split('.')[0]
454 """Return the current system's name.
456 This is simply a wrapper over C{socket.gethostname()}.
459 return socket.gethostname()
462 def LookupHostname(hostname):
466 @param hostname: hostname to look up
469 @return: a tuple (name, aliases, ipaddrs) as returned by
470 C{socket.gethostbyname_ex}
471 @raise errors.ResolverError: in case of errors in resolving
475 result = socket.gethostbyname_ex(hostname)
476 except socket.gaierror, err:
477 # hostname not found in DNS
478 raise errors.ResolverError(hostname, err.args[0], err.args[1])
483 def ListVolumeGroups():
484 """List volume groups and their size
488 Dictionary with keys volume name and values
489 the size of the volume
492 command = "vgs --noheadings --units m --nosuffix -o name,size"
493 result = RunCmd(command)
498 for line in result.stdout.splitlines():
500 name, size = line.split()
501 size = int(float(size))
502 except (IndexError, ValueError), err:
503 logging.error("Invalid output from vgs (%s): %s", err, line)
511 def BridgeExists(bridge):
512 """Check whether the given bridge exists in the system
515 @param bridge: the bridge name to check
517 @return: True if it does
520 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
523 def CheckBEParams(beparams):
524 """Checks whether the user-supplied be-params are valid,
525 and converts them from string format where appropriate.
528 @param beparams: new params dict
532 for item in beparams:
533 if item not in constants.BES_PARAMETERS:
534 raise errors.OpPrereqError("Unknown backend parameter %s" % item)
535 if item in (constants.BE_MEMORY, constants.BE_VCPUS):
537 if val != constants.VALUE_DEFAULT:
540 except ValueError, err:
541 raise errors.OpPrereqError("Invalid %s size: %s" % (item, str(err)))
543 if item in (constants.BE_AUTO_BALANCE):
545 if not isinstance(val, bool):
546 if val == constants.VALUE_TRUE:
547 beparams[item] = True
548 elif val == constants.VALUE_FALSE:
549 beparams[item] = False
551 raise errors.OpPrereqError("Invalid %s value: %s" % (item, val))
554 def NiceSort(name_list):
555 """Sort a list of strings based on digit and non-digit groupings.
557 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
558 will sort the list in the logical order C{['a1', 'a2', 'a10',
561 The sort algorithm breaks each name in groups of either only-digits
562 or no-digits. Only the first eight such groups are considered, and
563 after that we just use what's left of the string.
565 @type name_list: list
566 @param name_list: the names to be sorted
568 @return: a copy of the name list sorted with our algorithm
571 _SORTER_BASE = "(\D+|\d+)"
572 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
573 _SORTER_BASE, _SORTER_BASE,
574 _SORTER_BASE, _SORTER_BASE,
575 _SORTER_BASE, _SORTER_BASE)
576 _SORTER_RE = re.compile(_SORTER_FULL)
577 _SORTER_NODIGIT = re.compile("^\D*$")
579 """Attempts to convert a variable to integer."""
580 if val is None or _SORTER_NODIGIT.match(val):
585 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
586 for name in name_list]
588 return [tup[1] for tup in to_sort]
591 def TryConvert(fn, val):
592 """Try to convert a value ignoring errors.
594 This function tries to apply function I{fn} to I{val}. If no
595 C{ValueError} or C{TypeError} exceptions are raised, it will return
596 the result, else it will return the original value. Any other
597 exceptions are propagated to the caller.
600 @param fn: function to apply to the value
601 @param val: the value to be converted
602 @return: The converted value if the conversion was successful,
603 otherwise the original value.
608 except (ValueError, TypeError), err:
614 """Verifies the syntax of an IPv4 address.
616 This function checks if the IPv4 address passes is valid or not based
617 on syntax (not IP range, class calculations, etc.).
620 @param ip: the address to be checked
621 @rtype: a regular expression match object
622 @return: a regular epression match object, or None if the
626 unit = "(0|[1-9]\d{0,2})"
627 #TODO: convert and return only boolean
628 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
631 def IsValidShellParam(word):
632 """Verifies is the given word is safe from the shell's p.o.v.
634 This means that we can pass this to a command via the shell and be
635 sure that it doesn't alter the command line and is passed as such to
638 Note that we are overly restrictive here, in order to be on the safe
642 @param word: the word to check
644 @return: True if the word is 'safe'
647 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
650 def BuildShellCmd(template, *args):
651 """Build a safe shell command line from the given arguments.
653 This function will check all arguments in the args list so that they
654 are valid shell parameters (i.e. they don't contain shell
655 metacharaters). If everything is ok, it will return the result of
659 @param template: the string holding the template for the
662 @return: the expanded command line
666 if not IsValidShellParam(word):
667 raise errors.ProgrammerError("Shell argument '%s' contains"
668 " invalid characters" % word)
669 return template % args
672 def FormatUnit(value, units):
673 """Formats an incoming number of MiB with the appropriate unit.
676 @param value: integer representing the value in MiB (1048576)
678 @param units: the type of formatting we should do:
679 - 'h' for automatic scaling
684 @return: the formatted value (with suffix)
687 if units not in ('m', 'g', 't', 'h'):
688 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
692 if units == 'm' or (units == 'h' and value < 1024):
695 return "%d%s" % (round(value, 0), suffix)
697 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
700 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
705 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
708 def ParseUnit(input_string):
709 """Tries to extract number and scale from the given string.
711 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
712 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
713 is always an int in MiB.
716 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
718 raise errors.UnitParseError("Invalid format")
720 value = float(m.groups()[0])
724 lcunit = unit.lower()
728 if lcunit in ('m', 'mb', 'mib'):
729 # Value already in MiB
732 elif lcunit in ('g', 'gb', 'gib'):
735 elif lcunit in ('t', 'tb', 'tib'):
739 raise errors.UnitParseError("Unknown unit: %s" % unit)
741 # Make sure we round up
742 if int(value) < value:
745 # Round up to the next multiple of 4
748 value += 4 - value % 4
753 def AddAuthorizedKey(file_name, key):
754 """Adds an SSH public key to an authorized_keys file.
757 @param file_name: path to authorized_keys file
759 @param key: string containing key
762 key_fields = key.split()
764 f = open(file_name, 'a+')
768 # Ignore whitespace changes
769 if line.split() == key_fields:
771 nl = line.endswith('\n')
775 f.write(key.rstrip('\r\n'))
782 def RemoveAuthorizedKey(file_name, key):
783 """Removes an SSH public key from an authorized_keys file.
786 @param file_name: path to authorized_keys file
788 @param key: string containing key
791 key_fields = key.split()
793 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
795 out = os.fdopen(fd, 'w')
797 f = open(file_name, 'r')
800 # Ignore whitespace changes while comparing lines
801 if line.split() != key_fields:
805 os.rename(tmpname, file_name)
815 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
816 """Sets the name of an IP address and hostname in /etc/hosts.
819 @param file_name: path to the file to modify (usually C{/etc/hosts})
821 @param ip: the IP address
823 @param hostname: the hostname to be added
825 @param aliases: the list of aliases to add for the hostname
828 # Ensure aliases are unique
829 aliases = UniqueSequence([hostname] + aliases)[1:]
831 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
833 out = os.fdopen(fd, 'w')
835 f = open(file_name, 'r')
839 fields = line.split()
840 if fields and not fields[0].startswith('#') and ip == fields[0]:
844 out.write("%s\t%s" % (ip, hostname))
846 out.write(" %s" % ' '.join(aliases))
851 os.rename(tmpname, file_name)
861 def AddHostToEtcHosts(hostname):
862 """Wrapper around SetEtcHostsEntry.
865 @param hostname: a hostname that will be resolved and added to
866 L{constants.ETC_HOSTS}
869 hi = HostInfo(name=hostname)
870 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
873 def RemoveEtcHostsEntry(file_name, hostname):
874 """Removes a hostname from /etc/hosts.
876 IP addresses without names are removed from the file.
879 @param file_name: path to the file to modify (usually C{/etc/hosts})
881 @param hostname: the hostname to be removed
884 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
886 out = os.fdopen(fd, 'w')
888 f = open(file_name, 'r')
891 fields = line.split()
892 if len(fields) > 1 and not fields[0].startswith('#'):
894 if hostname in names:
895 while hostname in names:
896 names.remove(hostname)
898 out.write("%s %s\n" % (fields[0], ' '.join(names)))
905 os.rename(tmpname, file_name)
915 def RemoveHostFromEtcHosts(hostname):
916 """Wrapper around RemoveEtcHostsEntry.
919 @param hostname: hostname that will be resolved and its
920 full and shot name will be removed from
921 L{constants.ETC_HOSTS}
924 hi = HostInfo(name=hostname)
925 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
926 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
929 def CreateBackup(file_name):
930 """Creates a backup of a file.
933 @param file_name: file to be backed up
935 @return: the path to the newly created backup
936 @raise errors.ProgrammerError: for invalid file names
939 if not os.path.isfile(file_name):
940 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
943 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
944 dir_name = os.path.dirname(file_name)
946 fsrc = open(file_name, 'rb')
948 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
949 fdst = os.fdopen(fd, 'wb')
951 shutil.copyfileobj(fsrc, fdst)
960 def ShellQuote(value):
961 """Quotes shell argument according to POSIX.
964 @param value: the argument to be quoted
966 @return: the quoted value
969 if _re_shell_unquoted.match(value):
972 return "'%s'" % value.replace("'", "'\\''")
975 def ShellQuoteArgs(args):
976 """Quotes a list of shell arguments.
979 @param args: list of arguments to be quoted
981 @return: the quoted arguments concatenaned with spaces
984 return ' '.join([ShellQuote(i) for i in args])
987 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
988 """Simple ping implementation using TCP connect(2).
990 Check if the given IP is reachable by doing attempting a TCP connect
994 @param target: the IP or hostname to ping
996 @param port: the port to connect to
998 @param timeout: the timeout on the connection attemp
999 @type live_port_needed: boolean
1000 @param live_port_needed: whether a closed port will cause the
1001 function to return failure, as if there was a timeout
1002 @type source: str or None
1003 @param source: if specified, will cause the connect to be made
1004 from this specific source address; failures to bind other
1005 than C{EADDRNOTAVAIL} will be ignored
1008 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1012 if source is not None:
1014 sock.bind((source, 0))
1015 except socket.error, (errcode, errstring):
1016 if errcode == errno.EADDRNOTAVAIL:
1019 sock.settimeout(timeout)
1022 sock.connect((target, port))
1025 except socket.timeout:
1027 except socket.error, (errcode, errstring):
1028 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1033 def OwnIpAddress(address):
1034 """Check if the current host has the the given IP address.
1036 Currently this is done by TCP-pinging the address from the loopback
1039 @type address: string
1040 @param address: the addres to check
1042 @return: True if we own the address
1045 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1046 source=constants.LOCALHOST_IP_ADDRESS)
1049 def ListVisibleFiles(path):
1050 """Returns a list of visible files in a directory.
1053 @param path: the directory to enumerate
1055 @return: the list of all files not starting with a dot
1058 files = [i for i in os.listdir(path) if not i.startswith(".")]
1063 def GetHomeDir(user, default=None):
1064 """Try to get the homedir of the given user.
1066 The user can be passed either as a string (denoting the name) or as
1067 an integer (denoting the user id). If the user is not found, the
1068 'default' argument is returned, which defaults to None.
1072 if isinstance(user, basestring):
1073 result = pwd.getpwnam(user)
1074 elif isinstance(user, (int, long)):
1075 result = pwd.getpwuid(user)
1077 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1081 return result.pw_dir
1085 """Returns a random UUID.
1087 @note: This is a Linux-specific method as it uses the /proc
1092 f = open("/proc/sys/kernel/random/uuid", "r")
1094 return f.read(128).rstrip("\n")
1099 def GenerateSecret():
1100 """Generates a random secret.
1102 This will generate a pseudo-random secret, and return its sha digest
1103 (so that it can be used where an ASCII string is needed).
1106 @return: a sha1 hexdigest of a block of 64 random bytes
1109 return sha.new(os.urandom(64)).hexdigest()
1112 def ReadFile(file_name, size=None):
1115 @type size: None or int
1116 @param size: Read at most size bytes
1118 @return: the (possibly partial) conent of the file
1121 f = open(file_name, "r")
1131 def WriteFile(file_name, fn=None, data=None,
1132 mode=None, uid=-1, gid=-1,
1133 atime=None, mtime=None, close=True,
1134 dry_run=False, backup=False,
1135 prewrite=None, postwrite=None):
1136 """(Over)write a file atomically.
1138 The file_name and either fn (a function taking one argument, the
1139 file descriptor, and which should write the data to it) or data (the
1140 contents of the file) must be passed. The other arguments are
1141 optional and allow setting the file mode, owner and group, and the
1142 mtime/atime of the file.
1144 If the function doesn't raise an exception, it has succeeded and the
1145 target file has the new contents. If the file has raised an
1146 exception, an existing target file should be unmodified and the
1147 temporary file should be removed.
1149 @type file_name: str
1150 @param file_name: the target filename
1152 @param fn: content writing function, called with
1153 file descriptor as parameter
1155 @param data: contents of the file
1157 @param mode: file mode
1159 @param uid: the owner of the file
1161 @param gid: the group of the file
1163 @param atime: a custom access time to be set on the file
1165 @param mtime: a custom modification time to be set on the file
1166 @type close: boolean
1167 @param close: whether to close file after writing it
1168 @type prewrite: callable
1169 @param prewrite: function to be called before writing content
1170 @type postwrite: callable
1171 @param postwrite: function to be called after writing content
1174 @return: None if the 'close' parameter evaluates to True,
1175 otherwise the file descriptor
1177 @raise errors.ProgrammerError: if an of the arguments are not valid
1180 if not os.path.isabs(file_name):
1181 raise errors.ProgrammerError("Path passed to WriteFile is not"
1182 " absolute: '%s'" % file_name)
1184 if [fn, data].count(None) != 1:
1185 raise errors.ProgrammerError("fn or data required")
1187 if [atime, mtime].count(None) == 1:
1188 raise errors.ProgrammerError("Both atime and mtime must be either"
1191 if backup and not dry_run and os.path.isfile(file_name):
1192 CreateBackup(file_name)
1194 dir_name, base_name = os.path.split(file_name)
1195 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1196 # here we need to make sure we remove the temp file, if any error
1197 # leaves it in place
1199 if uid != -1 or gid != -1:
1200 os.chown(new_name, uid, gid)
1202 os.chmod(new_name, mode)
1203 if callable(prewrite):
1205 if data is not None:
1209 if callable(postwrite):
1212 if atime is not None and mtime is not None:
1213 os.utime(new_name, (atime, mtime))
1215 os.rename(new_name, file_name)
1222 RemoveFile(new_name)
1227 def FirstFree(seq, base=0):
1228 """Returns the first non-existing integer from seq.
1230 The seq argument should be a sorted list of positive integers. The
1231 first time the index of an element is smaller than the element
1232 value, the index will be returned.
1234 The base argument is used to start at a different offset,
1235 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1237 Example: C{[0, 1, 3]} will return I{2}.
1240 @param seq: the sequence to be analyzed.
1242 @param base: use this value as the base index of the sequence
1244 @return: the first non-used index in the sequence
1247 for idx, elem in enumerate(seq):
1248 assert elem >= base, "Passed element is higher than base offset"
1249 if elem > idx + base:
1255 def all(seq, pred=bool):
1256 "Returns True if pred(x) is True for every element in the iterable"
1257 for elem in itertools.ifilterfalse(pred, seq):
1262 def any(seq, pred=bool):
1263 "Returns True if pred(x) is True for at least one element in the iterable"
1264 for elem in itertools.ifilter(pred, seq):
1269 def UniqueSequence(seq):
1270 """Returns a list with unique elements.
1272 Element order is preserved.
1275 @param seq: the sequence with the source elementes
1277 @return: list of unique elements from seq
1281 return [i for i in seq if i not in seen and not seen.add(i)]
1284 def IsValidMac(mac):
1285 """Predicate to check if a MAC address is valid.
1287 Checks wether the supplied MAC address is formally correct, only
1288 accepts colon separated format.
1291 @param mac: the MAC to be validated
1293 @return: True is the MAC seems valid
1296 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1297 return mac_check.match(mac) is not None
1300 def TestDelay(duration):
1301 """Sleep for a fixed amount of time.
1303 @type duration: float
1304 @param duration: the sleep duration
1306 @return: False for negative value, True otherwise
1311 time.sleep(duration)
1315 def Daemonize(logfile, noclose_fds=None):
1316 """Daemonize the current process.
1318 This detaches the current process from the controlling terminal and
1319 runs it in the background as a daemon.
1322 @param logfile: the logfile to which we should redirect stdout/stderr
1323 @type noclose_fds: list or None
1324 @param noclose_fds: if given, it denotes a list of file descriptor
1325 that should not be closed
1327 @returns: the value zero
1332 # Default maximum for the number of available file descriptors.
1333 if 'SC_OPEN_MAX' in os.sysconf_names:
1335 MAXFD = os.sysconf('SC_OPEN_MAX')
1345 if (pid == 0): # The first child.
1348 pid = os.fork() # Fork a second child.
1349 if (pid == 0): # The second child.
1353 # exit() or _exit()? See below.
1354 os._exit(0) # Exit parent (the first child) of the second child.
1356 os._exit(0) # Exit parent of the first child.
1357 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1358 if (maxfd == resource.RLIM_INFINITY):
1361 # Iterate through and close all file descriptors.
1362 for fd in range(0, maxfd):
1363 if noclose_fds and fd in noclose_fds:
1367 except OSError: # ERROR, fd wasn't open to begin with (ignored)
1369 os.open(logfile, os.O_RDWR|os.O_CREAT|os.O_APPEND, 0600)
1370 # Duplicate standard input to standard output and standard error.
1371 os.dup2(0, 1) # standard output (1)
1372 os.dup2(0, 2) # standard error (2)
1376 def DaemonPidFileName(name):
1377 """Compute a ganeti pid file absolute path
1380 @param name: the daemon name
1382 @return: the full path to the pidfile corresponding to the given
1386 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1389 def WritePidFile(name):
1390 """Write the current process pidfile.
1392 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1395 @param name: the daemon name to use
1396 @raise errors.GenericError: if the pid file already exists and
1397 points to a live process
1401 pidfilename = DaemonPidFileName(name)
1402 if IsProcessAlive(ReadPidFile(pidfilename)):
1403 raise errors.GenericError("%s contains a live process" % pidfilename)
1405 WriteFile(pidfilename, data="%d\n" % pid)
1408 def RemovePidFile(name):
1409 """Remove the current process pidfile.
1411 Any errors are ignored.
1414 @param name: the daemon name used to derive the pidfile name
1418 pidfilename = DaemonPidFileName(name)
1419 # TODO: we could check here that the file contains our pid
1421 RemoveFile(pidfilename)
1426 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1428 """Kill a process given by its pid.
1431 @param pid: The PID to terminate.
1433 @param signal_: The signal to send, by default SIGTERM
1435 @param timeout: The timeout after which, if the process is still alive,
1436 a SIGKILL will be sent. If not positive, no such checking
1438 @type waitpid: boolean
1439 @param waitpid: If true, we should waitpid on this process after
1440 sending signals, since it's our own child and otherwise it
1441 would remain as zombie
1444 def _helper(pid, signal_, wait):
1445 """Simple helper to encapsulate the kill/waitpid sequence"""
1446 os.kill(pid, signal_)
1449 os.waitpid(pid, os.WNOHANG)
1454 # kill with pid=0 == suicide
1455 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1457 if not IsProcessAlive(pid):
1459 _helper(pid, signal_, waitpid)
1462 end = time.time() + timeout
1463 while time.time() < end and IsProcessAlive(pid):
1465 if IsProcessAlive(pid):
1466 _helper(pid, signal.SIGKILL, waitpid)
1469 def FindFile(name, search_path, test=os.path.exists):
1470 """Look for a filesystem object in a given path.
1472 This is an abstract method to search for filesystem object (files,
1473 dirs) under a given search path.
1476 @param name: the name to look for
1477 @type search_path: str
1478 @param search_path: location to start at
1479 @type test: callable
1480 @param test: a function taking one argument that should return True
1481 if the a given object is valid; the default value is
1482 os.path.exists, causing only existing files to be returned
1484 @return: full path to the object if found, None otherwise
1487 for dir_name in search_path:
1488 item_name = os.path.sep.join([dir_name, name])
1494 def CheckVolumeGroupSize(vglist, vgname, minsize):
1495 """Checks if the volume group list is valid.
1497 The function will check if a given volume group is in the list of
1498 volume groups and has a minimum size.
1501 @param vglist: dictionary of volume group names and their size
1503 @param vgname: the volume group we should check
1505 @param minsize: the minimum size we accept
1507 @return: None for success, otherwise the error message
1510 vgsize = vglist.get(vgname, None)
1512 return "volume group '%s' missing" % vgname
1513 elif vgsize < minsize:
1514 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1515 (vgname, minsize, vgsize))
1519 def SplitTime(value):
1520 """Splits time as floating point number into a tuple.
1522 @param value: Time in seconds
1523 @type value: int or float
1524 @return: Tuple containing (seconds, microseconds)
1527 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1529 assert 0 <= seconds, \
1530 "Seconds must be larger than or equal to 0, but are %s" % seconds
1531 assert 0 <= microseconds <= 999999, \
1532 "Microseconds must be 0-999999, but are %s" % microseconds
1534 return (int(seconds), int(microseconds))
1537 def MergeTime(timetuple):
1538 """Merges a tuple into time as a floating point number.
1540 @param timetuple: Time as tuple, (seconds, microseconds)
1541 @type timetuple: tuple
1542 @return: Time as a floating point number expressed in seconds
1545 (seconds, microseconds) = timetuple
1547 assert 0 <= seconds, \
1548 "Seconds must be larger than or equal to 0, but are %s" % seconds
1549 assert 0 <= microseconds <= 999999, \
1550 "Microseconds must be 0-999999, but are %s" % microseconds
1552 return float(seconds) + (float(microseconds) * 0.000001)
1555 def GetNodeDaemonPort():
1556 """Get the node daemon port for this cluster.
1558 Note that this routine does not read a ganeti-specific file, but
1559 instead uses C{socket.getservbyname} to allow pre-customization of
1560 this parameter outside of Ganeti.
1566 port = socket.getservbyname("ganeti-noded", "tcp")
1567 except socket.error:
1568 port = constants.DEFAULT_NODED_PORT
1573 def SetupLogging(logfile, debug=False, stderr_logging=False, program=""):
1574 """Configures the logging module.
1577 @param logfile: the filename to which we should log
1578 @type debug: boolean
1579 @param debug: whether to enable debug messages too or
1580 only those at C{INFO} and above level
1581 @type stderr_logging: boolean
1582 @param stderr_logging: whether we should also log to the standard error
1584 @param program: the name under which we should log messages
1585 @raise EnvironmentError: if we can't open the log file and
1586 stderr logging is disabled
1589 fmt = "%(asctime)s: " + program + " "
1591 fmt += ("pid=%(process)d/%(threadName)s %(levelname)s"
1592 " %(module)s:%(lineno)s %(message)s")
1594 fmt += "pid=%(process)d %(levelname)s %(message)s"
1595 formatter = logging.Formatter(fmt)
1597 root_logger = logging.getLogger("")
1598 root_logger.setLevel(logging.NOTSET)
1600 # Remove all previously setup handlers
1601 for handler in root_logger.handlers:
1602 root_logger.removeHandler(handler)
1605 stderr_handler = logging.StreamHandler()
1606 stderr_handler.setFormatter(formatter)
1608 stderr_handler.setLevel(logging.NOTSET)
1610 stderr_handler.setLevel(logging.CRITICAL)
1611 root_logger.addHandler(stderr_handler)
1613 # this can fail, if the logging directories are not setup or we have
1614 # a permisssion problem; in this case, it's best to log but ignore
1615 # the error if stderr_logging is True, and if false we re-raise the
1616 # exception since otherwise we could run but without any logs at all
1618 logfile_handler = logging.FileHandler(logfile)
1619 logfile_handler.setFormatter(formatter)
1621 logfile_handler.setLevel(logging.DEBUG)
1623 logfile_handler.setLevel(logging.INFO)
1624 root_logger.addHandler(logfile_handler)
1625 except EnvironmentError, err:
1627 logging.exception("Failed to enable logging to file '%s'", logfile)
1629 # we need to re-raise the exception
1633 def LockedMethod(fn):
1634 """Synchronized object access decorator.
1636 This decorator is intended to protect access to an object using the
1637 object's own lock which is hardcoded to '_lock'.
1640 def _LockDebug(*args, **kwargs):
1642 logging.debug(*args, **kwargs)
1644 def wrapper(self, *args, **kwargs):
1645 assert hasattr(self, '_lock')
1647 _LockDebug("Waiting for %s", lock)
1650 _LockDebug("Acquired %s", lock)
1651 result = fn(self, *args, **kwargs)
1653 _LockDebug("Releasing %s", lock)
1655 _LockDebug("Released %s", lock)
1661 """Locks a file using POSIX locks.
1664 @param fd: the file descriptor we need to lock
1668 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1669 except IOError, err:
1670 if err.errno == errno.EAGAIN:
1671 raise errors.LockError("File already locked")
1675 class FileLock(object):
1676 """Utility class for file locks.
1679 def __init__(self, filename):
1680 """Constructor for FileLock.
1682 This will open the file denoted by the I{filename} argument.
1685 @param filename: path to the file to be locked
1688 self.filename = filename
1689 self.fd = open(self.filename, "w")
1695 """Close the file and release the lock.
1702 def _flock(self, flag, blocking, timeout, errmsg):
1703 """Wrapper for fcntl.flock.
1706 @param flag: operation flag
1707 @type blocking: bool
1708 @param blocking: whether the operation should be done in blocking mode.
1709 @type timeout: None or float
1710 @param timeout: for how long the operation should be retried (implies
1712 @type errmsg: string
1713 @param errmsg: error message in case operation fails.
1716 assert self.fd, "Lock was closed"
1717 assert timeout is None or timeout >= 0, \
1718 "If specified, timeout must be positive"
1720 if timeout is not None:
1721 flag |= fcntl.LOCK_NB
1722 timeout_end = time.time() + timeout
1724 # Blocking doesn't have effect with timeout
1726 flag |= fcntl.LOCK_NB
1732 fcntl.flock(self.fd, flag)
1734 except IOError, err:
1735 if err.errno in (errno.EAGAIN, ):
1736 if timeout_end is not None and time.time() < timeout_end:
1737 # Wait before trying again
1738 time.sleep(max(0.1, min(1.0, timeout)))
1740 raise errors.LockError(errmsg)
1742 logging.exception("fcntl.flock failed")
1745 def Exclusive(self, blocking=False, timeout=None):
1746 """Locks the file in exclusive mode.
1748 @type blocking: boolean
1749 @param blocking: whether to block and wait until we
1750 can lock the file or return immediately
1751 @type timeout: int or None
1752 @param timeout: if not None, the duration to wait for the lock
1756 self._flock(fcntl.LOCK_EX, blocking, timeout,
1757 "Failed to lock %s in exclusive mode" % self.filename)
1759 def Shared(self, blocking=False, timeout=None):
1760 """Locks the file in shared mode.
1762 @type blocking: boolean
1763 @param blocking: whether to block and wait until we
1764 can lock the file or return immediately
1765 @type timeout: int or None
1766 @param timeout: if not None, the duration to wait for the lock
1770 self._flock(fcntl.LOCK_SH, blocking, timeout,
1771 "Failed to lock %s in shared mode" % self.filename)
1773 def Unlock(self, blocking=True, timeout=None):
1774 """Unlocks the file.
1776 According to C{flock(2)}, unlocking can also be a nonblocking
1779 To make a non-blocking request, include LOCK_NB with any of the above
1782 @type blocking: boolean
1783 @param blocking: whether to block and wait until we
1784 can lock the file or return immediately
1785 @type timeout: int or None
1786 @param timeout: if not None, the duration to wait for the lock
1790 self._flock(fcntl.LOCK_UN, blocking, timeout,
1791 "Failed to unlock %s" % self.filename)
1794 class SignalHandler(object):
1795 """Generic signal handler class.
1797 It automatically restores the original handler when deconstructed or
1798 when L{Reset} is called. You can either pass your own handler
1799 function in or query the L{called} attribute to detect whether the
1803 @ivar signum: the signals we handle
1804 @type called: boolean
1805 @ivar called: tracks whether any of the signals have been raised
1808 def __init__(self, signum):
1809 """Constructs a new SignalHandler instance.
1811 @type signum: int or list of ints
1812 @param signum: Single signal number or set of signal numbers
1815 if isinstance(signum, (int, long)):
1816 self.signum = set([signum])
1818 self.signum = set(signum)
1824 for signum in self.signum:
1826 prev_handler = signal.signal(signum, self._HandleSignal)
1828 self._previous[signum] = prev_handler
1830 # Restore previous handler
1831 signal.signal(signum, prev_handler)
1834 # Reset all handlers
1836 # Here we have a race condition: a handler may have already been called,
1837 # but there's not much we can do about it at this point.
1844 """Restore previous handler.
1846 This will reset all the signals to their previous handlers.
1849 for signum, prev_handler in self._previous.items():
1850 signal.signal(signum, prev_handler)
1851 # If successful, remove from dict
1852 del self._previous[signum]
1855 """Unsets the L{called} flag.
1857 This function can be used in case a signal may arrive several times.
1862 def _HandleSignal(self, signum, frame):
1863 """Actual signal handling function.
1866 # This is not nice and not absolutely atomic, but it appears to be the only
1867 # solution in Python -- there are no atomic types.
1871 class FieldSet(object):
1872 """A simple field set.
1874 Among the features are:
1875 - checking if a string is among a list of static string or regex objects
1876 - checking if a whole list of string matches
1877 - returning the matching groups from a regex match
1879 Internally, all fields are held as regular expression objects.
1882 def __init__(self, *items):
1883 self.items = [re.compile("^%s$" % value) for value in items]
1885 def Extend(self, other_set):
1886 """Extend the field set with the items from another one"""
1887 self.items.extend(other_set.items)
1889 def Matches(self, field):
1890 """Checks if a field matches the current set
1893 @param field: the string to match
1894 @return: either False or a regular expression match object
1897 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
1901 def NonMatching(self, items):
1902 """Returns the list of fields not matching the current set
1905 @param items: the list of fields to check
1907 @return: list of non-matching fields
1910 return [val for val in items if not self.Matches(val)]