4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
48 from cStringIO import StringIO
50 from ganeti import errors
51 from ganeti import constants
55 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60 #: when set to True, L{RunCmd} is disabled
64 class RunResult(object):
65 """Holds the result of running external programs.
68 @ivar exit_code: the exit code of the program, or None (if the program
70 @type signal: int or None
71 @ivar signal: the signal that caused the program to finish, or None
72 (if the program wasn't terminated by a signal)
74 @ivar stdout: the standard output of the program
76 @ivar stderr: the standard error of the program
78 @ivar failed: True in case the program was
79 terminated by a signal or exited with a non-zero exit code
80 @ivar fail_reason: a string detailing the termination reason
83 __slots__ = ["exit_code", "signal", "stdout", "stderr",
84 "failed", "fail_reason", "cmd"]
87 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
89 self.exit_code = exit_code
93 self.failed = (signal_ is not None or exit_code != 0)
95 if self.signal is not None:
96 self.fail_reason = "terminated by signal %s" % self.signal
97 elif self.exit_code is not None:
98 self.fail_reason = "exited with exit code %s" % self.exit_code
100 self.fail_reason = "unable to determine termination reason"
103 logging.debug("Command '%s' failed (%s); output: %s",
104 self.cmd, self.fail_reason, self.output)
106 def _GetOutput(self):
107 """Returns the combined stdout and stderr for easier usage.
110 return self.stdout + self.stderr
112 output = property(_GetOutput, None, None, "Return full output")
115 def RunCmd(cmd, env=None, output=None, cwd='/'):
116 """Execute a (shell) command.
118 The command should not read from its standard input, as it will be
121 @type cmd: string or list
122 @param cmd: Command to run
124 @param env: Additional environment
126 @param output: if desired, the output of the command can be
127 saved in a file instead of the RunResult instance; this
128 parameter denotes the file name (if not None)
130 @param cwd: if specified, will be used as the working
131 directory for the command; the default will be /
133 @return: RunResult instance
134 @raise erors.ProgrammerError: if we call this when forks are disabled
138 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
140 if isinstance(cmd, list):
141 cmd = [str(val) for val in cmd]
142 strcmd = " ".join(cmd)
147 logging.debug("RunCmd '%s'", strcmd)
149 cmd_env = os.environ.copy()
150 cmd_env["LC_ALL"] = "C"
155 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
157 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
167 return RunResult(exitcode, signal_, out, err, strcmd)
170 def _RunCmdPipe(cmd, env, via_shell, cwd):
171 """Run a command and return its output.
173 @type cmd: string or list
174 @param cmd: Command to run
176 @param env: The environment to use
177 @type via_shell: bool
178 @param via_shell: if we should run via the shell
180 @param cwd: the working directory for the program
182 @return: (out, err, status)
185 poller = select.poll()
186 child = subprocess.Popen(cmd, shell=via_shell,
187 stderr=subprocess.PIPE,
188 stdout=subprocess.PIPE,
189 stdin=subprocess.PIPE,
190 close_fds=True, env=env,
194 poller.register(child.stdout, select.POLLIN)
195 poller.register(child.stderr, select.POLLIN)
199 child.stdout.fileno(): (out, child.stdout),
200 child.stderr.fileno(): (err, child.stderr),
203 status = fcntl.fcntl(fd, fcntl.F_GETFL)
204 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
208 pollresult = poller.poll()
209 except EnvironmentError, eerr:
210 if eerr.errno == errno.EINTR:
213 except select.error, serr:
214 if serr[0] == errno.EINTR:
218 for fd, event in pollresult:
219 if event & select.POLLIN or event & select.POLLPRI:
220 data = fdmap[fd][1].read()
221 # no data from read signifies EOF (the same as POLLHUP)
223 poller.unregister(fd)
226 fdmap[fd][0].write(data)
227 if (event & select.POLLNVAL or event & select.POLLHUP or
228 event & select.POLLERR):
229 poller.unregister(fd)
235 status = child.wait()
236 return out, err, status
239 def _RunCmdFile(cmd, env, via_shell, output, cwd):
240 """Run a command and save its output to a file.
242 @type cmd: string or list
243 @param cmd: Command to run
245 @param env: The environment to use
246 @type via_shell: bool
247 @param via_shell: if we should run via the shell
249 @param output: the filename in which to save the output
251 @param cwd: the working directory for the program
253 @return: the exit status
256 fh = open(output, "a")
258 child = subprocess.Popen(cmd, shell=via_shell,
259 stderr=subprocess.STDOUT,
261 stdin=subprocess.PIPE,
262 close_fds=True, env=env,
266 status = child.wait()
272 def RemoveFile(filename):
273 """Remove a file ignoring some errors.
275 Remove a file, ignoring non-existing ones or directories. Other
279 @param filename: the file to be removed
285 if err.errno not in (errno.ENOENT, errno.EISDIR):
289 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
293 @param old: Original path
297 @param mkdir: Whether to create target directory if it doesn't exist
298 @type mkdir_mode: int
299 @param mkdir_mode: Mode for newly created directories
303 return os.rename(old, new)
305 # In at least one use case of this function, the job queue, directory
306 # creation is very rare. Checking for the directory before renaming is not
308 if mkdir and err.errno == errno.ENOENT:
309 # Create directory and try again
310 os.makedirs(os.path.dirname(new), mkdir_mode)
311 return os.rename(old, new)
315 def _FingerprintFile(filename):
316 """Compute the fingerprint of a file.
318 If the file does not exist, a None will be returned
322 @param filename: the filename to checksum
324 @return: the hex digest of the sha checksum of the contents
328 if not (os.path.exists(filename) and os.path.isfile(filename)):
341 return fp.hexdigest()
344 def FingerprintFiles(files):
345 """Compute fingerprints for a list of files.
348 @param files: the list of filename to fingerprint
350 @return: a dictionary filename: fingerprint, holding only
356 for filename in files:
357 cksum = _FingerprintFile(filename)
359 ret[filename] = cksum
364 def CheckDict(target, template, logname=None):
365 """Ensure a dictionary has a required set of keys.
367 For the given dictionaries I{target} and I{template}, ensure
368 I{target} has all the keys from I{template}. Missing keys are added
369 with values from template.
372 @param target: the dictionary to update
374 @param template: the dictionary holding the default values
375 @type logname: str or None
376 @param logname: if not None, causes the missing keys to be
377 logged with this name
384 target[k] = template[k]
386 if missing and logname:
387 logging.warning('%s missing keys %s', logname, ', '.join(missing))
390 def IsProcessAlive(pid):
391 """Check if a given pid exists on the system.
393 @note: zombie status is not handled, so zombie processes
394 will be returned as alive
396 @param pid: the process ID to check
398 @return: True if the process exists
405 os.stat("/proc/%d/status" % pid)
407 except EnvironmentError, err:
408 if err.errno in (errno.ENOENT, errno.ENOTDIR):
413 def ReadPidFile(pidfile):
414 """Read a pid from a file.
416 @type pidfile: string
417 @param pidfile: path to the file containing the pid
419 @return: The process id, if the file exista and contains a valid PID,
424 pf = open(pidfile, 'r')
425 except EnvironmentError, err:
426 if err.errno != errno.ENOENT:
427 logging.exception("Can't read pid file?!")
432 except ValueError, err:
433 logging.info("Can't parse pid file contents", exc_info=True)
439 def MatchNameComponent(key, name_list):
440 """Try to match a name against a list.
442 This function will try to match a name like test1 against a list
443 like C{['test1.example.com', 'test2.example.com', ...]}. Against
444 this list, I{'test1'} as well as I{'test1.example'} will match, but
445 not I{'test1.ex'}. A multiple match will be considered as no match
446 at all (e.g. I{'test1'} against C{['test1.example.com',
447 'test1.example.org']}).
450 @param key: the name to be searched
451 @type name_list: list
452 @param name_list: the list of strings against which to search the key
455 @return: None if there is no match I{or} if there are multiple matches,
456 otherwise the element from the list which matches
459 mo = re.compile("^%s(\..*)?$" % re.escape(key))
460 names_filtered = [name for name in name_list if mo.match(name) is not None]
461 if len(names_filtered) != 1:
463 return names_filtered[0]
467 """Class implementing resolver and hostname functionality
470 def __init__(self, name=None):
471 """Initialize the host name object.
473 If the name argument is not passed, it will use this system's
478 name = self.SysName()
481 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
482 self.ip = self.ipaddrs[0]
485 """Returns the hostname without domain.
488 return self.name.split('.')[0]
492 """Return the current system's name.
494 This is simply a wrapper over C{socket.gethostname()}.
497 return socket.gethostname()
500 def LookupHostname(hostname):
504 @param hostname: hostname to look up
507 @return: a tuple (name, aliases, ipaddrs) as returned by
508 C{socket.gethostbyname_ex}
509 @raise errors.ResolverError: in case of errors in resolving
513 result = socket.gethostbyname_ex(hostname)
514 except socket.gaierror, err:
515 # hostname not found in DNS
516 raise errors.ResolverError(hostname, err.args[0], err.args[1])
521 def ListVolumeGroups():
522 """List volume groups and their size
526 Dictionary with keys volume name and values
527 the size of the volume
530 command = "vgs --noheadings --units m --nosuffix -o name,size"
531 result = RunCmd(command)
536 for line in result.stdout.splitlines():
538 name, size = line.split()
539 size = int(float(size))
540 except (IndexError, ValueError), err:
541 logging.error("Invalid output from vgs (%s): %s", err, line)
549 def BridgeExists(bridge):
550 """Check whether the given bridge exists in the system
553 @param bridge: the bridge name to check
555 @return: True if it does
558 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
561 def CheckBEParams(beparams):
562 """Checks whether the user-supplied be-params are valid,
563 and converts them from string format where appropriate.
566 @param beparams: new params dict
570 for item in beparams:
571 if item not in constants.BES_PARAMETERS:
572 raise errors.OpPrereqError("Unknown backend parameter %s" % item)
573 if item in (constants.BE_MEMORY, constants.BE_VCPUS):
575 if val != constants.VALUE_DEFAULT:
578 except ValueError, err:
579 raise errors.OpPrereqError("Invalid %s size: %s" % (item, str(err)))
581 if item in (constants.BE_AUTO_BALANCE):
583 if not isinstance(val, bool):
584 if val == constants.VALUE_TRUE:
585 beparams[item] = True
586 elif val == constants.VALUE_FALSE:
587 beparams[item] = False
589 raise errors.OpPrereqError("Invalid %s value: %s" % (item, val))
592 def NiceSort(name_list):
593 """Sort a list of strings based on digit and non-digit groupings.
595 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
596 will sort the list in the logical order C{['a1', 'a2', 'a10',
599 The sort algorithm breaks each name in groups of either only-digits
600 or no-digits. Only the first eight such groups are considered, and
601 after that we just use what's left of the string.
603 @type name_list: list
604 @param name_list: the names to be sorted
606 @return: a copy of the name list sorted with our algorithm
609 _SORTER_BASE = "(\D+|\d+)"
610 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
611 _SORTER_BASE, _SORTER_BASE,
612 _SORTER_BASE, _SORTER_BASE,
613 _SORTER_BASE, _SORTER_BASE)
614 _SORTER_RE = re.compile(_SORTER_FULL)
615 _SORTER_NODIGIT = re.compile("^\D*$")
617 """Attempts to convert a variable to integer."""
618 if val is None or _SORTER_NODIGIT.match(val):
623 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
624 for name in name_list]
626 return [tup[1] for tup in to_sort]
629 def TryConvert(fn, val):
630 """Try to convert a value ignoring errors.
632 This function tries to apply function I{fn} to I{val}. If no
633 C{ValueError} or C{TypeError} exceptions are raised, it will return
634 the result, else it will return the original value. Any other
635 exceptions are propagated to the caller.
638 @param fn: function to apply to the value
639 @param val: the value to be converted
640 @return: The converted value if the conversion was successful,
641 otherwise the original value.
646 except (ValueError, TypeError), err:
652 """Verifies the syntax of an IPv4 address.
654 This function checks if the IPv4 address passes is valid or not based
655 on syntax (not IP range, class calculations, etc.).
658 @param ip: the address to be checked
659 @rtype: a regular expression match object
660 @return: a regular epression match object, or None if the
664 unit = "(0|[1-9]\d{0,2})"
665 #TODO: convert and return only boolean
666 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
669 def IsValidShellParam(word):
670 """Verifies is the given word is safe from the shell's p.o.v.
672 This means that we can pass this to a command via the shell and be
673 sure that it doesn't alter the command line and is passed as such to
676 Note that we are overly restrictive here, in order to be on the safe
680 @param word: the word to check
682 @return: True if the word is 'safe'
685 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
688 def BuildShellCmd(template, *args):
689 """Build a safe shell command line from the given arguments.
691 This function will check all arguments in the args list so that they
692 are valid shell parameters (i.e. they don't contain shell
693 metacharaters). If everything is ok, it will return the result of
697 @param template: the string holding the template for the
700 @return: the expanded command line
704 if not IsValidShellParam(word):
705 raise errors.ProgrammerError("Shell argument '%s' contains"
706 " invalid characters" % word)
707 return template % args
710 def FormatUnit(value, units):
711 """Formats an incoming number of MiB with the appropriate unit.
714 @param value: integer representing the value in MiB (1048576)
716 @param units: the type of formatting we should do:
717 - 'h' for automatic scaling
722 @return: the formatted value (with suffix)
725 if units not in ('m', 'g', 't', 'h'):
726 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
730 if units == 'm' or (units == 'h' and value < 1024):
733 return "%d%s" % (round(value, 0), suffix)
735 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
738 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
743 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
746 def ParseUnit(input_string):
747 """Tries to extract number and scale from the given string.
749 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
750 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
751 is always an int in MiB.
754 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
756 raise errors.UnitParseError("Invalid format")
758 value = float(m.groups()[0])
762 lcunit = unit.lower()
766 if lcunit in ('m', 'mb', 'mib'):
767 # Value already in MiB
770 elif lcunit in ('g', 'gb', 'gib'):
773 elif lcunit in ('t', 'tb', 'tib'):
777 raise errors.UnitParseError("Unknown unit: %s" % unit)
779 # Make sure we round up
780 if int(value) < value:
783 # Round up to the next multiple of 4
786 value += 4 - value % 4
791 def AddAuthorizedKey(file_name, key):
792 """Adds an SSH public key to an authorized_keys file.
795 @param file_name: path to authorized_keys file
797 @param key: string containing key
800 key_fields = key.split()
802 f = open(file_name, 'a+')
806 # Ignore whitespace changes
807 if line.split() == key_fields:
809 nl = line.endswith('\n')
813 f.write(key.rstrip('\r\n'))
820 def RemoveAuthorizedKey(file_name, key):
821 """Removes an SSH public key from an authorized_keys file.
824 @param file_name: path to authorized_keys file
826 @param key: string containing key
829 key_fields = key.split()
831 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
833 out = os.fdopen(fd, 'w')
835 f = open(file_name, 'r')
838 # Ignore whitespace changes while comparing lines
839 if line.split() != key_fields:
843 os.rename(tmpname, file_name)
853 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
854 """Sets the name of an IP address and hostname in /etc/hosts.
857 @param file_name: path to the file to modify (usually C{/etc/hosts})
859 @param ip: the IP address
861 @param hostname: the hostname to be added
863 @param aliases: the list of aliases to add for the hostname
866 # Ensure aliases are unique
867 aliases = UniqueSequence([hostname] + aliases)[1:]
869 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
871 out = os.fdopen(fd, 'w')
873 f = open(file_name, 'r')
876 fields = line.split()
877 if fields and not fields[0].startswith('#') and ip == fields[0]:
881 out.write("%s\t%s" % (ip, hostname))
883 out.write(" %s" % ' '.join(aliases))
888 os.rename(tmpname, file_name)
898 def AddHostToEtcHosts(hostname):
899 """Wrapper around SetEtcHostsEntry.
902 @param hostname: a hostname that will be resolved and added to
903 L{constants.ETC_HOSTS}
906 hi = HostInfo(name=hostname)
907 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
910 def RemoveEtcHostsEntry(file_name, hostname):
911 """Removes a hostname from /etc/hosts.
913 IP addresses without names are removed from the file.
916 @param file_name: path to the file to modify (usually C{/etc/hosts})
918 @param hostname: the hostname to be removed
921 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
923 out = os.fdopen(fd, 'w')
925 f = open(file_name, 'r')
928 fields = line.split()
929 if len(fields) > 1 and not fields[0].startswith('#'):
931 if hostname in names:
932 while hostname in names:
933 names.remove(hostname)
935 out.write("%s %s\n" % (fields[0], ' '.join(names)))
942 os.rename(tmpname, file_name)
952 def RemoveHostFromEtcHosts(hostname):
953 """Wrapper around RemoveEtcHostsEntry.
956 @param hostname: hostname that will be resolved and its
957 full and shot name will be removed from
958 L{constants.ETC_HOSTS}
961 hi = HostInfo(name=hostname)
962 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
963 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
966 def CreateBackup(file_name):
967 """Creates a backup of a file.
970 @param file_name: file to be backed up
972 @return: the path to the newly created backup
973 @raise errors.ProgrammerError: for invalid file names
976 if not os.path.isfile(file_name):
977 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
980 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
981 dir_name = os.path.dirname(file_name)
983 fsrc = open(file_name, 'rb')
985 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
986 fdst = os.fdopen(fd, 'wb')
988 shutil.copyfileobj(fsrc, fdst)
997 def ShellQuote(value):
998 """Quotes shell argument according to POSIX.
1001 @param value: the argument to be quoted
1003 @return: the quoted value
1006 if _re_shell_unquoted.match(value):
1009 return "'%s'" % value.replace("'", "'\\''")
1012 def ShellQuoteArgs(args):
1013 """Quotes a list of shell arguments.
1016 @param args: list of arguments to be quoted
1018 @return: the quoted arguments concatenaned with spaces
1021 return ' '.join([ShellQuote(i) for i in args])
1024 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1025 """Simple ping implementation using TCP connect(2).
1027 Check if the given IP is reachable by doing attempting a TCP connect
1031 @param target: the IP or hostname to ping
1033 @param port: the port to connect to
1035 @param timeout: the timeout on the connection attemp
1036 @type live_port_needed: boolean
1037 @param live_port_needed: whether a closed port will cause the
1038 function to return failure, as if there was a timeout
1039 @type source: str or None
1040 @param source: if specified, will cause the connect to be made
1041 from this specific source address; failures to bind other
1042 than C{EADDRNOTAVAIL} will be ignored
1045 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1049 if source is not None:
1051 sock.bind((source, 0))
1052 except socket.error, (errcode, errstring):
1053 if errcode == errno.EADDRNOTAVAIL:
1056 sock.settimeout(timeout)
1059 sock.connect((target, port))
1062 except socket.timeout:
1064 except socket.error, (errcode, errstring):
1065 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1070 def OwnIpAddress(address):
1071 """Check if the current host has the the given IP address.
1073 Currently this is done by TCP-pinging the address from the loopback
1076 @type address: string
1077 @param address: the addres to check
1079 @return: True if we own the address
1082 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1083 source=constants.LOCALHOST_IP_ADDRESS)
1086 def ListVisibleFiles(path):
1087 """Returns a list of visible files in a directory.
1090 @param path: the directory to enumerate
1092 @return: the list of all files not starting with a dot
1095 files = [i for i in os.listdir(path) if not i.startswith(".")]
1100 def GetHomeDir(user, default=None):
1101 """Try to get the homedir of the given user.
1103 The user can be passed either as a string (denoting the name) or as
1104 an integer (denoting the user id). If the user is not found, the
1105 'default' argument is returned, which defaults to None.
1109 if isinstance(user, basestring):
1110 result = pwd.getpwnam(user)
1111 elif isinstance(user, (int, long)):
1112 result = pwd.getpwuid(user)
1114 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1118 return result.pw_dir
1122 """Returns a random UUID.
1124 @note: This is a Linux-specific method as it uses the /proc
1129 f = open("/proc/sys/kernel/random/uuid", "r")
1131 return f.read(128).rstrip("\n")
1136 def GenerateSecret():
1137 """Generates a random secret.
1139 This will generate a pseudo-random secret, and return its sha digest
1140 (so that it can be used where an ASCII string is needed).
1143 @return: a sha1 hexdigest of a block of 64 random bytes
1146 return sha.new(os.urandom(64)).hexdigest()
1149 def ReadFile(file_name, size=None):
1152 @type size: None or int
1153 @param size: Read at most size bytes
1155 @return: the (possibly partial) conent of the file
1158 f = open(file_name, "r")
1168 def WriteFile(file_name, fn=None, data=None,
1169 mode=None, uid=-1, gid=-1,
1170 atime=None, mtime=None, close=True,
1171 dry_run=False, backup=False,
1172 prewrite=None, postwrite=None):
1173 """(Over)write a file atomically.
1175 The file_name and either fn (a function taking one argument, the
1176 file descriptor, and which should write the data to it) or data (the
1177 contents of the file) must be passed. The other arguments are
1178 optional and allow setting the file mode, owner and group, and the
1179 mtime/atime of the file.
1181 If the function doesn't raise an exception, it has succeeded and the
1182 target file has the new contents. If the file has raised an
1183 exception, an existing target file should be unmodified and the
1184 temporary file should be removed.
1186 @type file_name: str
1187 @param file_name: the target filename
1189 @param fn: content writing function, called with
1190 file descriptor as parameter
1192 @param data: contents of the file
1194 @param mode: file mode
1196 @param uid: the owner of the file
1198 @param gid: the group of the file
1200 @param atime: a custom access time to be set on the file
1202 @param mtime: a custom modification time to be set on the file
1203 @type close: boolean
1204 @param close: whether to close file after writing it
1205 @type prewrite: callable
1206 @param prewrite: function to be called before writing content
1207 @type postwrite: callable
1208 @param postwrite: function to be called after writing content
1211 @return: None if the 'close' parameter evaluates to True,
1212 otherwise the file descriptor
1214 @raise errors.ProgrammerError: if an of the arguments are not valid
1217 if not os.path.isabs(file_name):
1218 raise errors.ProgrammerError("Path passed to WriteFile is not"
1219 " absolute: '%s'" % file_name)
1221 if [fn, data].count(None) != 1:
1222 raise errors.ProgrammerError("fn or data required")
1224 if [atime, mtime].count(None) == 1:
1225 raise errors.ProgrammerError("Both atime and mtime must be either"
1228 if backup and not dry_run and os.path.isfile(file_name):
1229 CreateBackup(file_name)
1231 dir_name, base_name = os.path.split(file_name)
1232 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1233 # here we need to make sure we remove the temp file, if any error
1234 # leaves it in place
1236 if uid != -1 or gid != -1:
1237 os.chown(new_name, uid, gid)
1239 os.chmod(new_name, mode)
1240 if callable(prewrite):
1242 if data is not None:
1246 if callable(postwrite):
1249 if atime is not None and mtime is not None:
1250 os.utime(new_name, (atime, mtime))
1252 os.rename(new_name, file_name)
1259 RemoveFile(new_name)
1264 def FirstFree(seq, base=0):
1265 """Returns the first non-existing integer from seq.
1267 The seq argument should be a sorted list of positive integers. The
1268 first time the index of an element is smaller than the element
1269 value, the index will be returned.
1271 The base argument is used to start at a different offset,
1272 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1274 Example: C{[0, 1, 3]} will return I{2}.
1277 @param seq: the sequence to be analyzed.
1279 @param base: use this value as the base index of the sequence
1281 @return: the first non-used index in the sequence
1284 for idx, elem in enumerate(seq):
1285 assert elem >= base, "Passed element is higher than base offset"
1286 if elem > idx + base:
1292 def all(seq, pred=bool):
1293 "Returns True if pred(x) is True for every element in the iterable"
1294 for elem in itertools.ifilterfalse(pred, seq):
1299 def any(seq, pred=bool):
1300 "Returns True if pred(x) is True for at least one element in the iterable"
1301 for elem in itertools.ifilter(pred, seq):
1306 def UniqueSequence(seq):
1307 """Returns a list with unique elements.
1309 Element order is preserved.
1312 @param seq: the sequence with the source elementes
1314 @return: list of unique elements from seq
1318 return [i for i in seq if i not in seen and not seen.add(i)]
1321 def IsValidMac(mac):
1322 """Predicate to check if a MAC address is valid.
1324 Checks wether the supplied MAC address is formally correct, only
1325 accepts colon separated format.
1328 @param mac: the MAC to be validated
1330 @return: True is the MAC seems valid
1333 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1334 return mac_check.match(mac) is not None
1337 def TestDelay(duration):
1338 """Sleep for a fixed amount of time.
1340 @type duration: float
1341 @param duration: the sleep duration
1343 @return: False for negative value, True otherwise
1348 time.sleep(duration)
1352 def _CloseFDNoErr(fd, retries=5):
1353 """Close a file descriptor ignoring errors.
1356 @param fd: the file descriptor
1358 @param retries: how many retries to make, in case we get any
1359 other error than EBADF
1364 except OSError, err:
1365 if err.errno != errno.EBADF:
1367 _CloseFDNoErr(fd, retries - 1)
1368 # else either it's closed already or we're out of retries, so we
1369 # ignore this and go on
1372 def CloseFDs(noclose_fds=None):
1373 """Close file descriptors.
1375 This closes all file descriptors above 2 (i.e. except
1378 @type noclose_fds: list or None
1379 @param noclose_fds: if given, it denotes a list of file descriptor
1380 that should not be closed
1383 # Default maximum for the number of available file descriptors.
1384 if 'SC_OPEN_MAX' in os.sysconf_names:
1386 MAXFD = os.sysconf('SC_OPEN_MAX')
1393 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1394 if (maxfd == resource.RLIM_INFINITY):
1397 # Iterate through and close all file descriptors (except the standard ones)
1398 for fd in range(3, maxfd):
1399 if noclose_fds and fd in noclose_fds:
1404 def Daemonize(logfile):
1405 """Daemonize the current process.
1407 This detaches the current process from the controlling terminal and
1408 runs it in the background as a daemon.
1411 @param logfile: the logfile to which we should redirect stdout/stderr
1413 @returns: the value zero
1421 if (pid == 0): # The first child.
1424 pid = os.fork() # Fork a second child.
1425 if (pid == 0): # The second child.
1429 # exit() or _exit()? See below.
1430 os._exit(0) # Exit parent (the first child) of the second child.
1432 os._exit(0) # Exit parent of the first child.
1436 i = os.open("/dev/null", os.O_RDONLY) # stdin
1437 assert i == 0, "Can't close/reopen stdin"
1438 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1439 assert i == 1, "Can't close/reopen stdout"
1440 # Duplicate standard output to standard error.
1445 def DaemonPidFileName(name):
1446 """Compute a ganeti pid file absolute path
1449 @param name: the daemon name
1451 @return: the full path to the pidfile corresponding to the given
1455 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1458 def WritePidFile(name):
1459 """Write the current process pidfile.
1461 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1464 @param name: the daemon name to use
1465 @raise errors.GenericError: if the pid file already exists and
1466 points to a live process
1470 pidfilename = DaemonPidFileName(name)
1471 if IsProcessAlive(ReadPidFile(pidfilename)):
1472 raise errors.GenericError("%s contains a live process" % pidfilename)
1474 WriteFile(pidfilename, data="%d\n" % pid)
1477 def RemovePidFile(name):
1478 """Remove the current process pidfile.
1480 Any errors are ignored.
1483 @param name: the daemon name used to derive the pidfile name
1487 pidfilename = DaemonPidFileName(name)
1488 # TODO: we could check here that the file contains our pid
1490 RemoveFile(pidfilename)
1495 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1497 """Kill a process given by its pid.
1500 @param pid: The PID to terminate.
1502 @param signal_: The signal to send, by default SIGTERM
1504 @param timeout: The timeout after which, if the process is still alive,
1505 a SIGKILL will be sent. If not positive, no such checking
1507 @type waitpid: boolean
1508 @param waitpid: If true, we should waitpid on this process after
1509 sending signals, since it's our own child and otherwise it
1510 would remain as zombie
1513 def _helper(pid, signal_, wait):
1514 """Simple helper to encapsulate the kill/waitpid sequence"""
1515 os.kill(pid, signal_)
1518 os.waitpid(pid, os.WNOHANG)
1523 # kill with pid=0 == suicide
1524 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1526 if not IsProcessAlive(pid):
1528 _helper(pid, signal_, waitpid)
1532 # Wait up to $timeout seconds
1533 end = time.time() + timeout
1535 while time.time() < end and IsProcessAlive(pid):
1537 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1543 # Make wait time longer for next try
1547 if IsProcessAlive(pid):
1548 # Kill process if it's still alive
1549 _helper(pid, signal.SIGKILL, waitpid)
1552 def FindFile(name, search_path, test=os.path.exists):
1553 """Look for a filesystem object in a given path.
1555 This is an abstract method to search for filesystem object (files,
1556 dirs) under a given search path.
1559 @param name: the name to look for
1560 @type search_path: str
1561 @param search_path: location to start at
1562 @type test: callable
1563 @param test: a function taking one argument that should return True
1564 if the a given object is valid; the default value is
1565 os.path.exists, causing only existing files to be returned
1567 @return: full path to the object if found, None otherwise
1570 for dir_name in search_path:
1571 item_name = os.path.sep.join([dir_name, name])
1577 def CheckVolumeGroupSize(vglist, vgname, minsize):
1578 """Checks if the volume group list is valid.
1580 The function will check if a given volume group is in the list of
1581 volume groups and has a minimum size.
1584 @param vglist: dictionary of volume group names and their size
1586 @param vgname: the volume group we should check
1588 @param minsize: the minimum size we accept
1590 @return: None for success, otherwise the error message
1593 vgsize = vglist.get(vgname, None)
1595 return "volume group '%s' missing" % vgname
1596 elif vgsize < minsize:
1597 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1598 (vgname, minsize, vgsize))
1602 def SplitTime(value):
1603 """Splits time as floating point number into a tuple.
1605 @param value: Time in seconds
1606 @type value: int or float
1607 @return: Tuple containing (seconds, microseconds)
1610 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1612 assert 0 <= seconds, \
1613 "Seconds must be larger than or equal to 0, but are %s" % seconds
1614 assert 0 <= microseconds <= 999999, \
1615 "Microseconds must be 0-999999, but are %s" % microseconds
1617 return (int(seconds), int(microseconds))
1620 def MergeTime(timetuple):
1621 """Merges a tuple into time as a floating point number.
1623 @param timetuple: Time as tuple, (seconds, microseconds)
1624 @type timetuple: tuple
1625 @return: Time as a floating point number expressed in seconds
1628 (seconds, microseconds) = timetuple
1630 assert 0 <= seconds, \
1631 "Seconds must be larger than or equal to 0, but are %s" % seconds
1632 assert 0 <= microseconds <= 999999, \
1633 "Microseconds must be 0-999999, but are %s" % microseconds
1635 return float(seconds) + (float(microseconds) * 0.000001)
1638 def GetNodeDaemonPort():
1639 """Get the node daemon port for this cluster.
1641 Note that this routine does not read a ganeti-specific file, but
1642 instead uses C{socket.getservbyname} to allow pre-customization of
1643 this parameter outside of Ganeti.
1649 port = socket.getservbyname("ganeti-noded", "tcp")
1650 except socket.error:
1651 port = constants.DEFAULT_NODED_PORT
1656 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1657 multithreaded=False):
1658 """Configures the logging module.
1661 @param logfile: the filename to which we should log
1662 @type debug: boolean
1663 @param debug: whether to enable debug messages too or
1664 only those at C{INFO} and above level
1665 @type stderr_logging: boolean
1666 @param stderr_logging: whether we should also log to the standard error
1668 @param program: the name under which we should log messages
1669 @type multithreaded: boolean
1670 @param multithreaded: if True, will add the thread name to the log file
1671 @raise EnvironmentError: if we can't open the log file and
1672 stderr logging is disabled
1675 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1677 fmt += "/%(threadName)s"
1679 fmt += " %(module)s:%(lineno)s"
1680 fmt += " %(levelname)s %(message)s"
1681 formatter = logging.Formatter(fmt)
1683 root_logger = logging.getLogger("")
1684 root_logger.setLevel(logging.NOTSET)
1686 # Remove all previously setup handlers
1687 for handler in root_logger.handlers:
1689 root_logger.removeHandler(handler)
1692 stderr_handler = logging.StreamHandler()
1693 stderr_handler.setFormatter(formatter)
1695 stderr_handler.setLevel(logging.NOTSET)
1697 stderr_handler.setLevel(logging.CRITICAL)
1698 root_logger.addHandler(stderr_handler)
1700 # this can fail, if the logging directories are not setup or we have
1701 # a permisssion problem; in this case, it's best to log but ignore
1702 # the error if stderr_logging is True, and if false we re-raise the
1703 # exception since otherwise we could run but without any logs at all
1705 logfile_handler = logging.FileHandler(logfile)
1706 logfile_handler.setFormatter(formatter)
1708 logfile_handler.setLevel(logging.DEBUG)
1710 logfile_handler.setLevel(logging.INFO)
1711 root_logger.addHandler(logfile_handler)
1712 except EnvironmentError:
1714 logging.exception("Failed to enable logging to file '%s'", logfile)
1716 # we need to re-raise the exception
1720 def TailFile(fname, lines=20):
1721 """Return the last lines from a file.
1723 @note: this function will only read and parse the last 4KB of
1724 the file; if the lines are very long, it could be that less
1725 than the requested number of lines are returned
1727 @param fname: the file name
1729 @param lines: the (maximum) number of lines to return
1732 fd = open(fname, "r")
1736 pos = max(0, pos-4096)
1738 raw_data = fd.read()
1742 rows = raw_data.splitlines()
1743 return rows[-lines:]
1746 def LockedMethod(fn):
1747 """Synchronized object access decorator.
1749 This decorator is intended to protect access to an object using the
1750 object's own lock which is hardcoded to '_lock'.
1753 def _LockDebug(*args, **kwargs):
1755 logging.debug(*args, **kwargs)
1757 def wrapper(self, *args, **kwargs):
1758 assert hasattr(self, '_lock')
1760 _LockDebug("Waiting for %s", lock)
1763 _LockDebug("Acquired %s", lock)
1764 result = fn(self, *args, **kwargs)
1766 _LockDebug("Releasing %s", lock)
1768 _LockDebug("Released %s", lock)
1774 """Locks a file using POSIX locks.
1777 @param fd: the file descriptor we need to lock
1781 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1782 except IOError, err:
1783 if err.errno == errno.EAGAIN:
1784 raise errors.LockError("File already locked")
1788 class FileLock(object):
1789 """Utility class for file locks.
1792 def __init__(self, filename):
1793 """Constructor for FileLock.
1795 This will open the file denoted by the I{filename} argument.
1798 @param filename: path to the file to be locked
1801 self.filename = filename
1802 self.fd = open(self.filename, "w")
1808 """Close the file and release the lock.
1815 def _flock(self, flag, blocking, timeout, errmsg):
1816 """Wrapper for fcntl.flock.
1819 @param flag: operation flag
1820 @type blocking: bool
1821 @param blocking: whether the operation should be done in blocking mode.
1822 @type timeout: None or float
1823 @param timeout: for how long the operation should be retried (implies
1825 @type errmsg: string
1826 @param errmsg: error message in case operation fails.
1829 assert self.fd, "Lock was closed"
1830 assert timeout is None or timeout >= 0, \
1831 "If specified, timeout must be positive"
1833 if timeout is not None:
1834 flag |= fcntl.LOCK_NB
1835 timeout_end = time.time() + timeout
1837 # Blocking doesn't have effect with timeout
1839 flag |= fcntl.LOCK_NB
1845 fcntl.flock(self.fd, flag)
1847 except IOError, err:
1848 if err.errno in (errno.EAGAIN, ):
1849 if timeout_end is not None and time.time() < timeout_end:
1850 # Wait before trying again
1851 time.sleep(max(0.1, min(1.0, timeout)))
1853 raise errors.LockError(errmsg)
1855 logging.exception("fcntl.flock failed")
1858 def Exclusive(self, blocking=False, timeout=None):
1859 """Locks the file in exclusive mode.
1861 @type blocking: boolean
1862 @param blocking: whether to block and wait until we
1863 can lock the file or return immediately
1864 @type timeout: int or None
1865 @param timeout: if not None, the duration to wait for the lock
1869 self._flock(fcntl.LOCK_EX, blocking, timeout,
1870 "Failed to lock %s in exclusive mode" % self.filename)
1872 def Shared(self, blocking=False, timeout=None):
1873 """Locks the file in shared mode.
1875 @type blocking: boolean
1876 @param blocking: whether to block and wait until we
1877 can lock the file or return immediately
1878 @type timeout: int or None
1879 @param timeout: if not None, the duration to wait for the lock
1883 self._flock(fcntl.LOCK_SH, blocking, timeout,
1884 "Failed to lock %s in shared mode" % self.filename)
1886 def Unlock(self, blocking=True, timeout=None):
1887 """Unlocks the file.
1889 According to C{flock(2)}, unlocking can also be a nonblocking
1892 To make a non-blocking request, include LOCK_NB with any of the above
1895 @type blocking: boolean
1896 @param blocking: whether to block and wait until we
1897 can lock the file or return immediately
1898 @type timeout: int or None
1899 @param timeout: if not None, the duration to wait for the lock
1903 self._flock(fcntl.LOCK_UN, blocking, timeout,
1904 "Failed to unlock %s" % self.filename)
1907 class SignalHandler(object):
1908 """Generic signal handler class.
1910 It automatically restores the original handler when deconstructed or
1911 when L{Reset} is called. You can either pass your own handler
1912 function in or query the L{called} attribute to detect whether the
1916 @ivar signum: the signals we handle
1917 @type called: boolean
1918 @ivar called: tracks whether any of the signals have been raised
1921 def __init__(self, signum):
1922 """Constructs a new SignalHandler instance.
1924 @type signum: int or list of ints
1925 @param signum: Single signal number or set of signal numbers
1928 if isinstance(signum, (int, long)):
1929 self.signum = set([signum])
1931 self.signum = set(signum)
1937 for signum in self.signum:
1939 prev_handler = signal.signal(signum, self._HandleSignal)
1941 self._previous[signum] = prev_handler
1943 # Restore previous handler
1944 signal.signal(signum, prev_handler)
1947 # Reset all handlers
1949 # Here we have a race condition: a handler may have already been called,
1950 # but there's not much we can do about it at this point.
1957 """Restore previous handler.
1959 This will reset all the signals to their previous handlers.
1962 for signum, prev_handler in self._previous.items():
1963 signal.signal(signum, prev_handler)
1964 # If successful, remove from dict
1965 del self._previous[signum]
1968 """Unsets the L{called} flag.
1970 This function can be used in case a signal may arrive several times.
1975 def _HandleSignal(self, signum, frame):
1976 """Actual signal handling function.
1979 # This is not nice and not absolutely atomic, but it appears to be the only
1980 # solution in Python -- there are no atomic types.
1984 class FieldSet(object):
1985 """A simple field set.
1987 Among the features are:
1988 - checking if a string is among a list of static string or regex objects
1989 - checking if a whole list of string matches
1990 - returning the matching groups from a regex match
1992 Internally, all fields are held as regular expression objects.
1995 def __init__(self, *items):
1996 self.items = [re.compile("^%s$" % value) for value in items]
1998 def Extend(self, other_set):
1999 """Extend the field set with the items from another one"""
2000 self.items.extend(other_set.items)
2002 def Matches(self, field):
2003 """Checks if a field matches the current set
2006 @param field: the string to match
2007 @return: either False or a regular expression match object
2010 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2014 def NonMatching(self, items):
2015 """Returns the list of fields not matching the current set
2018 @param items: the list of fields to check
2020 @return: list of non-matching fields
2023 return [val for val in items if not self.Matches(val)]