4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
48 from cStringIO import StringIO
50 from ganeti import errors
51 from ganeti import constants
55 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60 #: when set to True, L{RunCmd} is disabled
64 class RunResult(object):
65 """Holds the result of running external programs.
68 @ivar exit_code: the exit code of the program, or None (if the program
70 @type signal: int or None
71 @ivar signal: the signal that caused the program to finish, or None
72 (if the program wasn't terminated by a signal)
74 @ivar stdout: the standard output of the program
76 @ivar stderr: the standard error of the program
78 @ivar failed: True in case the program was
79 terminated by a signal or exited with a non-zero exit code
80 @ivar fail_reason: a string detailing the termination reason
83 __slots__ = ["exit_code", "signal", "stdout", "stderr",
84 "failed", "fail_reason", "cmd"]
87 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
89 self.exit_code = exit_code
93 self.failed = (signal_ is not None or exit_code != 0)
95 if self.signal is not None:
96 self.fail_reason = "terminated by signal %s" % self.signal
97 elif self.exit_code is not None:
98 self.fail_reason = "exited with exit code %s" % self.exit_code
100 self.fail_reason = "unable to determine termination reason"
103 logging.debug("Command '%s' failed (%s); output: %s",
104 self.cmd, self.fail_reason, self.output)
106 def _GetOutput(self):
107 """Returns the combined stdout and stderr for easier usage.
110 return self.stdout + self.stderr
112 output = property(_GetOutput, None, None, "Return full output")
115 def RunCmd(cmd, env=None, output=None, cwd='/'):
116 """Execute a (shell) command.
118 The command should not read from its standard input, as it will be
121 @type cmd: string or list
122 @param cmd: Command to run
124 @param env: Additional environment
126 @param output: if desired, the output of the command can be
127 saved in a file instead of the RunResult instance; this
128 parameter denotes the file name (if not None)
130 @param cwd: if specified, will be used as the working
131 directory for the command; the default will be /
133 @return: RunResult instance
134 @raise erors.ProgrammerError: if we call this when forks are disabled
138 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
140 if isinstance(cmd, list):
141 cmd = [str(val) for val in cmd]
142 strcmd = " ".join(cmd)
147 logging.debug("RunCmd '%s'", strcmd)
149 cmd_env = os.environ.copy()
150 cmd_env["LC_ALL"] = "C"
155 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
157 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
167 return RunResult(exitcode, signal_, out, err, strcmd)
169 def _RunCmdPipe(cmd, env, via_shell, cwd):
170 """Run a command and return its output.
172 @type cmd: string or list
173 @param cmd: Command to run
175 @param env: The environment to use
176 @type via_shell: bool
177 @param via_shell: if we should run via the shell
179 @param cwd: the working directory for the program
181 @return: (out, err, status)
184 poller = select.poll()
185 child = subprocess.Popen(cmd, shell=via_shell,
186 stderr=subprocess.PIPE,
187 stdout=subprocess.PIPE,
188 stdin=subprocess.PIPE,
189 close_fds=True, env=env,
193 poller.register(child.stdout, select.POLLIN)
194 poller.register(child.stderr, select.POLLIN)
198 child.stdout.fileno(): (out, child.stdout),
199 child.stderr.fileno(): (err, child.stderr),
202 status = fcntl.fcntl(fd, fcntl.F_GETFL)
203 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
207 pollresult = poller.poll()
208 except EnvironmentError, eerr:
209 if eerr.errno == errno.EINTR:
212 except select.error, serr:
213 if serr[0] == errno.EINTR:
217 for fd, event in pollresult:
218 if event & select.POLLIN or event & select.POLLPRI:
219 data = fdmap[fd][1].read()
220 # no data from read signifies EOF (the same as POLLHUP)
222 poller.unregister(fd)
225 fdmap[fd][0].write(data)
226 if (event & select.POLLNVAL or event & select.POLLHUP or
227 event & select.POLLERR):
228 poller.unregister(fd)
234 status = child.wait()
235 return out, err, status
238 def _RunCmdFile(cmd, env, via_shell, output, cwd):
239 """Run a command and save its output to a file.
241 @type cmd: string or list
242 @param cmd: Command to run
244 @param env: The environment to use
245 @type via_shell: bool
246 @param via_shell: if we should run via the shell
248 @param output: the filename in which to save the output
250 @param cwd: the working directory for the program
252 @return: the exit status
255 fh = open(output, "a")
257 child = subprocess.Popen(cmd, shell=via_shell,
258 stderr=subprocess.STDOUT,
260 stdin=subprocess.PIPE,
261 close_fds=True, env=env,
265 status = child.wait()
271 def RemoveFile(filename):
272 """Remove a file ignoring some errors.
274 Remove a file, ignoring non-existing ones or directories. Other
278 @param filename: the file to be removed
284 if err.errno not in (errno.ENOENT, errno.EISDIR):
288 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
292 @param old: Original path
296 @param mkdir: Whether to create target directory if it doesn't exist
297 @type mkdir_mode: int
298 @param mkdir_mode: Mode for newly created directories
302 return os.rename(old, new)
304 # In at least one use case of this function, the job queue, directory
305 # creation is very rare. Checking for the directory before renaming is not
307 if mkdir and err.errno == errno.ENOENT:
308 # Create directory and try again
309 os.makedirs(os.path.dirname(new), mkdir_mode)
310 return os.rename(old, new)
314 def _FingerprintFile(filename):
315 """Compute the fingerprint of a file.
317 If the file does not exist, a None will be returned
321 @param filename: the filename to checksum
323 @return: the hex digest of the sha checksum of the contents
327 if not (os.path.exists(filename) and os.path.isfile(filename)):
340 return fp.hexdigest()
343 def FingerprintFiles(files):
344 """Compute fingerprints for a list of files.
347 @param files: the list of filename to fingerprint
349 @return: a dictionary filename: fingerprint, holding only
355 for filename in files:
356 cksum = _FingerprintFile(filename)
358 ret[filename] = cksum
363 def CheckDict(target, template, logname=None):
364 """Ensure a dictionary has a required set of keys.
366 For the given dictionaries I{target} and I{template}, ensure
367 I{target} has all the keys from I{template}. Missing keys are added
368 with values from template.
371 @param target: the dictionary to update
373 @param template: the dictionary holding the default values
374 @type logname: str or None
375 @param logname: if not None, causes the missing keys to be
376 logged with this name
383 target[k] = template[k]
385 if missing and logname:
386 logging.warning('%s missing keys %s', logname, ', '.join(missing))
389 def IsProcessAlive(pid):
390 """Check if a given pid exists on the system.
392 @note: zombie status is not handled, so zombie processes
393 will be returned as alive
395 @param pid: the process ID to check
397 @return: True if the process exists
404 os.stat("/proc/%d/status" % pid)
406 except EnvironmentError, err:
407 if err.errno in (errno.ENOENT, errno.ENOTDIR):
412 def ReadPidFile(pidfile):
413 """Read a pid from a file.
415 @type pidfile: string
416 @param pidfile: path to the file containing the pid
418 @return: The process id, if the file exista and contains a valid PID,
423 pf = open(pidfile, 'r')
424 except EnvironmentError, err:
425 if err.errno != errno.ENOENT:
426 logging.exception("Can't read pid file?!")
431 except ValueError, err:
432 logging.info("Can't parse pid file contents", exc_info=True)
438 def MatchNameComponent(key, name_list):
439 """Try to match a name against a list.
441 This function will try to match a name like test1 against a list
442 like C{['test1.example.com', 'test2.example.com', ...]}. Against
443 this list, I{'test1'} as well as I{'test1.example'} will match, but
444 not I{'test1.ex'}. A multiple match will be considered as no match
445 at all (e.g. I{'test1'} against C{['test1.example.com',
446 'test1.example.org']}).
449 @param key: the name to be searched
450 @type name_list: list
451 @param name_list: the list of strings against which to search the key
454 @return: None if there is no match I{or} if there are multiple matches,
455 otherwise the element from the list which matches
458 mo = re.compile("^%s(\..*)?$" % re.escape(key))
459 names_filtered = [name for name in name_list if mo.match(name) is not None]
460 if len(names_filtered) != 1:
462 return names_filtered[0]
466 """Class implementing resolver and hostname functionality
469 def __init__(self, name=None):
470 """Initialize the host name object.
472 If the name argument is not passed, it will use this system's
477 name = self.SysName()
480 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
481 self.ip = self.ipaddrs[0]
484 """Returns the hostname without domain.
487 return self.name.split('.')[0]
491 """Return the current system's name.
493 This is simply a wrapper over C{socket.gethostname()}.
496 return socket.gethostname()
499 def LookupHostname(hostname):
503 @param hostname: hostname to look up
506 @return: a tuple (name, aliases, ipaddrs) as returned by
507 C{socket.gethostbyname_ex}
508 @raise errors.ResolverError: in case of errors in resolving
512 result = socket.gethostbyname_ex(hostname)
513 except socket.gaierror, err:
514 # hostname not found in DNS
515 raise errors.ResolverError(hostname, err.args[0], err.args[1])
520 def ListVolumeGroups():
521 """List volume groups and their size
525 Dictionary with keys volume name and values
526 the size of the volume
529 command = "vgs --noheadings --units m --nosuffix -o name,size"
530 result = RunCmd(command)
535 for line in result.stdout.splitlines():
537 name, size = line.split()
538 size = int(float(size))
539 except (IndexError, ValueError), err:
540 logging.error("Invalid output from vgs (%s): %s", err, line)
548 def BridgeExists(bridge):
549 """Check whether the given bridge exists in the system
552 @param bridge: the bridge name to check
554 @return: True if it does
557 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
560 def CheckBEParams(beparams):
561 """Checks whether the user-supplied be-params are valid,
562 and converts them from string format where appropriate.
565 @param beparams: new params dict
569 for item in beparams:
570 if item not in constants.BES_PARAMETERS:
571 raise errors.OpPrereqError("Unknown backend parameter %s" % item)
572 if item in (constants.BE_MEMORY, constants.BE_VCPUS):
574 if val != constants.VALUE_DEFAULT:
577 except ValueError, err:
578 raise errors.OpPrereqError("Invalid %s size: %s" % (item, str(err)))
580 if item in (constants.BE_AUTO_BALANCE):
582 if not isinstance(val, bool):
583 if val == constants.VALUE_TRUE:
584 beparams[item] = True
585 elif val == constants.VALUE_FALSE:
586 beparams[item] = False
588 raise errors.OpPrereqError("Invalid %s value: %s" % (item, val))
591 def NiceSort(name_list):
592 """Sort a list of strings based on digit and non-digit groupings.
594 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
595 will sort the list in the logical order C{['a1', 'a2', 'a10',
598 The sort algorithm breaks each name in groups of either only-digits
599 or no-digits. Only the first eight such groups are considered, and
600 after that we just use what's left of the string.
602 @type name_list: list
603 @param name_list: the names to be sorted
605 @return: a copy of the name list sorted with our algorithm
608 _SORTER_BASE = "(\D+|\d+)"
609 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
610 _SORTER_BASE, _SORTER_BASE,
611 _SORTER_BASE, _SORTER_BASE,
612 _SORTER_BASE, _SORTER_BASE)
613 _SORTER_RE = re.compile(_SORTER_FULL)
614 _SORTER_NODIGIT = re.compile("^\D*$")
616 """Attempts to convert a variable to integer."""
617 if val is None or _SORTER_NODIGIT.match(val):
622 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
623 for name in name_list]
625 return [tup[1] for tup in to_sort]
628 def TryConvert(fn, val):
629 """Try to convert a value ignoring errors.
631 This function tries to apply function I{fn} to I{val}. If no
632 C{ValueError} or C{TypeError} exceptions are raised, it will return
633 the result, else it will return the original value. Any other
634 exceptions are propagated to the caller.
637 @param fn: function to apply to the value
638 @param val: the value to be converted
639 @return: The converted value if the conversion was successful,
640 otherwise the original value.
645 except (ValueError, TypeError), err:
651 """Verifies the syntax of an IPv4 address.
653 This function checks if the IPv4 address passes is valid or not based
654 on syntax (not IP range, class calculations, etc.).
657 @param ip: the address to be checked
658 @rtype: a regular expression match object
659 @return: a regular epression match object, or None if the
663 unit = "(0|[1-9]\d{0,2})"
664 #TODO: convert and return only boolean
665 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
668 def IsValidShellParam(word):
669 """Verifies is the given word is safe from the shell's p.o.v.
671 This means that we can pass this to a command via the shell and be
672 sure that it doesn't alter the command line and is passed as such to
675 Note that we are overly restrictive here, in order to be on the safe
679 @param word: the word to check
681 @return: True if the word is 'safe'
684 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
687 def BuildShellCmd(template, *args):
688 """Build a safe shell command line from the given arguments.
690 This function will check all arguments in the args list so that they
691 are valid shell parameters (i.e. they don't contain shell
692 metacharaters). If everything is ok, it will return the result of
696 @param template: the string holding the template for the
699 @return: the expanded command line
703 if not IsValidShellParam(word):
704 raise errors.ProgrammerError("Shell argument '%s' contains"
705 " invalid characters" % word)
706 return template % args
709 def FormatUnit(value, units):
710 """Formats an incoming number of MiB with the appropriate unit.
713 @param value: integer representing the value in MiB (1048576)
715 @param units: the type of formatting we should do:
716 - 'h' for automatic scaling
721 @return: the formatted value (with suffix)
724 if units not in ('m', 'g', 't', 'h'):
725 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
729 if units == 'm' or (units == 'h' and value < 1024):
732 return "%d%s" % (round(value, 0), suffix)
734 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
737 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
742 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
745 def ParseUnit(input_string):
746 """Tries to extract number and scale from the given string.
748 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
749 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
750 is always an int in MiB.
753 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
755 raise errors.UnitParseError("Invalid format")
757 value = float(m.groups()[0])
761 lcunit = unit.lower()
765 if lcunit in ('m', 'mb', 'mib'):
766 # Value already in MiB
769 elif lcunit in ('g', 'gb', 'gib'):
772 elif lcunit in ('t', 'tb', 'tib'):
776 raise errors.UnitParseError("Unknown unit: %s" % unit)
778 # Make sure we round up
779 if int(value) < value:
782 # Round up to the next multiple of 4
785 value += 4 - value % 4
790 def AddAuthorizedKey(file_name, key):
791 """Adds an SSH public key to an authorized_keys file.
794 @param file_name: path to authorized_keys file
796 @param key: string containing key
799 key_fields = key.split()
801 f = open(file_name, 'a+')
805 # Ignore whitespace changes
806 if line.split() == key_fields:
808 nl = line.endswith('\n')
812 f.write(key.rstrip('\r\n'))
819 def RemoveAuthorizedKey(file_name, key):
820 """Removes an SSH public key from an authorized_keys file.
823 @param file_name: path to authorized_keys file
825 @param key: string containing key
828 key_fields = key.split()
830 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
832 out = os.fdopen(fd, 'w')
834 f = open(file_name, 'r')
837 # Ignore whitespace changes while comparing lines
838 if line.split() != key_fields:
842 os.rename(tmpname, file_name)
852 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
853 """Sets the name of an IP address and hostname in /etc/hosts.
856 @param file_name: path to the file to modify (usually C{/etc/hosts})
858 @param ip: the IP address
860 @param hostname: the hostname to be added
862 @param aliases: the list of aliases to add for the hostname
865 # Ensure aliases are unique
866 aliases = UniqueSequence([hostname] + aliases)[1:]
868 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
870 out = os.fdopen(fd, 'w')
872 f = open(file_name, 'r')
875 fields = line.split()
876 if fields and not fields[0].startswith('#') and ip == fields[0]:
880 out.write("%s\t%s" % (ip, hostname))
882 out.write(" %s" % ' '.join(aliases))
887 os.rename(tmpname, file_name)
897 def AddHostToEtcHosts(hostname):
898 """Wrapper around SetEtcHostsEntry.
901 @param hostname: a hostname that will be resolved and added to
902 L{constants.ETC_HOSTS}
905 hi = HostInfo(name=hostname)
906 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
909 def RemoveEtcHostsEntry(file_name, hostname):
910 """Removes a hostname from /etc/hosts.
912 IP addresses without names are removed from the file.
915 @param file_name: path to the file to modify (usually C{/etc/hosts})
917 @param hostname: the hostname to be removed
920 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
922 out = os.fdopen(fd, 'w')
924 f = open(file_name, 'r')
927 fields = line.split()
928 if len(fields) > 1 and not fields[0].startswith('#'):
930 if hostname in names:
931 while hostname in names:
932 names.remove(hostname)
934 out.write("%s %s\n" % (fields[0], ' '.join(names)))
941 os.rename(tmpname, file_name)
951 def RemoveHostFromEtcHosts(hostname):
952 """Wrapper around RemoveEtcHostsEntry.
955 @param hostname: hostname that will be resolved and its
956 full and shot name will be removed from
957 L{constants.ETC_HOSTS}
960 hi = HostInfo(name=hostname)
961 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
962 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
965 def CreateBackup(file_name):
966 """Creates a backup of a file.
969 @param file_name: file to be backed up
971 @return: the path to the newly created backup
972 @raise errors.ProgrammerError: for invalid file names
975 if not os.path.isfile(file_name):
976 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
979 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
980 dir_name = os.path.dirname(file_name)
982 fsrc = open(file_name, 'rb')
984 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
985 fdst = os.fdopen(fd, 'wb')
987 shutil.copyfileobj(fsrc, fdst)
996 def ShellQuote(value):
997 """Quotes shell argument according to POSIX.
1000 @param value: the argument to be quoted
1002 @return: the quoted value
1005 if _re_shell_unquoted.match(value):
1008 return "'%s'" % value.replace("'", "'\\''")
1011 def ShellQuoteArgs(args):
1012 """Quotes a list of shell arguments.
1015 @param args: list of arguments to be quoted
1017 @return: the quoted arguments concatenaned with spaces
1020 return ' '.join([ShellQuote(i) for i in args])
1023 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1024 """Simple ping implementation using TCP connect(2).
1026 Check if the given IP is reachable by doing attempting a TCP connect
1030 @param target: the IP or hostname to ping
1032 @param port: the port to connect to
1034 @param timeout: the timeout on the connection attemp
1035 @type live_port_needed: boolean
1036 @param live_port_needed: whether a closed port will cause the
1037 function to return failure, as if there was a timeout
1038 @type source: str or None
1039 @param source: if specified, will cause the connect to be made
1040 from this specific source address; failures to bind other
1041 than C{EADDRNOTAVAIL} will be ignored
1044 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1048 if source is not None:
1050 sock.bind((source, 0))
1051 except socket.error, (errcode, errstring):
1052 if errcode == errno.EADDRNOTAVAIL:
1055 sock.settimeout(timeout)
1058 sock.connect((target, port))
1061 except socket.timeout:
1063 except socket.error, (errcode, errstring):
1064 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1069 def OwnIpAddress(address):
1070 """Check if the current host has the the given IP address.
1072 Currently this is done by TCP-pinging the address from the loopback
1075 @type address: string
1076 @param address: the addres to check
1078 @return: True if we own the address
1081 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1082 source=constants.LOCALHOST_IP_ADDRESS)
1085 def ListVisibleFiles(path):
1086 """Returns a list of visible files in a directory.
1089 @param path: the directory to enumerate
1091 @return: the list of all files not starting with a dot
1094 files = [i for i in os.listdir(path) if not i.startswith(".")]
1099 def GetHomeDir(user, default=None):
1100 """Try to get the homedir of the given user.
1102 The user can be passed either as a string (denoting the name) or as
1103 an integer (denoting the user id). If the user is not found, the
1104 'default' argument is returned, which defaults to None.
1108 if isinstance(user, basestring):
1109 result = pwd.getpwnam(user)
1110 elif isinstance(user, (int, long)):
1111 result = pwd.getpwuid(user)
1113 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1117 return result.pw_dir
1121 """Returns a random UUID.
1123 @note: This is a Linux-specific method as it uses the /proc
1128 f = open("/proc/sys/kernel/random/uuid", "r")
1130 return f.read(128).rstrip("\n")
1135 def GenerateSecret():
1136 """Generates a random secret.
1138 This will generate a pseudo-random secret, and return its sha digest
1139 (so that it can be used where an ASCII string is needed).
1142 @return: a sha1 hexdigest of a block of 64 random bytes
1145 return sha.new(os.urandom(64)).hexdigest()
1148 def ReadFile(file_name, size=None):
1151 @type size: None or int
1152 @param size: Read at most size bytes
1154 @return: the (possibly partial) conent of the file
1157 f = open(file_name, "r")
1167 def WriteFile(file_name, fn=None, data=None,
1168 mode=None, uid=-1, gid=-1,
1169 atime=None, mtime=None, close=True,
1170 dry_run=False, backup=False,
1171 prewrite=None, postwrite=None):
1172 """(Over)write a file atomically.
1174 The file_name and either fn (a function taking one argument, the
1175 file descriptor, and which should write the data to it) or data (the
1176 contents of the file) must be passed. The other arguments are
1177 optional and allow setting the file mode, owner and group, and the
1178 mtime/atime of the file.
1180 If the function doesn't raise an exception, it has succeeded and the
1181 target file has the new contents. If the file has raised an
1182 exception, an existing target file should be unmodified and the
1183 temporary file should be removed.
1185 @type file_name: str
1186 @param file_name: the target filename
1188 @param fn: content writing function, called with
1189 file descriptor as parameter
1191 @param data: contents of the file
1193 @param mode: file mode
1195 @param uid: the owner of the file
1197 @param gid: the group of the file
1199 @param atime: a custom access time to be set on the file
1201 @param mtime: a custom modification time to be set on the file
1202 @type close: boolean
1203 @param close: whether to close file after writing it
1204 @type prewrite: callable
1205 @param prewrite: function to be called before writing content
1206 @type postwrite: callable
1207 @param postwrite: function to be called after writing content
1210 @return: None if the 'close' parameter evaluates to True,
1211 otherwise the file descriptor
1213 @raise errors.ProgrammerError: if an of the arguments are not valid
1216 if not os.path.isabs(file_name):
1217 raise errors.ProgrammerError("Path passed to WriteFile is not"
1218 " absolute: '%s'" % file_name)
1220 if [fn, data].count(None) != 1:
1221 raise errors.ProgrammerError("fn or data required")
1223 if [atime, mtime].count(None) == 1:
1224 raise errors.ProgrammerError("Both atime and mtime must be either"
1227 if backup and not dry_run and os.path.isfile(file_name):
1228 CreateBackup(file_name)
1230 dir_name, base_name = os.path.split(file_name)
1231 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1232 # here we need to make sure we remove the temp file, if any error
1233 # leaves it in place
1235 if uid != -1 or gid != -1:
1236 os.chown(new_name, uid, gid)
1238 os.chmod(new_name, mode)
1239 if callable(prewrite):
1241 if data is not None:
1245 if callable(postwrite):
1248 if atime is not None and mtime is not None:
1249 os.utime(new_name, (atime, mtime))
1251 os.rename(new_name, file_name)
1258 RemoveFile(new_name)
1263 def FirstFree(seq, base=0):
1264 """Returns the first non-existing integer from seq.
1266 The seq argument should be a sorted list of positive integers. The
1267 first time the index of an element is smaller than the element
1268 value, the index will be returned.
1270 The base argument is used to start at a different offset,
1271 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1273 Example: C{[0, 1, 3]} will return I{2}.
1276 @param seq: the sequence to be analyzed.
1278 @param base: use this value as the base index of the sequence
1280 @return: the first non-used index in the sequence
1283 for idx, elem in enumerate(seq):
1284 assert elem >= base, "Passed element is higher than base offset"
1285 if elem > idx + base:
1291 def all(seq, pred=bool):
1292 "Returns True if pred(x) is True for every element in the iterable"
1293 for elem in itertools.ifilterfalse(pred, seq):
1298 def any(seq, pred=bool):
1299 "Returns True if pred(x) is True for at least one element in the iterable"
1300 for elem in itertools.ifilter(pred, seq):
1305 def UniqueSequence(seq):
1306 """Returns a list with unique elements.
1308 Element order is preserved.
1311 @param seq: the sequence with the source elementes
1313 @return: list of unique elements from seq
1317 return [i for i in seq if i not in seen and not seen.add(i)]
1320 def IsValidMac(mac):
1321 """Predicate to check if a MAC address is valid.
1323 Checks wether the supplied MAC address is formally correct, only
1324 accepts colon separated format.
1327 @param mac: the MAC to be validated
1329 @return: True is the MAC seems valid
1332 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1333 return mac_check.match(mac) is not None
1336 def TestDelay(duration):
1337 """Sleep for a fixed amount of time.
1339 @type duration: float
1340 @param duration: the sleep duration
1342 @return: False for negative value, True otherwise
1347 time.sleep(duration)
1351 def _CloseFDNoErr(fd, retries=5):
1352 """Close a file descriptor ignoring errors.
1355 @param fd: the file descriptor
1357 @param retries: how many retries to make, in case we get any
1358 other error than EBADF
1363 except OSError, err:
1364 if err.errno != errno.EBADF:
1366 _CloseFDNoErr(fd, retries - 1)
1367 # else either it's closed already or we're out of retries, so we
1368 # ignore this and go on
1371 def CloseFDs(noclose_fds=None):
1372 """Close file descriptors.
1374 This closes all file descriptors above 2 (i.e. except
1377 @type noclose_fds: list or None
1378 @param noclose_fds: if given, it denotes a list of file descriptor
1379 that should not be closed
1382 # Default maximum for the number of available file descriptors.
1383 if 'SC_OPEN_MAX' in os.sysconf_names:
1385 MAXFD = os.sysconf('SC_OPEN_MAX')
1392 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1393 if (maxfd == resource.RLIM_INFINITY):
1396 # Iterate through and close all file descriptors (except the standard ones)
1397 for fd in range(3, maxfd):
1398 if noclose_fds and fd in noclose_fds:
1403 def Daemonize(logfile):
1404 """Daemonize the current process.
1406 This detaches the current process from the controlling terminal and
1407 runs it in the background as a daemon.
1410 @param logfile: the logfile to which we should redirect stdout/stderr
1412 @returns: the value zero
1420 if (pid == 0): # The first child.
1423 pid = os.fork() # Fork a second child.
1424 if (pid == 0): # The second child.
1428 # exit() or _exit()? See below.
1429 os._exit(0) # Exit parent (the first child) of the second child.
1431 os._exit(0) # Exit parent of the first child.
1435 i = os.open("/dev/null", os.O_RDONLY) # stdin
1436 assert i == 0, "Can't close/reopen stdin"
1437 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1438 assert i == 1, "Can't close/reopen stdout"
1439 # Duplicate standard output to standard error.
1444 def DaemonPidFileName(name):
1445 """Compute a ganeti pid file absolute path
1448 @param name: the daemon name
1450 @return: the full path to the pidfile corresponding to the given
1454 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1457 def WritePidFile(name):
1458 """Write the current process pidfile.
1460 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1463 @param name: the daemon name to use
1464 @raise errors.GenericError: if the pid file already exists and
1465 points to a live process
1469 pidfilename = DaemonPidFileName(name)
1470 if IsProcessAlive(ReadPidFile(pidfilename)):
1471 raise errors.GenericError("%s contains a live process" % pidfilename)
1473 WriteFile(pidfilename, data="%d\n" % pid)
1476 def RemovePidFile(name):
1477 """Remove the current process pidfile.
1479 Any errors are ignored.
1482 @param name: the daemon name used to derive the pidfile name
1486 pidfilename = DaemonPidFileName(name)
1487 # TODO: we could check here that the file contains our pid
1489 RemoveFile(pidfilename)
1494 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1496 """Kill a process given by its pid.
1499 @param pid: The PID to terminate.
1501 @param signal_: The signal to send, by default SIGTERM
1503 @param timeout: The timeout after which, if the process is still alive,
1504 a SIGKILL will be sent. If not positive, no such checking
1506 @type waitpid: boolean
1507 @param waitpid: If true, we should waitpid on this process after
1508 sending signals, since it's our own child and otherwise it
1509 would remain as zombie
1512 def _helper(pid, signal_, wait):
1513 """Simple helper to encapsulate the kill/waitpid sequence"""
1514 os.kill(pid, signal_)
1517 os.waitpid(pid, os.WNOHANG)
1522 # kill with pid=0 == suicide
1523 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1525 if not IsProcessAlive(pid):
1527 _helper(pid, signal_, waitpid)
1531 # Wait up to $timeout seconds
1532 end = time.time() + timeout
1534 while time.time() < end and IsProcessAlive(pid):
1536 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1542 # Make wait time longer for next try
1546 if IsProcessAlive(pid):
1547 # Kill process if it's still alive
1548 _helper(pid, signal.SIGKILL, waitpid)
1551 def FindFile(name, search_path, test=os.path.exists):
1552 """Look for a filesystem object in a given path.
1554 This is an abstract method to search for filesystem object (files,
1555 dirs) under a given search path.
1558 @param name: the name to look for
1559 @type search_path: str
1560 @param search_path: location to start at
1561 @type test: callable
1562 @param test: a function taking one argument that should return True
1563 if the a given object is valid; the default value is
1564 os.path.exists, causing only existing files to be returned
1566 @return: full path to the object if found, None otherwise
1569 for dir_name in search_path:
1570 item_name = os.path.sep.join([dir_name, name])
1576 def CheckVolumeGroupSize(vglist, vgname, minsize):
1577 """Checks if the volume group list is valid.
1579 The function will check if a given volume group is in the list of
1580 volume groups and has a minimum size.
1583 @param vglist: dictionary of volume group names and their size
1585 @param vgname: the volume group we should check
1587 @param minsize: the minimum size we accept
1589 @return: None for success, otherwise the error message
1592 vgsize = vglist.get(vgname, None)
1594 return "volume group '%s' missing" % vgname
1595 elif vgsize < minsize:
1596 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1597 (vgname, minsize, vgsize))
1601 def SplitTime(value):
1602 """Splits time as floating point number into a tuple.
1604 @param value: Time in seconds
1605 @type value: int or float
1606 @return: Tuple containing (seconds, microseconds)
1609 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1611 assert 0 <= seconds, \
1612 "Seconds must be larger than or equal to 0, but are %s" % seconds
1613 assert 0 <= microseconds <= 999999, \
1614 "Microseconds must be 0-999999, but are %s" % microseconds
1616 return (int(seconds), int(microseconds))
1619 def MergeTime(timetuple):
1620 """Merges a tuple into time as a floating point number.
1622 @param timetuple: Time as tuple, (seconds, microseconds)
1623 @type timetuple: tuple
1624 @return: Time as a floating point number expressed in seconds
1627 (seconds, microseconds) = timetuple
1629 assert 0 <= seconds, \
1630 "Seconds must be larger than or equal to 0, but are %s" % seconds
1631 assert 0 <= microseconds <= 999999, \
1632 "Microseconds must be 0-999999, but are %s" % microseconds
1634 return float(seconds) + (float(microseconds) * 0.000001)
1637 def GetNodeDaemonPort():
1638 """Get the node daemon port for this cluster.
1640 Note that this routine does not read a ganeti-specific file, but
1641 instead uses C{socket.getservbyname} to allow pre-customization of
1642 this parameter outside of Ganeti.
1648 port = socket.getservbyname("ganeti-noded", "tcp")
1649 except socket.error:
1650 port = constants.DEFAULT_NODED_PORT
1655 def SetupLogging(logfile, debug=False, stderr_logging=False, program=""):
1656 """Configures the logging module.
1659 @param logfile: the filename to which we should log
1660 @type debug: boolean
1661 @param debug: whether to enable debug messages too or
1662 only those at C{INFO} and above level
1663 @type stderr_logging: boolean
1664 @param stderr_logging: whether we should also log to the standard error
1666 @param program: the name under which we should log messages
1667 @raise EnvironmentError: if we can't open the log file and
1668 stderr logging is disabled
1671 fmt = "%(asctime)s: " + program + " "
1673 fmt += ("pid=%(process)d/%(threadName)s %(levelname)s"
1674 " %(module)s:%(lineno)s %(message)s")
1676 fmt += "pid=%(process)d %(levelname)s %(message)s"
1677 formatter = logging.Formatter(fmt)
1679 root_logger = logging.getLogger("")
1680 root_logger.setLevel(logging.NOTSET)
1682 # Remove all previously setup handlers
1683 for handler in root_logger.handlers:
1685 root_logger.removeHandler(handler)
1688 stderr_handler = logging.StreamHandler()
1689 stderr_handler.setFormatter(formatter)
1691 stderr_handler.setLevel(logging.NOTSET)
1693 stderr_handler.setLevel(logging.CRITICAL)
1694 root_logger.addHandler(stderr_handler)
1696 # this can fail, if the logging directories are not setup or we have
1697 # a permisssion problem; in this case, it's best to log but ignore
1698 # the error if stderr_logging is True, and if false we re-raise the
1699 # exception since otherwise we could run but without any logs at all
1701 logfile_handler = logging.FileHandler(logfile)
1702 logfile_handler.setFormatter(formatter)
1704 logfile_handler.setLevel(logging.DEBUG)
1706 logfile_handler.setLevel(logging.INFO)
1707 root_logger.addHandler(logfile_handler)
1708 except EnvironmentError, err:
1710 logging.exception("Failed to enable logging to file '%s'", logfile)
1712 # we need to re-raise the exception
1716 def LockedMethod(fn):
1717 """Synchronized object access decorator.
1719 This decorator is intended to protect access to an object using the
1720 object's own lock which is hardcoded to '_lock'.
1723 def _LockDebug(*args, **kwargs):
1725 logging.debug(*args, **kwargs)
1727 def wrapper(self, *args, **kwargs):
1728 assert hasattr(self, '_lock')
1730 _LockDebug("Waiting for %s", lock)
1733 _LockDebug("Acquired %s", lock)
1734 result = fn(self, *args, **kwargs)
1736 _LockDebug("Releasing %s", lock)
1738 _LockDebug("Released %s", lock)
1744 """Locks a file using POSIX locks.
1747 @param fd: the file descriptor we need to lock
1751 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1752 except IOError, err:
1753 if err.errno == errno.EAGAIN:
1754 raise errors.LockError("File already locked")
1758 class FileLock(object):
1759 """Utility class for file locks.
1762 def __init__(self, filename):
1763 """Constructor for FileLock.
1765 This will open the file denoted by the I{filename} argument.
1768 @param filename: path to the file to be locked
1771 self.filename = filename
1772 self.fd = open(self.filename, "w")
1778 """Close the file and release the lock.
1785 def _flock(self, flag, blocking, timeout, errmsg):
1786 """Wrapper for fcntl.flock.
1789 @param flag: operation flag
1790 @type blocking: bool
1791 @param blocking: whether the operation should be done in blocking mode.
1792 @type timeout: None or float
1793 @param timeout: for how long the operation should be retried (implies
1795 @type errmsg: string
1796 @param errmsg: error message in case operation fails.
1799 assert self.fd, "Lock was closed"
1800 assert timeout is None or timeout >= 0, \
1801 "If specified, timeout must be positive"
1803 if timeout is not None:
1804 flag |= fcntl.LOCK_NB
1805 timeout_end = time.time() + timeout
1807 # Blocking doesn't have effect with timeout
1809 flag |= fcntl.LOCK_NB
1815 fcntl.flock(self.fd, flag)
1817 except IOError, err:
1818 if err.errno in (errno.EAGAIN, ):
1819 if timeout_end is not None and time.time() < timeout_end:
1820 # Wait before trying again
1821 time.sleep(max(0.1, min(1.0, timeout)))
1823 raise errors.LockError(errmsg)
1825 logging.exception("fcntl.flock failed")
1828 def Exclusive(self, blocking=False, timeout=None):
1829 """Locks the file in exclusive mode.
1831 @type blocking: boolean
1832 @param blocking: whether to block and wait until we
1833 can lock the file or return immediately
1834 @type timeout: int or None
1835 @param timeout: if not None, the duration to wait for the lock
1839 self._flock(fcntl.LOCK_EX, blocking, timeout,
1840 "Failed to lock %s in exclusive mode" % self.filename)
1842 def Shared(self, blocking=False, timeout=None):
1843 """Locks the file in shared mode.
1845 @type blocking: boolean
1846 @param blocking: whether to block and wait until we
1847 can lock the file or return immediately
1848 @type timeout: int or None
1849 @param timeout: if not None, the duration to wait for the lock
1853 self._flock(fcntl.LOCK_SH, blocking, timeout,
1854 "Failed to lock %s in shared mode" % self.filename)
1856 def Unlock(self, blocking=True, timeout=None):
1857 """Unlocks the file.
1859 According to C{flock(2)}, unlocking can also be a nonblocking
1862 To make a non-blocking request, include LOCK_NB with any of the above
1865 @type blocking: boolean
1866 @param blocking: whether to block and wait until we
1867 can lock the file or return immediately
1868 @type timeout: int or None
1869 @param timeout: if not None, the duration to wait for the lock
1873 self._flock(fcntl.LOCK_UN, blocking, timeout,
1874 "Failed to unlock %s" % self.filename)
1877 class SignalHandler(object):
1878 """Generic signal handler class.
1880 It automatically restores the original handler when deconstructed or
1881 when L{Reset} is called. You can either pass your own handler
1882 function in or query the L{called} attribute to detect whether the
1886 @ivar signum: the signals we handle
1887 @type called: boolean
1888 @ivar called: tracks whether any of the signals have been raised
1891 def __init__(self, signum):
1892 """Constructs a new SignalHandler instance.
1894 @type signum: int or list of ints
1895 @param signum: Single signal number or set of signal numbers
1898 if isinstance(signum, (int, long)):
1899 self.signum = set([signum])
1901 self.signum = set(signum)
1907 for signum in self.signum:
1909 prev_handler = signal.signal(signum, self._HandleSignal)
1911 self._previous[signum] = prev_handler
1913 # Restore previous handler
1914 signal.signal(signum, prev_handler)
1917 # Reset all handlers
1919 # Here we have a race condition: a handler may have already been called,
1920 # but there's not much we can do about it at this point.
1927 """Restore previous handler.
1929 This will reset all the signals to their previous handlers.
1932 for signum, prev_handler in self._previous.items():
1933 signal.signal(signum, prev_handler)
1934 # If successful, remove from dict
1935 del self._previous[signum]
1938 """Unsets the L{called} flag.
1940 This function can be used in case a signal may arrive several times.
1945 def _HandleSignal(self, signum, frame):
1946 """Actual signal handling function.
1949 # This is not nice and not absolutely atomic, but it appears to be the only
1950 # solution in Python -- there are no atomic types.
1954 class FieldSet(object):
1955 """A simple field set.
1957 Among the features are:
1958 - checking if a string is among a list of static string or regex objects
1959 - checking if a whole list of string matches
1960 - returning the matching groups from a regex match
1962 Internally, all fields are held as regular expression objects.
1965 def __init__(self, *items):
1966 self.items = [re.compile("^%s$" % value) for value in items]
1968 def Extend(self, other_set):
1969 """Extend the field set with the items from another one"""
1970 self.items.extend(other_set.items)
1972 def Matches(self, field):
1973 """Checks if a field matches the current set
1976 @param field: the string to match
1977 @return: either False or a regular expression match object
1980 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
1984 def NonMatching(self, items):
1985 """Returns the list of fields not matching the current set
1988 @param items: the list of fields to check
1990 @return: list of non-matching fields
1993 return [val for val in items if not self.Matches(val)]