4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
48 from cStringIO import StringIO
50 from ganeti import errors
51 from ganeti import constants
55 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60 #: when set to True, L{RunCmd} is disabled
64 class RunResult(object):
65 """Holds the result of running external programs.
68 @ivar exit_code: the exit code of the program, or None (if the program
70 @type signal: int or None
71 @ivar signal: the signal that caused the program to finish, or None
72 (if the program wasn't terminated by a signal)
74 @ivar stdout: the standard output of the program
76 @ivar stderr: the standard error of the program
78 @ivar failed: True in case the program was
79 terminated by a signal or exited with a non-zero exit code
80 @ivar fail_reason: a string detailing the termination reason
83 __slots__ = ["exit_code", "signal", "stdout", "stderr",
84 "failed", "fail_reason", "cmd"]
87 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
89 self.exit_code = exit_code
93 self.failed = (signal_ is not None or exit_code != 0)
95 if self.signal is not None:
96 self.fail_reason = "terminated by signal %s" % self.signal
97 elif self.exit_code is not None:
98 self.fail_reason = "exited with exit code %s" % self.exit_code
100 self.fail_reason = "unable to determine termination reason"
103 logging.debug("Command '%s' failed (%s); output: %s",
104 self.cmd, self.fail_reason, self.output)
106 def _GetOutput(self):
107 """Returns the combined stdout and stderr for easier usage.
110 return self.stdout + self.stderr
112 output = property(_GetOutput, None, None, "Return full output")
115 def RunCmd(cmd, env=None, output=None, cwd='/'):
116 """Execute a (shell) command.
118 The command should not read from its standard input, as it will be
121 @type cmd: string or list
122 @param cmd: Command to run
124 @param env: Additional environment
126 @param output: if desired, the output of the command can be
127 saved in a file instead of the RunResult instance; this
128 parameter denotes the file name (if not None)
130 @param cwd: if specified, will be used as the working
131 directory for the command; the default will be /
133 @return: RunResult instance
134 @raise erors.ProgrammerError: if we call this when forks are disabled
138 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
140 if isinstance(cmd, list):
141 cmd = [str(val) for val in cmd]
142 strcmd = " ".join(cmd)
147 logging.debug("RunCmd '%s'", strcmd)
149 cmd_env = os.environ.copy()
150 cmd_env["LC_ALL"] = "C"
155 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
157 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
167 return RunResult(exitcode, signal_, out, err, strcmd)
169 def _RunCmdPipe(cmd, env, via_shell, cwd):
170 """Run a command and return its output.
172 @type cmd: string or list
173 @param cmd: Command to run
175 @param env: The environment to use
176 @type via_shell: bool
177 @param via_shell: if we should run via the shell
179 @param cwd: the working directory for the program
181 @return: (out, err, status)
184 poller = select.poll()
185 child = subprocess.Popen(cmd, shell=via_shell,
186 stderr=subprocess.PIPE,
187 stdout=subprocess.PIPE,
188 stdin=subprocess.PIPE,
189 close_fds=True, env=env,
193 poller.register(child.stdout, select.POLLIN)
194 poller.register(child.stderr, select.POLLIN)
198 child.stdout.fileno(): (out, child.stdout),
199 child.stderr.fileno(): (err, child.stderr),
202 status = fcntl.fcntl(fd, fcntl.F_GETFL)
203 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
207 pollresult = poller.poll()
208 except EnvironmentError, eerr:
209 if eerr.errno == errno.EINTR:
212 except select.error, serr:
213 if serr[0] == errno.EINTR:
217 for fd, event in pollresult:
218 if event & select.POLLIN or event & select.POLLPRI:
219 data = fdmap[fd][1].read()
220 # no data from read signifies EOF (the same as POLLHUP)
222 poller.unregister(fd)
225 fdmap[fd][0].write(data)
226 if (event & select.POLLNVAL or event & select.POLLHUP or
227 event & select.POLLERR):
228 poller.unregister(fd)
234 status = child.wait()
235 return out, err, status
238 def _RunCmdFile(cmd, env, via_shell, output, cwd):
239 """Run a command and save its output to a file.
241 @type cmd: string or list
242 @param cmd: Command to run
244 @param env: The environment to use
245 @type via_shell: bool
246 @param via_shell: if we should run via the shell
248 @param output: the filename in which to save the output
250 @param cwd: the working directory for the program
252 @return: the exit status
255 fh = open(output, "a")
257 child = subprocess.Popen(cmd, shell=via_shell,
258 stderr=subprocess.STDOUT,
260 stdin=subprocess.PIPE,
261 close_fds=True, env=env,
265 status = child.wait()
271 def RemoveFile(filename):
272 """Remove a file ignoring some errors.
274 Remove a file, ignoring non-existing ones or directories. Other
278 @param filename: the file to be removed
284 if err.errno not in (errno.ENOENT, errno.EISDIR):
288 def _FingerprintFile(filename):
289 """Compute the fingerprint of a file.
291 If the file does not exist, a None will be returned
295 @param filename: the filename to checksum
297 @return: the hex digest of the sha checksum of the contents
301 if not (os.path.exists(filename) and os.path.isfile(filename)):
314 return fp.hexdigest()
317 def FingerprintFiles(files):
318 """Compute fingerprints for a list of files.
321 @param files: the list of filename to fingerprint
323 @return: a dictionary filename: fingerprint, holding only
329 for filename in files:
330 cksum = _FingerprintFile(filename)
332 ret[filename] = cksum
337 def CheckDict(target, template, logname=None):
338 """Ensure a dictionary has a required set of keys.
340 For the given dictionaries I{target} and I{template}, ensure
341 I{target} has all the keys from I{template}. Missing keys are added
342 with values from template.
345 @param target: the dictionary to update
347 @param template: the dictionary holding the default values
348 @type logname: str or None
349 @param logname: if not None, causes the missing keys to be
350 logged with this name
357 target[k] = template[k]
359 if missing and logname:
360 logging.warning('%s missing keys %s', logname, ', '.join(missing))
363 def IsProcessAlive(pid):
364 """Check if a given pid exists on the system.
366 @note: zombie status is not handled, so zombie processes
367 will be returned as alive
369 @param pid: the process ID to check
371 @return: True if the process exists
378 os.stat("/proc/%d/status" % pid)
380 except EnvironmentError, err:
381 if err.errno in (errno.ENOENT, errno.ENOTDIR):
386 def ReadPidFile(pidfile):
387 """Read a pid from a file.
389 @type pidfile: string
390 @param pidfile: path to the file containing the pid
392 @return: The process id, if the file exista and contains a valid PID,
397 pf = open(pidfile, 'r')
398 except EnvironmentError, err:
399 if err.errno != errno.ENOENT:
400 logging.exception("Can't read pid file?!")
405 except ValueError, err:
406 logging.info("Can't parse pid file contents", exc_info=True)
412 def MatchNameComponent(key, name_list):
413 """Try to match a name against a list.
415 This function will try to match a name like test1 against a list
416 like C{['test1.example.com', 'test2.example.com', ...]}. Against
417 this list, I{'test1'} as well as I{'test1.example'} will match, but
418 not I{'test1.ex'}. A multiple match will be considered as no match
419 at all (e.g. I{'test1'} against C{['test1.example.com',
420 'test1.example.org']}).
423 @param key: the name to be searched
424 @type name_list: list
425 @param name_list: the list of strings against which to search the key
428 @return: None if there is no match I{or} if there are multiple matches,
429 otherwise the element from the list which matches
432 mo = re.compile("^%s(\..*)?$" % re.escape(key))
433 names_filtered = [name for name in name_list if mo.match(name) is not None]
434 if len(names_filtered) != 1:
436 return names_filtered[0]
440 """Class implementing resolver and hostname functionality
443 def __init__(self, name=None):
444 """Initialize the host name object.
446 If the name argument is not passed, it will use this system's
451 name = self.SysName()
454 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
455 self.ip = self.ipaddrs[0]
458 """Returns the hostname without domain.
461 return self.name.split('.')[0]
465 """Return the current system's name.
467 This is simply a wrapper over C{socket.gethostname()}.
470 return socket.gethostname()
473 def LookupHostname(hostname):
477 @param hostname: hostname to look up
480 @return: a tuple (name, aliases, ipaddrs) as returned by
481 C{socket.gethostbyname_ex}
482 @raise errors.ResolverError: in case of errors in resolving
486 result = socket.gethostbyname_ex(hostname)
487 except socket.gaierror, err:
488 # hostname not found in DNS
489 raise errors.ResolverError(hostname, err.args[0], err.args[1])
494 def ListVolumeGroups():
495 """List volume groups and their size
499 Dictionary with keys volume name and values
500 the size of the volume
503 command = "vgs --noheadings --units m --nosuffix -o name,size"
504 result = RunCmd(command)
509 for line in result.stdout.splitlines():
511 name, size = line.split()
512 size = int(float(size))
513 except (IndexError, ValueError), err:
514 logging.error("Invalid output from vgs (%s): %s", err, line)
522 def BridgeExists(bridge):
523 """Check whether the given bridge exists in the system
526 @param bridge: the bridge name to check
528 @return: True if it does
531 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
534 def CheckBEParams(beparams):
535 """Checks whether the user-supplied be-params are valid,
536 and converts them from string format where appropriate.
539 @param beparams: new params dict
543 for item in beparams:
544 if item not in constants.BES_PARAMETERS:
545 raise errors.OpPrereqError("Unknown backend parameter %s" % item)
546 if item in (constants.BE_MEMORY, constants.BE_VCPUS):
548 if val != constants.VALUE_DEFAULT:
551 except ValueError, err:
552 raise errors.OpPrereqError("Invalid %s size: %s" % (item, str(err)))
554 if item in (constants.BE_AUTO_BALANCE):
556 if not isinstance(val, bool):
557 if val == constants.VALUE_TRUE:
558 beparams[item] = True
559 elif val == constants.VALUE_FALSE:
560 beparams[item] = False
562 raise errors.OpPrereqError("Invalid %s value: %s" % (item, val))
565 def NiceSort(name_list):
566 """Sort a list of strings based on digit and non-digit groupings.
568 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
569 will sort the list in the logical order C{['a1', 'a2', 'a10',
572 The sort algorithm breaks each name in groups of either only-digits
573 or no-digits. Only the first eight such groups are considered, and
574 after that we just use what's left of the string.
576 @type name_list: list
577 @param name_list: the names to be sorted
579 @return: a copy of the name list sorted with our algorithm
582 _SORTER_BASE = "(\D+|\d+)"
583 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
584 _SORTER_BASE, _SORTER_BASE,
585 _SORTER_BASE, _SORTER_BASE,
586 _SORTER_BASE, _SORTER_BASE)
587 _SORTER_RE = re.compile(_SORTER_FULL)
588 _SORTER_NODIGIT = re.compile("^\D*$")
590 """Attempts to convert a variable to integer."""
591 if val is None or _SORTER_NODIGIT.match(val):
596 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
597 for name in name_list]
599 return [tup[1] for tup in to_sort]
602 def TryConvert(fn, val):
603 """Try to convert a value ignoring errors.
605 This function tries to apply function I{fn} to I{val}. If no
606 C{ValueError} or C{TypeError} exceptions are raised, it will return
607 the result, else it will return the original value. Any other
608 exceptions are propagated to the caller.
611 @param fn: function to apply to the value
612 @param val: the value to be converted
613 @return: The converted value if the conversion was successful,
614 otherwise the original value.
619 except (ValueError, TypeError), err:
625 """Verifies the syntax of an IPv4 address.
627 This function checks if the IPv4 address passes is valid or not based
628 on syntax (not IP range, class calculations, etc.).
631 @param ip: the address to be checked
632 @rtype: a regular expression match object
633 @return: a regular epression match object, or None if the
637 unit = "(0|[1-9]\d{0,2})"
638 #TODO: convert and return only boolean
639 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
642 def IsValidShellParam(word):
643 """Verifies is the given word is safe from the shell's p.o.v.
645 This means that we can pass this to a command via the shell and be
646 sure that it doesn't alter the command line and is passed as such to
649 Note that we are overly restrictive here, in order to be on the safe
653 @param word: the word to check
655 @return: True if the word is 'safe'
658 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
661 def BuildShellCmd(template, *args):
662 """Build a safe shell command line from the given arguments.
664 This function will check all arguments in the args list so that they
665 are valid shell parameters (i.e. they don't contain shell
666 metacharaters). If everything is ok, it will return the result of
670 @param template: the string holding the template for the
673 @return: the expanded command line
677 if not IsValidShellParam(word):
678 raise errors.ProgrammerError("Shell argument '%s' contains"
679 " invalid characters" % word)
680 return template % args
683 def FormatUnit(value, units):
684 """Formats an incoming number of MiB with the appropriate unit.
687 @param value: integer representing the value in MiB (1048576)
689 @param units: the type of formatting we should do:
690 - 'h' for automatic scaling
695 @return: the formatted value (with suffix)
698 if units not in ('m', 'g', 't', 'h'):
699 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
703 if units == 'm' or (units == 'h' and value < 1024):
706 return "%d%s" % (round(value, 0), suffix)
708 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
711 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
716 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
719 def ParseUnit(input_string):
720 """Tries to extract number and scale from the given string.
722 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
723 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
724 is always an int in MiB.
727 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
729 raise errors.UnitParseError("Invalid format")
731 value = float(m.groups()[0])
735 lcunit = unit.lower()
739 if lcunit in ('m', 'mb', 'mib'):
740 # Value already in MiB
743 elif lcunit in ('g', 'gb', 'gib'):
746 elif lcunit in ('t', 'tb', 'tib'):
750 raise errors.UnitParseError("Unknown unit: %s" % unit)
752 # Make sure we round up
753 if int(value) < value:
756 # Round up to the next multiple of 4
759 value += 4 - value % 4
764 def AddAuthorizedKey(file_name, key):
765 """Adds an SSH public key to an authorized_keys file.
768 @param file_name: path to authorized_keys file
770 @param key: string containing key
773 key_fields = key.split()
775 f = open(file_name, 'a+')
779 # Ignore whitespace changes
780 if line.split() == key_fields:
782 nl = line.endswith('\n')
786 f.write(key.rstrip('\r\n'))
793 def RemoveAuthorizedKey(file_name, key):
794 """Removes an SSH public key from an authorized_keys file.
797 @param file_name: path to authorized_keys file
799 @param key: string containing key
802 key_fields = key.split()
804 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
806 out = os.fdopen(fd, 'w')
808 f = open(file_name, 'r')
811 # Ignore whitespace changes while comparing lines
812 if line.split() != key_fields:
816 os.rename(tmpname, file_name)
826 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
827 """Sets the name of an IP address and hostname in /etc/hosts.
830 @param file_name: path to the file to modify (usually C{/etc/hosts})
832 @param ip: the IP address
834 @param hostname: the hostname to be added
836 @param aliases: the list of aliases to add for the hostname
839 # Ensure aliases are unique
840 aliases = UniqueSequence([hostname] + aliases)[1:]
842 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
844 out = os.fdopen(fd, 'w')
846 f = open(file_name, 'r')
849 fields = line.split()
850 if fields and not fields[0].startswith('#') and ip == fields[0]:
854 out.write("%s\t%s" % (ip, hostname))
856 out.write(" %s" % ' '.join(aliases))
861 os.rename(tmpname, file_name)
871 def AddHostToEtcHosts(hostname):
872 """Wrapper around SetEtcHostsEntry.
875 @param hostname: a hostname that will be resolved and added to
876 L{constants.ETC_HOSTS}
879 hi = HostInfo(name=hostname)
880 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
883 def RemoveEtcHostsEntry(file_name, hostname):
884 """Removes a hostname from /etc/hosts.
886 IP addresses without names are removed from the file.
889 @param file_name: path to the file to modify (usually C{/etc/hosts})
891 @param hostname: the hostname to be removed
894 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
896 out = os.fdopen(fd, 'w')
898 f = open(file_name, 'r')
901 fields = line.split()
902 if len(fields) > 1 and not fields[0].startswith('#'):
904 if hostname in names:
905 while hostname in names:
906 names.remove(hostname)
908 out.write("%s %s\n" % (fields[0], ' '.join(names)))
915 os.rename(tmpname, file_name)
925 def RemoveHostFromEtcHosts(hostname):
926 """Wrapper around RemoveEtcHostsEntry.
929 @param hostname: hostname that will be resolved and its
930 full and shot name will be removed from
931 L{constants.ETC_HOSTS}
934 hi = HostInfo(name=hostname)
935 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
936 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
939 def CreateBackup(file_name):
940 """Creates a backup of a file.
943 @param file_name: file to be backed up
945 @return: the path to the newly created backup
946 @raise errors.ProgrammerError: for invalid file names
949 if not os.path.isfile(file_name):
950 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
953 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
954 dir_name = os.path.dirname(file_name)
956 fsrc = open(file_name, 'rb')
958 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
959 fdst = os.fdopen(fd, 'wb')
961 shutil.copyfileobj(fsrc, fdst)
970 def ShellQuote(value):
971 """Quotes shell argument according to POSIX.
974 @param value: the argument to be quoted
976 @return: the quoted value
979 if _re_shell_unquoted.match(value):
982 return "'%s'" % value.replace("'", "'\\''")
985 def ShellQuoteArgs(args):
986 """Quotes a list of shell arguments.
989 @param args: list of arguments to be quoted
991 @return: the quoted arguments concatenaned with spaces
994 return ' '.join([ShellQuote(i) for i in args])
997 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
998 """Simple ping implementation using TCP connect(2).
1000 Check if the given IP is reachable by doing attempting a TCP connect
1004 @param target: the IP or hostname to ping
1006 @param port: the port to connect to
1008 @param timeout: the timeout on the connection attemp
1009 @type live_port_needed: boolean
1010 @param live_port_needed: whether a closed port will cause the
1011 function to return failure, as if there was a timeout
1012 @type source: str or None
1013 @param source: if specified, will cause the connect to be made
1014 from this specific source address; failures to bind other
1015 than C{EADDRNOTAVAIL} will be ignored
1018 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1022 if source is not None:
1024 sock.bind((source, 0))
1025 except socket.error, (errcode, errstring):
1026 if errcode == errno.EADDRNOTAVAIL:
1029 sock.settimeout(timeout)
1032 sock.connect((target, port))
1035 except socket.timeout:
1037 except socket.error, (errcode, errstring):
1038 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1043 def OwnIpAddress(address):
1044 """Check if the current host has the the given IP address.
1046 Currently this is done by TCP-pinging the address from the loopback
1049 @type address: string
1050 @param address: the addres to check
1052 @return: True if we own the address
1055 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1056 source=constants.LOCALHOST_IP_ADDRESS)
1059 def ListVisibleFiles(path):
1060 """Returns a list of visible files in a directory.
1063 @param path: the directory to enumerate
1065 @return: the list of all files not starting with a dot
1068 files = [i for i in os.listdir(path) if not i.startswith(".")]
1073 def GetHomeDir(user, default=None):
1074 """Try to get the homedir of the given user.
1076 The user can be passed either as a string (denoting the name) or as
1077 an integer (denoting the user id). If the user is not found, the
1078 'default' argument is returned, which defaults to None.
1082 if isinstance(user, basestring):
1083 result = pwd.getpwnam(user)
1084 elif isinstance(user, (int, long)):
1085 result = pwd.getpwuid(user)
1087 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1091 return result.pw_dir
1095 """Returns a random UUID.
1097 @note: This is a Linux-specific method as it uses the /proc
1102 f = open("/proc/sys/kernel/random/uuid", "r")
1104 return f.read(128).rstrip("\n")
1109 def GenerateSecret():
1110 """Generates a random secret.
1112 This will generate a pseudo-random secret, and return its sha digest
1113 (so that it can be used where an ASCII string is needed).
1116 @return: a sha1 hexdigest of a block of 64 random bytes
1119 return sha.new(os.urandom(64)).hexdigest()
1122 def ReadFile(file_name, size=None):
1125 @type size: None or int
1126 @param size: Read at most size bytes
1128 @return: the (possibly partial) conent of the file
1131 f = open(file_name, "r")
1141 def WriteFile(file_name, fn=None, data=None,
1142 mode=None, uid=-1, gid=-1,
1143 atime=None, mtime=None, close=True,
1144 dry_run=False, backup=False,
1145 prewrite=None, postwrite=None):
1146 """(Over)write a file atomically.
1148 The file_name and either fn (a function taking one argument, the
1149 file descriptor, and which should write the data to it) or data (the
1150 contents of the file) must be passed. The other arguments are
1151 optional and allow setting the file mode, owner and group, and the
1152 mtime/atime of the file.
1154 If the function doesn't raise an exception, it has succeeded and the
1155 target file has the new contents. If the file has raised an
1156 exception, an existing target file should be unmodified and the
1157 temporary file should be removed.
1159 @type file_name: str
1160 @param file_name: the target filename
1162 @param fn: content writing function, called with
1163 file descriptor as parameter
1165 @param data: contents of the file
1167 @param mode: file mode
1169 @param uid: the owner of the file
1171 @param gid: the group of the file
1173 @param atime: a custom access time to be set on the file
1175 @param mtime: a custom modification time to be set on the file
1176 @type close: boolean
1177 @param close: whether to close file after writing it
1178 @type prewrite: callable
1179 @param prewrite: function to be called before writing content
1180 @type postwrite: callable
1181 @param postwrite: function to be called after writing content
1184 @return: None if the 'close' parameter evaluates to True,
1185 otherwise the file descriptor
1187 @raise errors.ProgrammerError: if an of the arguments are not valid
1190 if not os.path.isabs(file_name):
1191 raise errors.ProgrammerError("Path passed to WriteFile is not"
1192 " absolute: '%s'" % file_name)
1194 if [fn, data].count(None) != 1:
1195 raise errors.ProgrammerError("fn or data required")
1197 if [atime, mtime].count(None) == 1:
1198 raise errors.ProgrammerError("Both atime and mtime must be either"
1201 if backup and not dry_run and os.path.isfile(file_name):
1202 CreateBackup(file_name)
1204 dir_name, base_name = os.path.split(file_name)
1205 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1206 # here we need to make sure we remove the temp file, if any error
1207 # leaves it in place
1209 if uid != -1 or gid != -1:
1210 os.chown(new_name, uid, gid)
1212 os.chmod(new_name, mode)
1213 if callable(prewrite):
1215 if data is not None:
1219 if callable(postwrite):
1222 if atime is not None and mtime is not None:
1223 os.utime(new_name, (atime, mtime))
1225 os.rename(new_name, file_name)
1232 RemoveFile(new_name)
1237 def FirstFree(seq, base=0):
1238 """Returns the first non-existing integer from seq.
1240 The seq argument should be a sorted list of positive integers. The
1241 first time the index of an element is smaller than the element
1242 value, the index will be returned.
1244 The base argument is used to start at a different offset,
1245 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1247 Example: C{[0, 1, 3]} will return I{2}.
1250 @param seq: the sequence to be analyzed.
1252 @param base: use this value as the base index of the sequence
1254 @return: the first non-used index in the sequence
1257 for idx, elem in enumerate(seq):
1258 assert elem >= base, "Passed element is higher than base offset"
1259 if elem > idx + base:
1265 def all(seq, pred=bool):
1266 "Returns True if pred(x) is True for every element in the iterable"
1267 for elem in itertools.ifilterfalse(pred, seq):
1272 def any(seq, pred=bool):
1273 "Returns True if pred(x) is True for at least one element in the iterable"
1274 for elem in itertools.ifilter(pred, seq):
1279 def UniqueSequence(seq):
1280 """Returns a list with unique elements.
1282 Element order is preserved.
1285 @param seq: the sequence with the source elementes
1287 @return: list of unique elements from seq
1291 return [i for i in seq if i not in seen and not seen.add(i)]
1294 def IsValidMac(mac):
1295 """Predicate to check if a MAC address is valid.
1297 Checks wether the supplied MAC address is formally correct, only
1298 accepts colon separated format.
1301 @param mac: the MAC to be validated
1303 @return: True is the MAC seems valid
1306 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1307 return mac_check.match(mac) is not None
1310 def TestDelay(duration):
1311 """Sleep for a fixed amount of time.
1313 @type duration: float
1314 @param duration: the sleep duration
1316 @return: False for negative value, True otherwise
1321 time.sleep(duration)
1325 def Daemonize(logfile, noclose_fds=None):
1326 """Daemonize the current process.
1328 This detaches the current process from the controlling terminal and
1329 runs it in the background as a daemon.
1332 @param logfile: the logfile to which we should redirect stdout/stderr
1333 @type noclose_fds: list or None
1334 @param noclose_fds: if given, it denotes a list of file descriptor
1335 that should not be closed
1337 @returns: the value zero
1342 # Default maximum for the number of available file descriptors.
1343 if 'SC_OPEN_MAX' in os.sysconf_names:
1345 MAXFD = os.sysconf('SC_OPEN_MAX')
1355 if (pid == 0): # The first child.
1358 pid = os.fork() # Fork a second child.
1359 if (pid == 0): # The second child.
1363 # exit() or _exit()? See below.
1364 os._exit(0) # Exit parent (the first child) of the second child.
1366 os._exit(0) # Exit parent of the first child.
1367 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1368 if (maxfd == resource.RLIM_INFINITY):
1371 # Iterate through and close all file descriptors.
1372 for fd in range(0, maxfd):
1373 if noclose_fds and fd in noclose_fds:
1377 except OSError: # ERROR, fd wasn't open to begin with (ignored)
1379 os.open(logfile, os.O_RDWR|os.O_CREAT|os.O_APPEND, 0600)
1380 # Duplicate standard input to standard output and standard error.
1381 os.dup2(0, 1) # standard output (1)
1382 os.dup2(0, 2) # standard error (2)
1386 def DaemonPidFileName(name):
1387 """Compute a ganeti pid file absolute path
1390 @param name: the daemon name
1392 @return: the full path to the pidfile corresponding to the given
1396 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1399 def WritePidFile(name):
1400 """Write the current process pidfile.
1402 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1405 @param name: the daemon name to use
1406 @raise errors.GenericError: if the pid file already exists and
1407 points to a live process
1411 pidfilename = DaemonPidFileName(name)
1412 if IsProcessAlive(ReadPidFile(pidfilename)):
1413 raise errors.GenericError("%s contains a live process" % pidfilename)
1415 WriteFile(pidfilename, data="%d\n" % pid)
1418 def RemovePidFile(name):
1419 """Remove the current process pidfile.
1421 Any errors are ignored.
1424 @param name: the daemon name used to derive the pidfile name
1428 pidfilename = DaemonPidFileName(name)
1429 # TODO: we could check here that the file contains our pid
1431 RemoveFile(pidfilename)
1436 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1438 """Kill a process given by its pid.
1441 @param pid: The PID to terminate.
1443 @param signal_: The signal to send, by default SIGTERM
1445 @param timeout: The timeout after which, if the process is still alive,
1446 a SIGKILL will be sent. If not positive, no such checking
1448 @type waitpid: boolean
1449 @param waitpid: If true, we should waitpid on this process after
1450 sending signals, since it's our own child and otherwise it
1451 would remain as zombie
1454 def _helper(pid, signal_, wait):
1455 """Simple helper to encapsulate the kill/waitpid sequence"""
1456 os.kill(pid, signal_)
1459 os.waitpid(pid, os.WNOHANG)
1464 # kill with pid=0 == suicide
1465 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1467 if not IsProcessAlive(pid):
1469 _helper(pid, signal_, waitpid)
1473 # Wait up to $timeout seconds
1474 end = time.time() + timeout
1476 while time.time() < end and IsProcessAlive(pid):
1478 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1484 # Make wait time longer for next try
1488 if IsProcessAlive(pid):
1489 # Kill process if it's still alive
1490 _helper(pid, signal.SIGKILL, waitpid)
1493 def FindFile(name, search_path, test=os.path.exists):
1494 """Look for a filesystem object in a given path.
1496 This is an abstract method to search for filesystem object (files,
1497 dirs) under a given search path.
1500 @param name: the name to look for
1501 @type search_path: str
1502 @param search_path: location to start at
1503 @type test: callable
1504 @param test: a function taking one argument that should return True
1505 if the a given object is valid; the default value is
1506 os.path.exists, causing only existing files to be returned
1508 @return: full path to the object if found, None otherwise
1511 for dir_name in search_path:
1512 item_name = os.path.sep.join([dir_name, name])
1518 def CheckVolumeGroupSize(vglist, vgname, minsize):
1519 """Checks if the volume group list is valid.
1521 The function will check if a given volume group is in the list of
1522 volume groups and has a minimum size.
1525 @param vglist: dictionary of volume group names and their size
1527 @param vgname: the volume group we should check
1529 @param minsize: the minimum size we accept
1531 @return: None for success, otherwise the error message
1534 vgsize = vglist.get(vgname, None)
1536 return "volume group '%s' missing" % vgname
1537 elif vgsize < minsize:
1538 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1539 (vgname, minsize, vgsize))
1543 def SplitTime(value):
1544 """Splits time as floating point number into a tuple.
1546 @param value: Time in seconds
1547 @type value: int or float
1548 @return: Tuple containing (seconds, microseconds)
1551 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1553 assert 0 <= seconds, \
1554 "Seconds must be larger than or equal to 0, but are %s" % seconds
1555 assert 0 <= microseconds <= 999999, \
1556 "Microseconds must be 0-999999, but are %s" % microseconds
1558 return (int(seconds), int(microseconds))
1561 def MergeTime(timetuple):
1562 """Merges a tuple into time as a floating point number.
1564 @param timetuple: Time as tuple, (seconds, microseconds)
1565 @type timetuple: tuple
1566 @return: Time as a floating point number expressed in seconds
1569 (seconds, microseconds) = timetuple
1571 assert 0 <= seconds, \
1572 "Seconds must be larger than or equal to 0, but are %s" % seconds
1573 assert 0 <= microseconds <= 999999, \
1574 "Microseconds must be 0-999999, but are %s" % microseconds
1576 return float(seconds) + (float(microseconds) * 0.000001)
1579 def GetNodeDaemonPort():
1580 """Get the node daemon port for this cluster.
1582 Note that this routine does not read a ganeti-specific file, but
1583 instead uses C{socket.getservbyname} to allow pre-customization of
1584 this parameter outside of Ganeti.
1590 port = socket.getservbyname("ganeti-noded", "tcp")
1591 except socket.error:
1592 port = constants.DEFAULT_NODED_PORT
1597 def SetupLogging(logfile, debug=False, stderr_logging=False, program=""):
1598 """Configures the logging module.
1601 @param logfile: the filename to which we should log
1602 @type debug: boolean
1603 @param debug: whether to enable debug messages too or
1604 only those at C{INFO} and above level
1605 @type stderr_logging: boolean
1606 @param stderr_logging: whether we should also log to the standard error
1608 @param program: the name under which we should log messages
1609 @raise EnvironmentError: if we can't open the log file and
1610 stderr logging is disabled
1613 fmt = "%(asctime)s: " + program + " "
1615 fmt += ("pid=%(process)d/%(threadName)s %(levelname)s"
1616 " %(module)s:%(lineno)s %(message)s")
1618 fmt += "pid=%(process)d %(levelname)s %(message)s"
1619 formatter = logging.Formatter(fmt)
1621 root_logger = logging.getLogger("")
1622 root_logger.setLevel(logging.NOTSET)
1624 # Remove all previously setup handlers
1625 for handler in root_logger.handlers:
1626 root_logger.removeHandler(handler)
1629 stderr_handler = logging.StreamHandler()
1630 stderr_handler.setFormatter(formatter)
1632 stderr_handler.setLevel(logging.NOTSET)
1634 stderr_handler.setLevel(logging.CRITICAL)
1635 root_logger.addHandler(stderr_handler)
1637 # this can fail, if the logging directories are not setup or we have
1638 # a permisssion problem; in this case, it's best to log but ignore
1639 # the error if stderr_logging is True, and if false we re-raise the
1640 # exception since otherwise we could run but without any logs at all
1642 logfile_handler = logging.FileHandler(logfile)
1643 logfile_handler.setFormatter(formatter)
1645 logfile_handler.setLevel(logging.DEBUG)
1647 logfile_handler.setLevel(logging.INFO)
1648 root_logger.addHandler(logfile_handler)
1649 except EnvironmentError, err:
1651 logging.exception("Failed to enable logging to file '%s'", logfile)
1653 # we need to re-raise the exception
1657 def LockedMethod(fn):
1658 """Synchronized object access decorator.
1660 This decorator is intended to protect access to an object using the
1661 object's own lock which is hardcoded to '_lock'.
1664 def _LockDebug(*args, **kwargs):
1666 logging.debug(*args, **kwargs)
1668 def wrapper(self, *args, **kwargs):
1669 assert hasattr(self, '_lock')
1671 _LockDebug("Waiting for %s", lock)
1674 _LockDebug("Acquired %s", lock)
1675 result = fn(self, *args, **kwargs)
1677 _LockDebug("Releasing %s", lock)
1679 _LockDebug("Released %s", lock)
1685 """Locks a file using POSIX locks.
1688 @param fd: the file descriptor we need to lock
1692 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1693 except IOError, err:
1694 if err.errno == errno.EAGAIN:
1695 raise errors.LockError("File already locked")
1699 class FileLock(object):
1700 """Utility class for file locks.
1703 def __init__(self, filename):
1704 """Constructor for FileLock.
1706 This will open the file denoted by the I{filename} argument.
1709 @param filename: path to the file to be locked
1712 self.filename = filename
1713 self.fd = open(self.filename, "w")
1719 """Close the file and release the lock.
1726 def _flock(self, flag, blocking, timeout, errmsg):
1727 """Wrapper for fcntl.flock.
1730 @param flag: operation flag
1731 @type blocking: bool
1732 @param blocking: whether the operation should be done in blocking mode.
1733 @type timeout: None or float
1734 @param timeout: for how long the operation should be retried (implies
1736 @type errmsg: string
1737 @param errmsg: error message in case operation fails.
1740 assert self.fd, "Lock was closed"
1741 assert timeout is None or timeout >= 0, \
1742 "If specified, timeout must be positive"
1744 if timeout is not None:
1745 flag |= fcntl.LOCK_NB
1746 timeout_end = time.time() + timeout
1748 # Blocking doesn't have effect with timeout
1750 flag |= fcntl.LOCK_NB
1756 fcntl.flock(self.fd, flag)
1758 except IOError, err:
1759 if err.errno in (errno.EAGAIN, ):
1760 if timeout_end is not None and time.time() < timeout_end:
1761 # Wait before trying again
1762 time.sleep(max(0.1, min(1.0, timeout)))
1764 raise errors.LockError(errmsg)
1766 logging.exception("fcntl.flock failed")
1769 def Exclusive(self, blocking=False, timeout=None):
1770 """Locks the file in exclusive mode.
1772 @type blocking: boolean
1773 @param blocking: whether to block and wait until we
1774 can lock the file or return immediately
1775 @type timeout: int or None
1776 @param timeout: if not None, the duration to wait for the lock
1780 self._flock(fcntl.LOCK_EX, blocking, timeout,
1781 "Failed to lock %s in exclusive mode" % self.filename)
1783 def Shared(self, blocking=False, timeout=None):
1784 """Locks the file in shared mode.
1786 @type blocking: boolean
1787 @param blocking: whether to block and wait until we
1788 can lock the file or return immediately
1789 @type timeout: int or None
1790 @param timeout: if not None, the duration to wait for the lock
1794 self._flock(fcntl.LOCK_SH, blocking, timeout,
1795 "Failed to lock %s in shared mode" % self.filename)
1797 def Unlock(self, blocking=True, timeout=None):
1798 """Unlocks the file.
1800 According to C{flock(2)}, unlocking can also be a nonblocking
1803 To make a non-blocking request, include LOCK_NB with any of the above
1806 @type blocking: boolean
1807 @param blocking: whether to block and wait until we
1808 can lock the file or return immediately
1809 @type timeout: int or None
1810 @param timeout: if not None, the duration to wait for the lock
1814 self._flock(fcntl.LOCK_UN, blocking, timeout,
1815 "Failed to unlock %s" % self.filename)
1818 class SignalHandler(object):
1819 """Generic signal handler class.
1821 It automatically restores the original handler when deconstructed or
1822 when L{Reset} is called. You can either pass your own handler
1823 function in or query the L{called} attribute to detect whether the
1827 @ivar signum: the signals we handle
1828 @type called: boolean
1829 @ivar called: tracks whether any of the signals have been raised
1832 def __init__(self, signum):
1833 """Constructs a new SignalHandler instance.
1835 @type signum: int or list of ints
1836 @param signum: Single signal number or set of signal numbers
1839 if isinstance(signum, (int, long)):
1840 self.signum = set([signum])
1842 self.signum = set(signum)
1848 for signum in self.signum:
1850 prev_handler = signal.signal(signum, self._HandleSignal)
1852 self._previous[signum] = prev_handler
1854 # Restore previous handler
1855 signal.signal(signum, prev_handler)
1858 # Reset all handlers
1860 # Here we have a race condition: a handler may have already been called,
1861 # but there's not much we can do about it at this point.
1868 """Restore previous handler.
1870 This will reset all the signals to their previous handlers.
1873 for signum, prev_handler in self._previous.items():
1874 signal.signal(signum, prev_handler)
1875 # If successful, remove from dict
1876 del self._previous[signum]
1879 """Unsets the L{called} flag.
1881 This function can be used in case a signal may arrive several times.
1886 def _HandleSignal(self, signum, frame):
1887 """Actual signal handling function.
1890 # This is not nice and not absolutely atomic, but it appears to be the only
1891 # solution in Python -- there are no atomic types.
1895 class FieldSet(object):
1896 """A simple field set.
1898 Among the features are:
1899 - checking if a string is among a list of static string or regex objects
1900 - checking if a whole list of string matches
1901 - returning the matching groups from a regex match
1903 Internally, all fields are held as regular expression objects.
1906 def __init__(self, *items):
1907 self.items = [re.compile("^%s$" % value) for value in items]
1909 def Extend(self, other_set):
1910 """Extend the field set with the items from another one"""
1911 self.items.extend(other_set.items)
1913 def Matches(self, field):
1914 """Checks if a field matches the current set
1917 @param field: the string to match
1918 @return: either False or a regular expression match object
1921 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
1925 def NonMatching(self, items):
1926 """Returns the list of fields not matching the current set
1929 @param items: the list of fields to check
1931 @return: list of non-matching fields
1934 return [val for val in items if not self.Matches(val)]