4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
47 from cStringIO import StringIO
50 from hashlib import sha1
55 from ganeti import errors
56 from ganeti import constants
60 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
65 #: when set to True, L{RunCmd} is disabled
69 class RunResult(object):
70 """Holds the result of running external programs.
73 @ivar exit_code: the exit code of the program, or None (if the program
75 @type signal: int or None
76 @ivar signal: the signal that caused the program to finish, or None
77 (if the program wasn't terminated by a signal)
79 @ivar stdout: the standard output of the program
81 @ivar stderr: the standard error of the program
83 @ivar failed: True in case the program was
84 terminated by a signal or exited with a non-zero exit code
85 @ivar fail_reason: a string detailing the termination reason
88 __slots__ = ["exit_code", "signal", "stdout", "stderr",
89 "failed", "fail_reason", "cmd"]
92 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
94 self.exit_code = exit_code
98 self.failed = (signal_ is not None or exit_code != 0)
100 if self.signal is not None:
101 self.fail_reason = "terminated by signal %s" % self.signal
102 elif self.exit_code is not None:
103 self.fail_reason = "exited with exit code %s" % self.exit_code
105 self.fail_reason = "unable to determine termination reason"
108 logging.debug("Command '%s' failed (%s); output: %s",
109 self.cmd, self.fail_reason, self.output)
111 def _GetOutput(self):
112 """Returns the combined stdout and stderr for easier usage.
115 return self.stdout + self.stderr
117 output = property(_GetOutput, None, None, "Return full output")
120 def RunCmd(cmd, env=None, output=None, cwd='/'):
121 """Execute a (shell) command.
123 The command should not read from its standard input, as it will be
126 @type cmd: string or list
127 @param cmd: Command to run
129 @param env: Additional environment
131 @param output: if desired, the output of the command can be
132 saved in a file instead of the RunResult instance; this
133 parameter denotes the file name (if not None)
135 @param cwd: if specified, will be used as the working
136 directory for the command; the default will be /
138 @return: RunResult instance
139 @raise erors.ProgrammerError: if we call this when forks are disabled
143 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
145 if isinstance(cmd, list):
146 cmd = [str(val) for val in cmd]
147 strcmd = " ".join(cmd)
152 logging.debug("RunCmd '%s'", strcmd)
154 cmd_env = os.environ.copy()
155 cmd_env["LC_ALL"] = "C"
160 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
162 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
172 return RunResult(exitcode, signal_, out, err, strcmd)
175 def _RunCmdPipe(cmd, env, via_shell, cwd):
176 """Run a command and return its output.
178 @type cmd: string or list
179 @param cmd: Command to run
181 @param env: The environment to use
182 @type via_shell: bool
183 @param via_shell: if we should run via the shell
185 @param cwd: the working directory for the program
187 @return: (out, err, status)
190 poller = select.poll()
191 child = subprocess.Popen(cmd, shell=via_shell,
192 stderr=subprocess.PIPE,
193 stdout=subprocess.PIPE,
194 stdin=subprocess.PIPE,
195 close_fds=True, env=env,
199 poller.register(child.stdout, select.POLLIN)
200 poller.register(child.stderr, select.POLLIN)
204 child.stdout.fileno(): (out, child.stdout),
205 child.stderr.fileno(): (err, child.stderr),
208 status = fcntl.fcntl(fd, fcntl.F_GETFL)
209 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
213 pollresult = poller.poll()
214 except EnvironmentError, eerr:
215 if eerr.errno == errno.EINTR:
218 except select.error, serr:
219 if serr[0] == errno.EINTR:
223 for fd, event in pollresult:
224 if event & select.POLLIN or event & select.POLLPRI:
225 data = fdmap[fd][1].read()
226 # no data from read signifies EOF (the same as POLLHUP)
228 poller.unregister(fd)
231 fdmap[fd][0].write(data)
232 if (event & select.POLLNVAL or event & select.POLLHUP or
233 event & select.POLLERR):
234 poller.unregister(fd)
240 status = child.wait()
241 return out, err, status
244 def _RunCmdFile(cmd, env, via_shell, output, cwd):
245 """Run a command and save its output to a file.
247 @type cmd: string or list
248 @param cmd: Command to run
250 @param env: The environment to use
251 @type via_shell: bool
252 @param via_shell: if we should run via the shell
254 @param output: the filename in which to save the output
256 @param cwd: the working directory for the program
258 @return: the exit status
261 fh = open(output, "a")
263 child = subprocess.Popen(cmd, shell=via_shell,
264 stderr=subprocess.STDOUT,
266 stdin=subprocess.PIPE,
267 close_fds=True, env=env,
271 status = child.wait()
277 def RemoveFile(filename):
278 """Remove a file ignoring some errors.
280 Remove a file, ignoring non-existing ones or directories. Other
284 @param filename: the file to be removed
290 if err.errno not in (errno.ENOENT, errno.EISDIR):
294 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
298 @param old: Original path
302 @param mkdir: Whether to create target directory if it doesn't exist
303 @type mkdir_mode: int
304 @param mkdir_mode: Mode for newly created directories
308 return os.rename(old, new)
310 # In at least one use case of this function, the job queue, directory
311 # creation is very rare. Checking for the directory before renaming is not
313 if mkdir and err.errno == errno.ENOENT:
314 # Create directory and try again
315 os.makedirs(os.path.dirname(new), mkdir_mode)
316 return os.rename(old, new)
320 def _FingerprintFile(filename):
321 """Compute the fingerprint of a file.
323 If the file does not exist, a None will be returned
327 @param filename: the filename to checksum
329 @return: the hex digest of the sha checksum of the contents
333 if not (os.path.exists(filename) and os.path.isfile(filename)):
346 return fp.hexdigest()
349 def FingerprintFiles(files):
350 """Compute fingerprints for a list of files.
353 @param files: the list of filename to fingerprint
355 @return: a dictionary filename: fingerprint, holding only
361 for filename in files:
362 cksum = _FingerprintFile(filename)
364 ret[filename] = cksum
369 def CheckDict(target, template, logname=None):
370 """Ensure a dictionary has a required set of keys.
372 For the given dictionaries I{target} and I{template}, ensure
373 I{target} has all the keys from I{template}. Missing keys are added
374 with values from template.
377 @param target: the dictionary to update
379 @param template: the dictionary holding the default values
380 @type logname: str or None
381 @param logname: if not None, causes the missing keys to be
382 logged with this name
389 target[k] = template[k]
391 if missing and logname:
392 logging.warning('%s missing keys %s', logname, ', '.join(missing))
395 def ForceDictType(target, key_types, allowed_values=None):
396 """Force the values of a dict to have certain types.
399 @param target: the dict to update
400 @type key_types: dict
401 @param key_types: dict mapping target dict keys to types
402 in constants.ENFORCEABLE_TYPES
403 @type allowed_values: list
404 @keyword allowed_values: list of specially allowed values
407 if allowed_values is None:
411 if key not in key_types:
412 msg = "Unknown key '%s'" % key
413 raise errors.TypeEnforcementError(msg)
415 if target[key] in allowed_values:
418 type = key_types[key]
419 if type not in constants.ENFORCEABLE_TYPES:
420 msg = "'%s' has non-enforceable type %s" % (key, type)
421 raise errors.ProgrammerError(msg)
423 if type == constants.VTYPE_STRING:
424 if not isinstance(target[key], basestring):
425 if isinstance(target[key], bool) and not target[key]:
428 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
429 raise errors.TypeEnforcementError(msg)
430 elif type == constants.VTYPE_BOOL:
431 if isinstance(target[key], basestring) and target[key]:
432 if target[key].lower() == constants.VALUE_FALSE:
434 elif target[key].lower() == constants.VALUE_TRUE:
437 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
438 raise errors.TypeEnforcementError(msg)
443 elif type == constants.VTYPE_SIZE:
445 target[key] = ParseUnit(target[key])
446 except errors.UnitParseError, err:
447 msg = "'%s' (value %s) is not a valid size. error: %s" % \
448 (key, target[key], err)
449 raise errors.TypeEnforcementError(msg)
450 elif type == constants.VTYPE_INT:
452 target[key] = int(target[key])
453 except (ValueError, TypeError):
454 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
455 raise errors.TypeEnforcementError(msg)
458 def IsProcessAlive(pid):
459 """Check if a given pid exists on the system.
461 @note: zombie status is not handled, so zombie processes
462 will be returned as alive
464 @param pid: the process ID to check
466 @return: True if the process exists
473 os.stat("/proc/%d/status" % pid)
475 except EnvironmentError, err:
476 if err.errno in (errno.ENOENT, errno.ENOTDIR):
481 def ReadPidFile(pidfile):
482 """Read a pid from a file.
484 @type pidfile: string
485 @param pidfile: path to the file containing the pid
487 @return: The process id, if the file exists and contains a valid PID,
492 pf = open(pidfile, 'r')
493 except EnvironmentError, err:
494 if err.errno != errno.ENOENT:
495 logging.exception("Can't read pid file?!")
500 except ValueError, err:
501 logging.info("Can't parse pid file contents", exc_info=True)
507 def MatchNameComponent(key, name_list):
508 """Try to match a name against a list.
510 This function will try to match a name like test1 against a list
511 like C{['test1.example.com', 'test2.example.com', ...]}. Against
512 this list, I{'test1'} as well as I{'test1.example'} will match, but
513 not I{'test1.ex'}. A multiple match will be considered as no match
514 at all (e.g. I{'test1'} against C{['test1.example.com',
515 'test1.example.org']}).
518 @param key: the name to be searched
519 @type name_list: list
520 @param name_list: the list of strings against which to search the key
523 @return: None if there is no match I{or} if there are multiple matches,
524 otherwise the element from the list which matches
527 mo = re.compile("^%s(\..*)?$" % re.escape(key))
528 names_filtered = [name for name in name_list if mo.match(name) is not None]
529 if len(names_filtered) != 1:
531 return names_filtered[0]
535 """Class implementing resolver and hostname functionality
538 def __init__(self, name=None):
539 """Initialize the host name object.
541 If the name argument is not passed, it will use this system's
546 name = self.SysName()
549 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
550 self.ip = self.ipaddrs[0]
553 """Returns the hostname without domain.
556 return self.name.split('.')[0]
560 """Return the current system's name.
562 This is simply a wrapper over C{socket.gethostname()}.
565 return socket.gethostname()
568 def LookupHostname(hostname):
572 @param hostname: hostname to look up
575 @return: a tuple (name, aliases, ipaddrs) as returned by
576 C{socket.gethostbyname_ex}
577 @raise errors.ResolverError: in case of errors in resolving
581 result = socket.gethostbyname_ex(hostname)
582 except socket.gaierror, err:
583 # hostname not found in DNS
584 raise errors.ResolverError(hostname, err.args[0], err.args[1])
589 def ListVolumeGroups():
590 """List volume groups and their size
594 Dictionary with keys volume name and values
595 the size of the volume
598 command = "vgs --noheadings --units m --nosuffix -o name,size"
599 result = RunCmd(command)
604 for line in result.stdout.splitlines():
606 name, size = line.split()
607 size = int(float(size))
608 except (IndexError, ValueError), err:
609 logging.error("Invalid output from vgs (%s): %s", err, line)
617 def BridgeExists(bridge):
618 """Check whether the given bridge exists in the system
621 @param bridge: the bridge name to check
623 @return: True if it does
626 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
629 def NiceSort(name_list):
630 """Sort a list of strings based on digit and non-digit groupings.
632 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
633 will sort the list in the logical order C{['a1', 'a2', 'a10',
636 The sort algorithm breaks each name in groups of either only-digits
637 or no-digits. Only the first eight such groups are considered, and
638 after that we just use what's left of the string.
640 @type name_list: list
641 @param name_list: the names to be sorted
643 @return: a copy of the name list sorted with our algorithm
646 _SORTER_BASE = "(\D+|\d+)"
647 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
648 _SORTER_BASE, _SORTER_BASE,
649 _SORTER_BASE, _SORTER_BASE,
650 _SORTER_BASE, _SORTER_BASE)
651 _SORTER_RE = re.compile(_SORTER_FULL)
652 _SORTER_NODIGIT = re.compile("^\D*$")
654 """Attempts to convert a variable to integer."""
655 if val is None or _SORTER_NODIGIT.match(val):
660 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
661 for name in name_list]
663 return [tup[1] for tup in to_sort]
666 def TryConvert(fn, val):
667 """Try to convert a value ignoring errors.
669 This function tries to apply function I{fn} to I{val}. If no
670 C{ValueError} or C{TypeError} exceptions are raised, it will return
671 the result, else it will return the original value. Any other
672 exceptions are propagated to the caller.
675 @param fn: function to apply to the value
676 @param val: the value to be converted
677 @return: The converted value if the conversion was successful,
678 otherwise the original value.
683 except (ValueError, TypeError), err:
689 """Verifies the syntax of an IPv4 address.
691 This function checks if the IPv4 address passes is valid or not based
692 on syntax (not IP range, class calculations, etc.).
695 @param ip: the address to be checked
696 @rtype: a regular expression match object
697 @return: a regular epression match object, or None if the
701 unit = "(0|[1-9]\d{0,2})"
702 #TODO: convert and return only boolean
703 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
706 def IsValidShellParam(word):
707 """Verifies is the given word is safe from the shell's p.o.v.
709 This means that we can pass this to a command via the shell and be
710 sure that it doesn't alter the command line and is passed as such to
713 Note that we are overly restrictive here, in order to be on the safe
717 @param word: the word to check
719 @return: True if the word is 'safe'
722 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
725 def BuildShellCmd(template, *args):
726 """Build a safe shell command line from the given arguments.
728 This function will check all arguments in the args list so that they
729 are valid shell parameters (i.e. they don't contain shell
730 metacharaters). If everything is ok, it will return the result of
734 @param template: the string holding the template for the
737 @return: the expanded command line
741 if not IsValidShellParam(word):
742 raise errors.ProgrammerError("Shell argument '%s' contains"
743 " invalid characters" % word)
744 return template % args
747 def FormatUnit(value, units):
748 """Formats an incoming number of MiB with the appropriate unit.
751 @param value: integer representing the value in MiB (1048576)
753 @param units: the type of formatting we should do:
754 - 'h' for automatic scaling
759 @return: the formatted value (with suffix)
762 if units not in ('m', 'g', 't', 'h'):
763 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
767 if units == 'm' or (units == 'h' and value < 1024):
770 return "%d%s" % (round(value, 0), suffix)
772 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
775 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
780 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
783 def ParseUnit(input_string):
784 """Tries to extract number and scale from the given string.
786 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
787 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
788 is always an int in MiB.
791 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
793 raise errors.UnitParseError("Invalid format")
795 value = float(m.groups()[0])
799 lcunit = unit.lower()
803 if lcunit in ('m', 'mb', 'mib'):
804 # Value already in MiB
807 elif lcunit in ('g', 'gb', 'gib'):
810 elif lcunit in ('t', 'tb', 'tib'):
814 raise errors.UnitParseError("Unknown unit: %s" % unit)
816 # Make sure we round up
817 if int(value) < value:
820 # Round up to the next multiple of 4
823 value += 4 - value % 4
828 def AddAuthorizedKey(file_name, key):
829 """Adds an SSH public key to an authorized_keys file.
832 @param file_name: path to authorized_keys file
834 @param key: string containing key
837 key_fields = key.split()
839 f = open(file_name, 'a+')
843 # Ignore whitespace changes
844 if line.split() == key_fields:
846 nl = line.endswith('\n')
850 f.write(key.rstrip('\r\n'))
857 def RemoveAuthorizedKey(file_name, key):
858 """Removes an SSH public key from an authorized_keys file.
861 @param file_name: path to authorized_keys file
863 @param key: string containing key
866 key_fields = key.split()
868 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
870 out = os.fdopen(fd, 'w')
872 f = open(file_name, 'r')
875 # Ignore whitespace changes while comparing lines
876 if line.split() != key_fields:
880 os.rename(tmpname, file_name)
890 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
891 """Sets the name of an IP address and hostname in /etc/hosts.
894 @param file_name: path to the file to modify (usually C{/etc/hosts})
896 @param ip: the IP address
898 @param hostname: the hostname to be added
900 @param aliases: the list of aliases to add for the hostname
903 # FIXME: use WriteFile + fn rather than duplicating its efforts
904 # Ensure aliases are unique
905 aliases = UniqueSequence([hostname] + aliases)[1:]
907 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
909 out = os.fdopen(fd, 'w')
911 f = open(file_name, 'r')
914 fields = line.split()
915 if fields and not fields[0].startswith('#') and ip == fields[0]:
919 out.write("%s\t%s" % (ip, hostname))
921 out.write(" %s" % ' '.join(aliases))
926 os.chmod(tmpname, 0644)
927 os.rename(tmpname, file_name)
937 def AddHostToEtcHosts(hostname):
938 """Wrapper around SetEtcHostsEntry.
941 @param hostname: a hostname that will be resolved and added to
942 L{constants.ETC_HOSTS}
945 hi = HostInfo(name=hostname)
946 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
949 def RemoveEtcHostsEntry(file_name, hostname):
950 """Removes a hostname from /etc/hosts.
952 IP addresses without names are removed from the file.
955 @param file_name: path to the file to modify (usually C{/etc/hosts})
957 @param hostname: the hostname to be removed
960 # FIXME: use WriteFile + fn rather than duplicating its efforts
961 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
963 out = os.fdopen(fd, 'w')
965 f = open(file_name, 'r')
968 fields = line.split()
969 if len(fields) > 1 and not fields[0].startswith('#'):
971 if hostname in names:
972 while hostname in names:
973 names.remove(hostname)
975 out.write("%s %s\n" % (fields[0], ' '.join(names)))
982 os.chmod(tmpname, 0644)
983 os.rename(tmpname, file_name)
993 def RemoveHostFromEtcHosts(hostname):
994 """Wrapper around RemoveEtcHostsEntry.
997 @param hostname: hostname that will be resolved and its
998 full and shot name will be removed from
999 L{constants.ETC_HOSTS}
1002 hi = HostInfo(name=hostname)
1003 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1004 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1007 def CreateBackup(file_name):
1008 """Creates a backup of a file.
1010 @type file_name: str
1011 @param file_name: file to be backed up
1013 @return: the path to the newly created backup
1014 @raise errors.ProgrammerError: for invalid file names
1017 if not os.path.isfile(file_name):
1018 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1021 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1022 dir_name = os.path.dirname(file_name)
1024 fsrc = open(file_name, 'rb')
1026 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1027 fdst = os.fdopen(fd, 'wb')
1029 shutil.copyfileobj(fsrc, fdst)
1038 def ShellQuote(value):
1039 """Quotes shell argument according to POSIX.
1042 @param value: the argument to be quoted
1044 @return: the quoted value
1047 if _re_shell_unquoted.match(value):
1050 return "'%s'" % value.replace("'", "'\\''")
1053 def ShellQuoteArgs(args):
1054 """Quotes a list of shell arguments.
1057 @param args: list of arguments to be quoted
1059 @return: the quoted arguments concatenaned with spaces
1062 return ' '.join([ShellQuote(i) for i in args])
1065 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1066 """Simple ping implementation using TCP connect(2).
1068 Check if the given IP is reachable by doing attempting a TCP connect
1072 @param target: the IP or hostname to ping
1074 @param port: the port to connect to
1076 @param timeout: the timeout on the connection attemp
1077 @type live_port_needed: boolean
1078 @param live_port_needed: whether a closed port will cause the
1079 function to return failure, as if there was a timeout
1080 @type source: str or None
1081 @param source: if specified, will cause the connect to be made
1082 from this specific source address; failures to bind other
1083 than C{EADDRNOTAVAIL} will be ignored
1086 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1090 if source is not None:
1092 sock.bind((source, 0))
1093 except socket.error, (errcode, errstring):
1094 if errcode == errno.EADDRNOTAVAIL:
1097 sock.settimeout(timeout)
1100 sock.connect((target, port))
1103 except socket.timeout:
1105 except socket.error, (errcode, errstring):
1106 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1111 def OwnIpAddress(address):
1112 """Check if the current host has the the given IP address.
1114 Currently this is done by TCP-pinging the address from the loopback
1117 @type address: string
1118 @param address: the addres to check
1120 @return: True if we own the address
1123 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1124 source=constants.LOCALHOST_IP_ADDRESS)
1127 def ListVisibleFiles(path):
1128 """Returns a list of visible files in a directory.
1131 @param path: the directory to enumerate
1133 @return: the list of all files not starting with a dot
1136 files = [i for i in os.listdir(path) if not i.startswith(".")]
1141 def GetHomeDir(user, default=None):
1142 """Try to get the homedir of the given user.
1144 The user can be passed either as a string (denoting the name) or as
1145 an integer (denoting the user id). If the user is not found, the
1146 'default' argument is returned, which defaults to None.
1150 if isinstance(user, basestring):
1151 result = pwd.getpwnam(user)
1152 elif isinstance(user, (int, long)):
1153 result = pwd.getpwuid(user)
1155 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1159 return result.pw_dir
1163 """Returns a random UUID.
1165 @note: This is a Linux-specific method as it uses the /proc
1170 f = open("/proc/sys/kernel/random/uuid", "r")
1172 return f.read(128).rstrip("\n")
1177 def GenerateSecret():
1178 """Generates a random secret.
1180 This will generate a pseudo-random secret, and return its sha digest
1181 (so that it can be used where an ASCII string is needed).
1184 @return: a sha1 hexdigest of a block of 64 random bytes
1187 return sha1(os.urandom(64)).hexdigest()
1190 def EnsureDirs(dirs):
1191 """Make required directories, if they don't exist.
1193 @param dirs: list of tuples (dir_name, dir_mode)
1194 @type dirs: list of (string, integer)
1197 for dir_name, dir_mode in dirs:
1199 os.mkdir(dir_name, dir_mode)
1200 except EnvironmentError, err:
1201 if err.errno != errno.EEXIST:
1202 raise errors.GenericError("Cannot create needed directory"
1203 " '%s': %s" % (dir_name, err))
1204 if not os.path.isdir(dir_name):
1205 raise errors.GenericError("%s is not a directory" % dir_name)
1208 def ReadFile(file_name, size=None):
1211 @type size: None or int
1212 @param size: Read at most size bytes
1214 @return: the (possibly partial) conent of the file
1217 f = open(file_name, "r")
1227 def WriteFile(file_name, fn=None, data=None,
1228 mode=None, uid=-1, gid=-1,
1229 atime=None, mtime=None, close=True,
1230 dry_run=False, backup=False,
1231 prewrite=None, postwrite=None):
1232 """(Over)write a file atomically.
1234 The file_name and either fn (a function taking one argument, the
1235 file descriptor, and which should write the data to it) or data (the
1236 contents of the file) must be passed. The other arguments are
1237 optional and allow setting the file mode, owner and group, and the
1238 mtime/atime of the file.
1240 If the function doesn't raise an exception, it has succeeded and the
1241 target file has the new contents. If the function has raised an
1242 exception, an existing target file should be unmodified and the
1243 temporary file should be removed.
1245 @type file_name: str
1246 @param file_name: the target filename
1248 @param fn: content writing function, called with
1249 file descriptor as parameter
1251 @param data: contents of the file
1253 @param mode: file mode
1255 @param uid: the owner of the file
1257 @param gid: the group of the file
1259 @param atime: a custom access time to be set on the file
1261 @param mtime: a custom modification time to be set on the file
1262 @type close: boolean
1263 @param close: whether to close file after writing it
1264 @type prewrite: callable
1265 @param prewrite: function to be called before writing content
1266 @type postwrite: callable
1267 @param postwrite: function to be called after writing content
1270 @return: None if the 'close' parameter evaluates to True,
1271 otherwise the file descriptor
1273 @raise errors.ProgrammerError: if any of the arguments are not valid
1276 if not os.path.isabs(file_name):
1277 raise errors.ProgrammerError("Path passed to WriteFile is not"
1278 " absolute: '%s'" % file_name)
1280 if [fn, data].count(None) != 1:
1281 raise errors.ProgrammerError("fn or data required")
1283 if [atime, mtime].count(None) == 1:
1284 raise errors.ProgrammerError("Both atime and mtime must be either"
1287 if backup and not dry_run and os.path.isfile(file_name):
1288 CreateBackup(file_name)
1290 dir_name, base_name = os.path.split(file_name)
1291 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1292 # here we need to make sure we remove the temp file, if any error
1293 # leaves it in place
1295 if uid != -1 or gid != -1:
1296 os.chown(new_name, uid, gid)
1298 os.chmod(new_name, mode)
1299 if callable(prewrite):
1301 if data is not None:
1305 if callable(postwrite):
1308 if atime is not None and mtime is not None:
1309 os.utime(new_name, (atime, mtime))
1311 os.rename(new_name, file_name)
1318 RemoveFile(new_name)
1323 def FirstFree(seq, base=0):
1324 """Returns the first non-existing integer from seq.
1326 The seq argument should be a sorted list of positive integers. The
1327 first time the index of an element is smaller than the element
1328 value, the index will be returned.
1330 The base argument is used to start at a different offset,
1331 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1333 Example: C{[0, 1, 3]} will return I{2}.
1336 @param seq: the sequence to be analyzed.
1338 @param base: use this value as the base index of the sequence
1340 @return: the first non-used index in the sequence
1343 for idx, elem in enumerate(seq):
1344 assert elem >= base, "Passed element is higher than base offset"
1345 if elem > idx + base:
1351 def all(seq, pred=bool):
1352 "Returns True if pred(x) is True for every element in the iterable"
1353 for elem in itertools.ifilterfalse(pred, seq):
1358 def any(seq, pred=bool):
1359 "Returns True if pred(x) is True for at least one element in the iterable"
1360 for elem in itertools.ifilter(pred, seq):
1365 def UniqueSequence(seq):
1366 """Returns a list with unique elements.
1368 Element order is preserved.
1371 @param seq: the sequence with the source elementes
1373 @return: list of unique elements from seq
1377 return [i for i in seq if i not in seen and not seen.add(i)]
1380 def IsValidMac(mac):
1381 """Predicate to check if a MAC address is valid.
1383 Checks wether the supplied MAC address is formally correct, only
1384 accepts colon separated format.
1387 @param mac: the MAC to be validated
1389 @return: True is the MAC seems valid
1392 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1393 return mac_check.match(mac) is not None
1396 def TestDelay(duration):
1397 """Sleep for a fixed amount of time.
1399 @type duration: float
1400 @param duration: the sleep duration
1402 @return: False for negative value, True otherwise
1407 time.sleep(duration)
1411 def _CloseFDNoErr(fd, retries=5):
1412 """Close a file descriptor ignoring errors.
1415 @param fd: the file descriptor
1417 @param retries: how many retries to make, in case we get any
1418 other error than EBADF
1423 except OSError, err:
1424 if err.errno != errno.EBADF:
1426 _CloseFDNoErr(fd, retries - 1)
1427 # else either it's closed already or we're out of retries, so we
1428 # ignore this and go on
1431 def CloseFDs(noclose_fds=None):
1432 """Close file descriptors.
1434 This closes all file descriptors above 2 (i.e. except
1437 @type noclose_fds: list or None
1438 @param noclose_fds: if given, it denotes a list of file descriptor
1439 that should not be closed
1442 # Default maximum for the number of available file descriptors.
1443 if 'SC_OPEN_MAX' in os.sysconf_names:
1445 MAXFD = os.sysconf('SC_OPEN_MAX')
1452 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1453 if (maxfd == resource.RLIM_INFINITY):
1456 # Iterate through and close all file descriptors (except the standard ones)
1457 for fd in range(3, maxfd):
1458 if noclose_fds and fd in noclose_fds:
1463 def Daemonize(logfile):
1464 """Daemonize the current process.
1466 This detaches the current process from the controlling terminal and
1467 runs it in the background as a daemon.
1470 @param logfile: the logfile to which we should redirect stdout/stderr
1472 @return: the value zero
1480 if (pid == 0): # The first child.
1483 pid = os.fork() # Fork a second child.
1484 if (pid == 0): # The second child.
1488 # exit() or _exit()? See below.
1489 os._exit(0) # Exit parent (the first child) of the second child.
1491 os._exit(0) # Exit parent of the first child.
1495 i = os.open("/dev/null", os.O_RDONLY) # stdin
1496 assert i == 0, "Can't close/reopen stdin"
1497 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1498 assert i == 1, "Can't close/reopen stdout"
1499 # Duplicate standard output to standard error.
1504 def DaemonPidFileName(name):
1505 """Compute a ganeti pid file absolute path
1508 @param name: the daemon name
1510 @return: the full path to the pidfile corresponding to the given
1514 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1517 def WritePidFile(name):
1518 """Write the current process pidfile.
1520 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1523 @param name: the daemon name to use
1524 @raise errors.GenericError: if the pid file already exists and
1525 points to a live process
1529 pidfilename = DaemonPidFileName(name)
1530 if IsProcessAlive(ReadPidFile(pidfilename)):
1531 raise errors.GenericError("%s contains a live process" % pidfilename)
1533 WriteFile(pidfilename, data="%d\n" % pid)
1536 def RemovePidFile(name):
1537 """Remove the current process pidfile.
1539 Any errors are ignored.
1542 @param name: the daemon name used to derive the pidfile name
1546 pidfilename = DaemonPidFileName(name)
1547 # TODO: we could check here that the file contains our pid
1549 RemoveFile(pidfilename)
1554 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1556 """Kill a process given by its pid.
1559 @param pid: The PID to terminate.
1561 @param signal_: The signal to send, by default SIGTERM
1563 @param timeout: The timeout after which, if the process is still alive,
1564 a SIGKILL will be sent. If not positive, no such checking
1566 @type waitpid: boolean
1567 @param waitpid: If true, we should waitpid on this process after
1568 sending signals, since it's our own child and otherwise it
1569 would remain as zombie
1572 def _helper(pid, signal_, wait):
1573 """Simple helper to encapsulate the kill/waitpid sequence"""
1574 os.kill(pid, signal_)
1577 os.waitpid(pid, os.WNOHANG)
1582 # kill with pid=0 == suicide
1583 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1585 if not IsProcessAlive(pid):
1587 _helper(pid, signal_, waitpid)
1591 # Wait up to $timeout seconds
1592 end = time.time() + timeout
1594 while time.time() < end and IsProcessAlive(pid):
1596 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1602 # Make wait time longer for next try
1606 if IsProcessAlive(pid):
1607 # Kill process if it's still alive
1608 _helper(pid, signal.SIGKILL, waitpid)
1611 def FindFile(name, search_path, test=os.path.exists):
1612 """Look for a filesystem object in a given path.
1614 This is an abstract method to search for filesystem object (files,
1615 dirs) under a given search path.
1618 @param name: the name to look for
1619 @type search_path: str
1620 @param search_path: location to start at
1621 @type test: callable
1622 @param test: a function taking one argument that should return True
1623 if the a given object is valid; the default value is
1624 os.path.exists, causing only existing files to be returned
1626 @return: full path to the object if found, None otherwise
1629 for dir_name in search_path:
1630 item_name = os.path.sep.join([dir_name, name])
1636 def CheckVolumeGroupSize(vglist, vgname, minsize):
1637 """Checks if the volume group list is valid.
1639 The function will check if a given volume group is in the list of
1640 volume groups and has a minimum size.
1643 @param vglist: dictionary of volume group names and their size
1645 @param vgname: the volume group we should check
1647 @param minsize: the minimum size we accept
1649 @return: None for success, otherwise the error message
1652 vgsize = vglist.get(vgname, None)
1654 return "volume group '%s' missing" % vgname
1655 elif vgsize < minsize:
1656 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1657 (vgname, minsize, vgsize))
1661 def SplitTime(value):
1662 """Splits time as floating point number into a tuple.
1664 @param value: Time in seconds
1665 @type value: int or float
1666 @return: Tuple containing (seconds, microseconds)
1669 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1671 assert 0 <= seconds, \
1672 "Seconds must be larger than or equal to 0, but are %s" % seconds
1673 assert 0 <= microseconds <= 999999, \
1674 "Microseconds must be 0-999999, but are %s" % microseconds
1676 return (int(seconds), int(microseconds))
1679 def MergeTime(timetuple):
1680 """Merges a tuple into time as a floating point number.
1682 @param timetuple: Time as tuple, (seconds, microseconds)
1683 @type timetuple: tuple
1684 @return: Time as a floating point number expressed in seconds
1687 (seconds, microseconds) = timetuple
1689 assert 0 <= seconds, \
1690 "Seconds must be larger than or equal to 0, but are %s" % seconds
1691 assert 0 <= microseconds <= 999999, \
1692 "Microseconds must be 0-999999, but are %s" % microseconds
1694 return float(seconds) + (float(microseconds) * 0.000001)
1697 def GetNodeDaemonPort():
1698 """Get the node daemon port for this cluster.
1700 Note that this routine does not read a ganeti-specific file, but
1701 instead uses C{socket.getservbyname} to allow pre-customization of
1702 this parameter outside of Ganeti.
1708 port = socket.getservbyname("ganeti-noded", "tcp")
1709 except socket.error:
1710 port = constants.DEFAULT_NODED_PORT
1715 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1716 multithreaded=False):
1717 """Configures the logging module.
1720 @param logfile: the filename to which we should log
1721 @type debug: boolean
1722 @param debug: whether to enable debug messages too or
1723 only those at C{INFO} and above level
1724 @type stderr_logging: boolean
1725 @param stderr_logging: whether we should also log to the standard error
1727 @param program: the name under which we should log messages
1728 @type multithreaded: boolean
1729 @param multithreaded: if True, will add the thread name to the log file
1730 @raise EnvironmentError: if we can't open the log file and
1731 stderr logging is disabled
1734 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1736 fmt += "/%(threadName)s"
1738 fmt += " %(module)s:%(lineno)s"
1739 fmt += " %(levelname)s %(message)s"
1740 formatter = logging.Formatter(fmt)
1742 root_logger = logging.getLogger("")
1743 root_logger.setLevel(logging.NOTSET)
1745 # Remove all previously setup handlers
1746 for handler in root_logger.handlers:
1748 root_logger.removeHandler(handler)
1751 stderr_handler = logging.StreamHandler()
1752 stderr_handler.setFormatter(formatter)
1754 stderr_handler.setLevel(logging.NOTSET)
1756 stderr_handler.setLevel(logging.CRITICAL)
1757 root_logger.addHandler(stderr_handler)
1759 # this can fail, if the logging directories are not setup or we have
1760 # a permisssion problem; in this case, it's best to log but ignore
1761 # the error if stderr_logging is True, and if false we re-raise the
1762 # exception since otherwise we could run but without any logs at all
1764 logfile_handler = logging.FileHandler(logfile)
1765 logfile_handler.setFormatter(formatter)
1767 logfile_handler.setLevel(logging.DEBUG)
1769 logfile_handler.setLevel(logging.INFO)
1770 root_logger.addHandler(logfile_handler)
1771 except EnvironmentError:
1773 logging.exception("Failed to enable logging to file '%s'", logfile)
1775 # we need to re-raise the exception
1779 def TailFile(fname, lines=20):
1780 """Return the last lines from a file.
1782 @note: this function will only read and parse the last 4KB of
1783 the file; if the lines are very long, it could be that less
1784 than the requested number of lines are returned
1786 @param fname: the file name
1788 @param lines: the (maximum) number of lines to return
1791 fd = open(fname, "r")
1795 pos = max(0, pos-4096)
1797 raw_data = fd.read()
1801 rows = raw_data.splitlines()
1802 return rows[-lines:]
1805 def SafeEncode(text):
1806 """Return a 'safe' version of a source string.
1808 This function mangles the input string and returns a version that
1809 should be safe to disply/encode as ASCII. To this end, we first
1810 convert it to ASCII using the 'backslashreplace' encoding which
1811 should get rid of any non-ASCII chars, and then we again encode it
1812 via 'string_escape' which converts '\n' into '\\n' so that log
1813 messages remain one-line.
1815 @type text: str or unicode
1816 @param text: input data
1818 @return: a safe version of text
1821 text = text.encode('ascii', 'backslashreplace')
1822 text = text.encode('string_escape')
1826 def LockedMethod(fn):
1827 """Synchronized object access decorator.
1829 This decorator is intended to protect access to an object using the
1830 object's own lock which is hardcoded to '_lock'.
1833 def _LockDebug(*args, **kwargs):
1835 logging.debug(*args, **kwargs)
1837 def wrapper(self, *args, **kwargs):
1838 assert hasattr(self, '_lock')
1840 _LockDebug("Waiting for %s", lock)
1843 _LockDebug("Acquired %s", lock)
1844 result = fn(self, *args, **kwargs)
1846 _LockDebug("Releasing %s", lock)
1848 _LockDebug("Released %s", lock)
1854 """Locks a file using POSIX locks.
1857 @param fd: the file descriptor we need to lock
1861 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1862 except IOError, err:
1863 if err.errno == errno.EAGAIN:
1864 raise errors.LockError("File already locked")
1868 class FileLock(object):
1869 """Utility class for file locks.
1872 def __init__(self, filename):
1873 """Constructor for FileLock.
1875 This will open the file denoted by the I{filename} argument.
1878 @param filename: path to the file to be locked
1881 self.filename = filename
1882 self.fd = open(self.filename, "w")
1888 """Close the file and release the lock.
1895 def _flock(self, flag, blocking, timeout, errmsg):
1896 """Wrapper for fcntl.flock.
1899 @param flag: operation flag
1900 @type blocking: bool
1901 @param blocking: whether the operation should be done in blocking mode.
1902 @type timeout: None or float
1903 @param timeout: for how long the operation should be retried (implies
1905 @type errmsg: string
1906 @param errmsg: error message in case operation fails.
1909 assert self.fd, "Lock was closed"
1910 assert timeout is None or timeout >= 0, \
1911 "If specified, timeout must be positive"
1913 if timeout is not None:
1914 flag |= fcntl.LOCK_NB
1915 timeout_end = time.time() + timeout
1917 # Blocking doesn't have effect with timeout
1919 flag |= fcntl.LOCK_NB
1925 fcntl.flock(self.fd, flag)
1927 except IOError, err:
1928 if err.errno in (errno.EAGAIN, ):
1929 if timeout_end is not None and time.time() < timeout_end:
1930 # Wait before trying again
1931 time.sleep(max(0.1, min(1.0, timeout)))
1933 raise errors.LockError(errmsg)
1935 logging.exception("fcntl.flock failed")
1938 def Exclusive(self, blocking=False, timeout=None):
1939 """Locks the file in exclusive mode.
1941 @type blocking: boolean
1942 @param blocking: whether to block and wait until we
1943 can lock the file or return immediately
1944 @type timeout: int or None
1945 @param timeout: if not None, the duration to wait for the lock
1949 self._flock(fcntl.LOCK_EX, blocking, timeout,
1950 "Failed to lock %s in exclusive mode" % self.filename)
1952 def Shared(self, blocking=False, timeout=None):
1953 """Locks the file in shared mode.
1955 @type blocking: boolean
1956 @param blocking: whether to block and wait until we
1957 can lock the file or return immediately
1958 @type timeout: int or None
1959 @param timeout: if not None, the duration to wait for the lock
1963 self._flock(fcntl.LOCK_SH, blocking, timeout,
1964 "Failed to lock %s in shared mode" % self.filename)
1966 def Unlock(self, blocking=True, timeout=None):
1967 """Unlocks the file.
1969 According to C{flock(2)}, unlocking can also be a nonblocking
1972 To make a non-blocking request, include LOCK_NB with any of the above
1975 @type blocking: boolean
1976 @param blocking: whether to block and wait until we
1977 can lock the file or return immediately
1978 @type timeout: int or None
1979 @param timeout: if not None, the duration to wait for the lock
1983 self._flock(fcntl.LOCK_UN, blocking, timeout,
1984 "Failed to unlock %s" % self.filename)
1987 class SignalHandler(object):
1988 """Generic signal handler class.
1990 It automatically restores the original handler when deconstructed or
1991 when L{Reset} is called. You can either pass your own handler
1992 function in or query the L{called} attribute to detect whether the
1996 @ivar signum: the signals we handle
1997 @type called: boolean
1998 @ivar called: tracks whether any of the signals have been raised
2001 def __init__(self, signum):
2002 """Constructs a new SignalHandler instance.
2004 @type signum: int or list of ints
2005 @param signum: Single signal number or set of signal numbers
2008 if isinstance(signum, (int, long)):
2009 self.signum = set([signum])
2011 self.signum = set(signum)
2017 for signum in self.signum:
2019 prev_handler = signal.signal(signum, self._HandleSignal)
2021 self._previous[signum] = prev_handler
2023 # Restore previous handler
2024 signal.signal(signum, prev_handler)
2027 # Reset all handlers
2029 # Here we have a race condition: a handler may have already been called,
2030 # but there's not much we can do about it at this point.
2037 """Restore previous handler.
2039 This will reset all the signals to their previous handlers.
2042 for signum, prev_handler in self._previous.items():
2043 signal.signal(signum, prev_handler)
2044 # If successful, remove from dict
2045 del self._previous[signum]
2048 """Unsets the L{called} flag.
2050 This function can be used in case a signal may arrive several times.
2055 def _HandleSignal(self, signum, frame):
2056 """Actual signal handling function.
2059 # This is not nice and not absolutely atomic, but it appears to be the only
2060 # solution in Python -- there are no atomic types.
2064 class FieldSet(object):
2065 """A simple field set.
2067 Among the features are:
2068 - checking if a string is among a list of static string or regex objects
2069 - checking if a whole list of string matches
2070 - returning the matching groups from a regex match
2072 Internally, all fields are held as regular expression objects.
2075 def __init__(self, *items):
2076 self.items = [re.compile("^%s$" % value) for value in items]
2078 def Extend(self, other_set):
2079 """Extend the field set with the items from another one"""
2080 self.items.extend(other_set.items)
2082 def Matches(self, field):
2083 """Checks if a field matches the current set
2086 @param field: the string to match
2087 @return: either False or a regular expression match object
2090 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2094 def NonMatching(self, items):
2095 """Returns the list of fields not matching the current set
2098 @param items: the list of fields to check
2100 @return: list of non-matching fields
2103 return [val for val in items if not self.Matches(val)]