4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti utility module.
24 This module holds functions that can be used in both daemons (all) and
25 the command line scripts.
46 from cStringIO import StringIO
49 from hashlib import sha1
54 from ganeti import errors
55 from ganeti import constants
59 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
63 #: when set to True, L{RunCmd} is disabled
67 class RunResult(object):
68 """Holds the result of running external programs.
71 @ivar exit_code: the exit code of the program, or None (if the program
73 @type signal: int or None
74 @ivar signal: the signal that caused the program to finish, or None
75 (if the program wasn't terminated by a signal)
77 @ivar stdout: the standard output of the program
79 @ivar stderr: the standard error of the program
81 @ivar failed: True in case the program was
82 terminated by a signal or exited with a non-zero exit code
83 @ivar fail_reason: a string detailing the termination reason
86 __slots__ = ["exit_code", "signal", "stdout", "stderr",
87 "failed", "fail_reason", "cmd"]
90 def __init__(self, exit_code, signal_, stdout, stderr, cmd):
92 self.exit_code = exit_code
96 self.failed = (signal_ is not None or exit_code != 0)
98 if self.signal is not None:
99 self.fail_reason = "terminated by signal %s" % self.signal
100 elif self.exit_code is not None:
101 self.fail_reason = "exited with exit code %s" % self.exit_code
103 self.fail_reason = "unable to determine termination reason"
106 logging.debug("Command '%s' failed (%s); output: %s",
107 self.cmd, self.fail_reason, self.output)
109 def _GetOutput(self):
110 """Returns the combined stdout and stderr for easier usage.
113 return self.stdout + self.stderr
115 output = property(_GetOutput, None, None, "Return full output")
118 def RunCmd(cmd, env=None, output=None, cwd='/'):
119 """Execute a (shell) command.
121 The command should not read from its standard input, as it will be
124 @type cmd: string or list
125 @param cmd: Command to run
127 @param env: Additional environment
129 @param output: if desired, the output of the command can be
130 saved in a file instead of the RunResult instance; this
131 parameter denotes the file name (if not None)
133 @param cwd: if specified, will be used as the working
134 directory for the command; the default will be /
136 @return: RunResult instance
137 @raise errors.ProgrammerError: if we call this when forks are disabled
141 raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
143 if isinstance(cmd, list):
144 cmd = [str(val) for val in cmd]
145 strcmd = " ".join(cmd)
150 logging.debug("RunCmd '%s'", strcmd)
152 cmd_env = os.environ.copy()
153 cmd_env["LC_ALL"] = "C"
159 out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
161 status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
164 if err.errno == errno.ENOENT:
165 raise errors.OpExecError("Can't execute '%s': not found (%s)" %
177 return RunResult(exitcode, signal_, out, err, strcmd)
180 def _RunCmdPipe(cmd, env, via_shell, cwd):
181 """Run a command and return its output.
183 @type cmd: string or list
184 @param cmd: Command to run
186 @param env: The environment to use
187 @type via_shell: bool
188 @param via_shell: if we should run via the shell
190 @param cwd: the working directory for the program
192 @return: (out, err, status)
195 poller = select.poll()
196 child = subprocess.Popen(cmd, shell=via_shell,
197 stderr=subprocess.PIPE,
198 stdout=subprocess.PIPE,
199 stdin=subprocess.PIPE,
200 close_fds=True, env=env,
204 poller.register(child.stdout, select.POLLIN)
205 poller.register(child.stderr, select.POLLIN)
209 child.stdout.fileno(): (out, child.stdout),
210 child.stderr.fileno(): (err, child.stderr),
213 status = fcntl.fcntl(fd, fcntl.F_GETFL)
214 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
218 pollresult = poller.poll()
219 except EnvironmentError, eerr:
220 if eerr.errno == errno.EINTR:
223 except select.error, serr:
224 if serr[0] == errno.EINTR:
228 for fd, event in pollresult:
229 if event & select.POLLIN or event & select.POLLPRI:
230 data = fdmap[fd][1].read()
231 # no data from read signifies EOF (the same as POLLHUP)
233 poller.unregister(fd)
236 fdmap[fd][0].write(data)
237 if (event & select.POLLNVAL or event & select.POLLHUP or
238 event & select.POLLERR):
239 poller.unregister(fd)
245 status = child.wait()
246 return out, err, status
249 def _RunCmdFile(cmd, env, via_shell, output, cwd):
250 """Run a command and save its output to a file.
252 @type cmd: string or list
253 @param cmd: Command to run
255 @param env: The environment to use
256 @type via_shell: bool
257 @param via_shell: if we should run via the shell
259 @param output: the filename in which to save the output
261 @param cwd: the working directory for the program
263 @return: the exit status
266 fh = open(output, "a")
268 child = subprocess.Popen(cmd, shell=via_shell,
269 stderr=subprocess.STDOUT,
271 stdin=subprocess.PIPE,
272 close_fds=True, env=env,
276 status = child.wait()
282 def RemoveFile(filename):
283 """Remove a file ignoring some errors.
285 Remove a file, ignoring non-existing ones or directories. Other
289 @param filename: the file to be removed
295 if err.errno not in (errno.ENOENT, errno.EISDIR):
299 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
303 @param old: Original path
307 @param mkdir: Whether to create target directory if it doesn't exist
308 @type mkdir_mode: int
309 @param mkdir_mode: Mode for newly created directories
313 return os.rename(old, new)
315 # In at least one use case of this function, the job queue, directory
316 # creation is very rare. Checking for the directory before renaming is not
318 if mkdir and err.errno == errno.ENOENT:
319 # Create directory and try again
320 os.makedirs(os.path.dirname(new), mkdir_mode)
321 return os.rename(old, new)
325 def _FingerprintFile(filename):
326 """Compute the fingerprint of a file.
328 If the file does not exist, a None will be returned
332 @param filename: the filename to checksum
334 @return: the hex digest of the sha checksum of the contents
338 if not (os.path.exists(filename) and os.path.isfile(filename)):
351 return fp.hexdigest()
354 def FingerprintFiles(files):
355 """Compute fingerprints for a list of files.
358 @param files: the list of filename to fingerprint
360 @return: a dictionary filename: fingerprint, holding only
366 for filename in files:
367 cksum = _FingerprintFile(filename)
369 ret[filename] = cksum
374 def ForceDictType(target, key_types, allowed_values=None):
375 """Force the values of a dict to have certain types.
378 @param target: the dict to update
379 @type key_types: dict
380 @param key_types: dict mapping target dict keys to types
381 in constants.ENFORCEABLE_TYPES
382 @type allowed_values: list
383 @keyword allowed_values: list of specially allowed values
386 if allowed_values is None:
389 if not isinstance(target, dict):
390 msg = "Expected dictionary, got '%s'" % target
391 raise errors.TypeEnforcementError(msg)
394 if key not in key_types:
395 msg = "Unknown key '%s'" % key
396 raise errors.TypeEnforcementError(msg)
398 if target[key] in allowed_values:
401 ktype = key_types[key]
402 if ktype not in constants.ENFORCEABLE_TYPES:
403 msg = "'%s' has non-enforceable type %s" % (key, ktype)
404 raise errors.ProgrammerError(msg)
406 if ktype == constants.VTYPE_STRING:
407 if not isinstance(target[key], basestring):
408 if isinstance(target[key], bool) and not target[key]:
411 msg = "'%s' (value %s) is not a valid string" % (key, target[key])
412 raise errors.TypeEnforcementError(msg)
413 elif ktype == constants.VTYPE_BOOL:
414 if isinstance(target[key], basestring) and target[key]:
415 if target[key].lower() == constants.VALUE_FALSE:
417 elif target[key].lower() == constants.VALUE_TRUE:
420 msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
421 raise errors.TypeEnforcementError(msg)
426 elif ktype == constants.VTYPE_SIZE:
428 target[key] = ParseUnit(target[key])
429 except errors.UnitParseError, err:
430 msg = "'%s' (value %s) is not a valid size. error: %s" % \
431 (key, target[key], err)
432 raise errors.TypeEnforcementError(msg)
433 elif ktype == constants.VTYPE_INT:
435 target[key] = int(target[key])
436 except (ValueError, TypeError):
437 msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
438 raise errors.TypeEnforcementError(msg)
441 def IsProcessAlive(pid):
442 """Check if a given pid exists on the system.
444 @note: zombie status is not handled, so zombie processes
445 will be returned as alive
447 @param pid: the process ID to check
449 @return: True if the process exists
456 os.stat("/proc/%d/status" % pid)
458 except EnvironmentError, err:
459 if err.errno in (errno.ENOENT, errno.ENOTDIR):
464 def ReadPidFile(pidfile):
465 """Read a pid from a file.
467 @type pidfile: string
468 @param pidfile: path to the file containing the pid
470 @return: The process id, if the file exists and contains a valid PID,
475 pf = open(pidfile, 'r')
476 except EnvironmentError, err:
477 if err.errno != errno.ENOENT:
478 logging.exception("Can't read pid file?!")
483 except ValueError, err:
484 logging.info("Can't parse pid file contents", exc_info=True)
490 def MatchNameComponent(key, name_list):
491 """Try to match a name against a list.
493 This function will try to match a name like test1 against a list
494 like C{['test1.example.com', 'test2.example.com', ...]}. Against
495 this list, I{'test1'} as well as I{'test1.example'} will match, but
496 not I{'test1.ex'}. A multiple match will be considered as no match
497 at all (e.g. I{'test1'} against C{['test1.example.com',
498 'test1.example.org']}).
501 @param key: the name to be searched
502 @type name_list: list
503 @param name_list: the list of strings against which to search the key
506 @return: None if there is no match I{or} if there are multiple matches,
507 otherwise the element from the list which matches
510 mo = re.compile("^%s(\..*)?$" % re.escape(key))
511 names_filtered = [name for name in name_list if mo.match(name) is not None]
512 if len(names_filtered) != 1:
514 return names_filtered[0]
518 """Class implementing resolver and hostname functionality
521 def __init__(self, name=None):
522 """Initialize the host name object.
524 If the name argument is not passed, it will use this system's
529 name = self.SysName()
532 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
533 self.ip = self.ipaddrs[0]
536 """Returns the hostname without domain.
539 return self.name.split('.')[0]
543 """Return the current system's name.
545 This is simply a wrapper over C{socket.gethostname()}.
548 return socket.gethostname()
551 def LookupHostname(hostname):
555 @param hostname: hostname to look up
558 @return: a tuple (name, aliases, ipaddrs) as returned by
559 C{socket.gethostbyname_ex}
560 @raise errors.ResolverError: in case of errors in resolving
564 result = socket.gethostbyname_ex(hostname)
565 except socket.gaierror, err:
566 # hostname not found in DNS
567 raise errors.ResolverError(hostname, err.args[0], err.args[1])
572 def ListVolumeGroups():
573 """List volume groups and their size
577 Dictionary with keys volume name and values
578 the size of the volume
581 command = "vgs --noheadings --units m --nosuffix -o name,size"
582 result = RunCmd(command)
587 for line in result.stdout.splitlines():
589 name, size = line.split()
590 size = int(float(size))
591 except (IndexError, ValueError), err:
592 logging.error("Invalid output from vgs (%s): %s", err, line)
600 def BridgeExists(bridge):
601 """Check whether the given bridge exists in the system
604 @param bridge: the bridge name to check
606 @return: True if it does
609 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
612 def NiceSort(name_list):
613 """Sort a list of strings based on digit and non-digit groupings.
615 Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
616 will sort the list in the logical order C{['a1', 'a2', 'a10',
619 The sort algorithm breaks each name in groups of either only-digits
620 or no-digits. Only the first eight such groups are considered, and
621 after that we just use what's left of the string.
623 @type name_list: list
624 @param name_list: the names to be sorted
626 @return: a copy of the name list sorted with our algorithm
629 _SORTER_BASE = "(\D+|\d+)"
630 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
631 _SORTER_BASE, _SORTER_BASE,
632 _SORTER_BASE, _SORTER_BASE,
633 _SORTER_BASE, _SORTER_BASE)
634 _SORTER_RE = re.compile(_SORTER_FULL)
635 _SORTER_NODIGIT = re.compile("^\D*$")
637 """Attempts to convert a variable to integer."""
638 if val is None or _SORTER_NODIGIT.match(val):
643 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
644 for name in name_list]
646 return [tup[1] for tup in to_sort]
649 def TryConvert(fn, val):
650 """Try to convert a value ignoring errors.
652 This function tries to apply function I{fn} to I{val}. If no
653 C{ValueError} or C{TypeError} exceptions are raised, it will return
654 the result, else it will return the original value. Any other
655 exceptions are propagated to the caller.
658 @param fn: function to apply to the value
659 @param val: the value to be converted
660 @return: The converted value if the conversion was successful,
661 otherwise the original value.
666 except (ValueError, TypeError):
672 """Verifies the syntax of an IPv4 address.
674 This function checks if the IPv4 address passes is valid or not based
675 on syntax (not IP range, class calculations, etc.).
678 @param ip: the address to be checked
679 @rtype: a regular expression match object
680 @return: a regular expression match object, or None if the
684 unit = "(0|[1-9]\d{0,2})"
685 #TODO: convert and return only boolean
686 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
689 def IsValidShellParam(word):
690 """Verifies is the given word is safe from the shell's p.o.v.
692 This means that we can pass this to a command via the shell and be
693 sure that it doesn't alter the command line and is passed as such to
696 Note that we are overly restrictive here, in order to be on the safe
700 @param word: the word to check
702 @return: True if the word is 'safe'
705 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
708 def BuildShellCmd(template, *args):
709 """Build a safe shell command line from the given arguments.
711 This function will check all arguments in the args list so that they
712 are valid shell parameters (i.e. they don't contain shell
713 metacharacters). If everything is ok, it will return the result of
717 @param template: the string holding the template for the
720 @return: the expanded command line
724 if not IsValidShellParam(word):
725 raise errors.ProgrammerError("Shell argument '%s' contains"
726 " invalid characters" % word)
727 return template % args
730 def FormatUnit(value, units):
731 """Formats an incoming number of MiB with the appropriate unit.
734 @param value: integer representing the value in MiB (1048576)
736 @param units: the type of formatting we should do:
737 - 'h' for automatic scaling
742 @return: the formatted value (with suffix)
745 if units not in ('m', 'g', 't', 'h'):
746 raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
750 if units == 'm' or (units == 'h' and value < 1024):
753 return "%d%s" % (round(value, 0), suffix)
755 elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
758 return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
763 return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
766 def ParseUnit(input_string):
767 """Tries to extract number and scale from the given string.
769 Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
770 [UNIT]}. If no unit is specified, it defaults to MiB. Return value
771 is always an int in MiB.
774 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
776 raise errors.UnitParseError("Invalid format")
778 value = float(m.groups()[0])
782 lcunit = unit.lower()
786 if lcunit in ('m', 'mb', 'mib'):
787 # Value already in MiB
790 elif lcunit in ('g', 'gb', 'gib'):
793 elif lcunit in ('t', 'tb', 'tib'):
797 raise errors.UnitParseError("Unknown unit: %s" % unit)
799 # Make sure we round up
800 if int(value) < value:
803 # Round up to the next multiple of 4
806 value += 4 - value % 4
811 def AddAuthorizedKey(file_name, key):
812 """Adds an SSH public key to an authorized_keys file.
815 @param file_name: path to authorized_keys file
817 @param key: string containing key
820 key_fields = key.split()
822 f = open(file_name, 'a+')
826 # Ignore whitespace changes
827 if line.split() == key_fields:
829 nl = line.endswith('\n')
833 f.write(key.rstrip('\r\n'))
840 def RemoveAuthorizedKey(file_name, key):
841 """Removes an SSH public key from an authorized_keys file.
844 @param file_name: path to authorized_keys file
846 @param key: string containing key
849 key_fields = key.split()
851 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
853 out = os.fdopen(fd, 'w')
855 f = open(file_name, 'r')
858 # Ignore whitespace changes while comparing lines
859 if line.split() != key_fields:
863 os.rename(tmpname, file_name)
873 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
874 """Sets the name of an IP address and hostname in /etc/hosts.
877 @param file_name: path to the file to modify (usually C{/etc/hosts})
879 @param ip: the IP address
881 @param hostname: the hostname to be added
883 @param aliases: the list of aliases to add for the hostname
886 # FIXME: use WriteFile + fn rather than duplicating its efforts
887 # Ensure aliases are unique
888 aliases = UniqueSequence([hostname] + aliases)[1:]
890 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
892 out = os.fdopen(fd, 'w')
894 f = open(file_name, 'r')
897 fields = line.split()
898 if fields and not fields[0].startswith('#') and ip == fields[0]:
902 out.write("%s\t%s" % (ip, hostname))
904 out.write(" %s" % ' '.join(aliases))
909 os.chmod(tmpname, 0644)
910 os.rename(tmpname, file_name)
920 def AddHostToEtcHosts(hostname):
921 """Wrapper around SetEtcHostsEntry.
924 @param hostname: a hostname that will be resolved and added to
925 L{constants.ETC_HOSTS}
928 hi = HostInfo(name=hostname)
929 SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
932 def RemoveEtcHostsEntry(file_name, hostname):
933 """Removes a hostname from /etc/hosts.
935 IP addresses without names are removed from the file.
938 @param file_name: path to the file to modify (usually C{/etc/hosts})
940 @param hostname: the hostname to be removed
943 # FIXME: use WriteFile + fn rather than duplicating its efforts
944 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
946 out = os.fdopen(fd, 'w')
948 f = open(file_name, 'r')
951 fields = line.split()
952 if len(fields) > 1 and not fields[0].startswith('#'):
954 if hostname in names:
955 while hostname in names:
956 names.remove(hostname)
958 out.write("%s %s\n" % (fields[0], ' '.join(names)))
965 os.chmod(tmpname, 0644)
966 os.rename(tmpname, file_name)
976 def RemoveHostFromEtcHosts(hostname):
977 """Wrapper around RemoveEtcHostsEntry.
980 @param hostname: hostname that will be resolved and its
981 full and shot name will be removed from
982 L{constants.ETC_HOSTS}
985 hi = HostInfo(name=hostname)
986 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
987 RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
990 def CreateBackup(file_name):
991 """Creates a backup of a file.
994 @param file_name: file to be backed up
996 @return: the path to the newly created backup
997 @raise errors.ProgrammerError: for invalid file names
1000 if not os.path.isfile(file_name):
1001 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1004 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1005 dir_name = os.path.dirname(file_name)
1007 fsrc = open(file_name, 'rb')
1009 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1010 fdst = os.fdopen(fd, 'wb')
1012 shutil.copyfileobj(fsrc, fdst)
1021 def ShellQuote(value):
1022 """Quotes shell argument according to POSIX.
1025 @param value: the argument to be quoted
1027 @return: the quoted value
1030 if _re_shell_unquoted.match(value):
1033 return "'%s'" % value.replace("'", "'\\''")
1036 def ShellQuoteArgs(args):
1037 """Quotes a list of shell arguments.
1040 @param args: list of arguments to be quoted
1042 @return: the quoted arguments concatenated with spaces
1045 return ' '.join([ShellQuote(i) for i in args])
1048 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1049 """Simple ping implementation using TCP connect(2).
1051 Check if the given IP is reachable by doing attempting a TCP connect
1055 @param target: the IP or hostname to ping
1057 @param port: the port to connect to
1059 @param timeout: the timeout on the connection attempt
1060 @type live_port_needed: boolean
1061 @param live_port_needed: whether a closed port will cause the
1062 function to return failure, as if there was a timeout
1063 @type source: str or None
1064 @param source: if specified, will cause the connect to be made
1065 from this specific source address; failures to bind other
1066 than C{EADDRNOTAVAIL} will be ignored
1069 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1073 if source is not None:
1075 sock.bind((source, 0))
1076 except socket.error, (errcode, _):
1077 if errcode == errno.EADDRNOTAVAIL:
1080 sock.settimeout(timeout)
1083 sock.connect((target, port))
1086 except socket.timeout:
1088 except socket.error, (errcode, errstring):
1089 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1094 def OwnIpAddress(address):
1095 """Check if the current host has the the given IP address.
1097 Currently this is done by TCP-pinging the address from the loopback
1100 @type address: string
1101 @param address: the address to check
1103 @return: True if we own the address
1106 return TcpPing(address, constants.DEFAULT_NODED_PORT,
1107 source=constants.LOCALHOST_IP_ADDRESS)
1110 def ListVisibleFiles(path):
1111 """Returns a list of visible files in a directory.
1114 @param path: the directory to enumerate
1116 @return: the list of all files not starting with a dot
1119 files = [i for i in os.listdir(path) if not i.startswith(".")]
1124 def GetHomeDir(user, default=None):
1125 """Try to get the homedir of the given user.
1127 The user can be passed either as a string (denoting the name) or as
1128 an integer (denoting the user id). If the user is not found, the
1129 'default' argument is returned, which defaults to None.
1133 if isinstance(user, basestring):
1134 result = pwd.getpwnam(user)
1135 elif isinstance(user, (int, long)):
1136 result = pwd.getpwuid(user)
1138 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1142 return result.pw_dir
1146 """Returns a random UUID.
1148 @note: This is a Linux-specific method as it uses the /proc
1153 f = open("/proc/sys/kernel/random/uuid", "r")
1155 return f.read(128).rstrip("\n")
1160 def GenerateSecret():
1161 """Generates a random secret.
1163 This will generate a pseudo-random secret, and return its sha digest
1164 (so that it can be used where an ASCII string is needed).
1167 @return: a sha1 hexdigest of a block of 64 random bytes
1170 return sha1(os.urandom(64)).hexdigest()
1173 def EnsureDirs(dirs):
1174 """Make required directories, if they don't exist.
1176 @param dirs: list of tuples (dir_name, dir_mode)
1177 @type dirs: list of (string, integer)
1180 for dir_name, dir_mode in dirs:
1182 os.mkdir(dir_name, dir_mode)
1183 except EnvironmentError, err:
1184 if err.errno != errno.EEXIST:
1185 raise errors.GenericError("Cannot create needed directory"
1186 " '%s': %s" % (dir_name, err))
1187 if not os.path.isdir(dir_name):
1188 raise errors.GenericError("%s is not a directory" % dir_name)
1191 def ReadFile(file_name, size=None):
1194 @type size: None or int
1195 @param size: Read at most size bytes
1197 @return: the (possibly partial) content of the file
1200 f = open(file_name, "r")
1210 def WriteFile(file_name, fn=None, data=None,
1211 mode=None, uid=-1, gid=-1,
1212 atime=None, mtime=None, close=True,
1213 dry_run=False, backup=False,
1214 prewrite=None, postwrite=None):
1215 """(Over)write a file atomically.
1217 The file_name and either fn (a function taking one argument, the
1218 file descriptor, and which should write the data to it) or data (the
1219 contents of the file) must be passed. The other arguments are
1220 optional and allow setting the file mode, owner and group, and the
1221 mtime/atime of the file.
1223 If the function doesn't raise an exception, it has succeeded and the
1224 target file has the new contents. If the function has raised an
1225 exception, an existing target file should be unmodified and the
1226 temporary file should be removed.
1228 @type file_name: str
1229 @param file_name: the target filename
1231 @param fn: content writing function, called with
1232 file descriptor as parameter
1234 @param data: contents of the file
1236 @param mode: file mode
1238 @param uid: the owner of the file
1240 @param gid: the group of the file
1242 @param atime: a custom access time to be set on the file
1244 @param mtime: a custom modification time to be set on the file
1245 @type close: boolean
1246 @param close: whether to close file after writing it
1247 @type prewrite: callable
1248 @param prewrite: function to be called before writing content
1249 @type postwrite: callable
1250 @param postwrite: function to be called after writing content
1253 @return: None if the 'close' parameter evaluates to True,
1254 otherwise the file descriptor
1256 @raise errors.ProgrammerError: if any of the arguments are not valid
1259 if not os.path.isabs(file_name):
1260 raise errors.ProgrammerError("Path passed to WriteFile is not"
1261 " absolute: '%s'" % file_name)
1263 if [fn, data].count(None) != 1:
1264 raise errors.ProgrammerError("fn or data required")
1266 if [atime, mtime].count(None) == 1:
1267 raise errors.ProgrammerError("Both atime and mtime must be either"
1270 if backup and not dry_run and os.path.isfile(file_name):
1271 CreateBackup(file_name)
1273 dir_name, base_name = os.path.split(file_name)
1274 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1276 # here we need to make sure we remove the temp file, if any error
1277 # leaves it in place
1279 if uid != -1 or gid != -1:
1280 os.chown(new_name, uid, gid)
1282 os.chmod(new_name, mode)
1283 if callable(prewrite):
1285 if data is not None:
1289 if callable(postwrite):
1292 if atime is not None and mtime is not None:
1293 os.utime(new_name, (atime, mtime))
1295 os.rename(new_name, file_name)
1304 RemoveFile(new_name)
1309 def FirstFree(seq, base=0):
1310 """Returns the first non-existing integer from seq.
1312 The seq argument should be a sorted list of positive integers. The
1313 first time the index of an element is smaller than the element
1314 value, the index will be returned.
1316 The base argument is used to start at a different offset,
1317 i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1319 Example: C{[0, 1, 3]} will return I{2}.
1322 @param seq: the sequence to be analyzed.
1324 @param base: use this value as the base index of the sequence
1326 @return: the first non-used index in the sequence
1329 for idx, elem in enumerate(seq):
1330 assert elem >= base, "Passed element is higher than base offset"
1331 if elem > idx + base:
1337 def all(seq, pred=bool):
1338 "Returns True if pred(x) is True for every element in the iterable"
1339 for _ in itertools.ifilterfalse(pred, seq):
1344 def any(seq, pred=bool):
1345 "Returns True if pred(x) is True for at least one element in the iterable"
1346 for _ in itertools.ifilter(pred, seq):
1351 def UniqueSequence(seq):
1352 """Returns a list with unique elements.
1354 Element order is preserved.
1357 @param seq: the sequence with the source elements
1359 @return: list of unique elements from seq
1363 return [i for i in seq if i not in seen and not seen.add(i)]
1366 def IsValidMac(mac):
1367 """Predicate to check if a MAC address is valid.
1369 Checks whether the supplied MAC address is formally correct, only
1370 accepts colon separated format.
1373 @param mac: the MAC to be validated
1375 @return: True is the MAC seems valid
1378 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1379 return mac_check.match(mac) is not None
1382 def TestDelay(duration):
1383 """Sleep for a fixed amount of time.
1385 @type duration: float
1386 @param duration: the sleep duration
1388 @return: False for negative value, True otherwise
1392 return False, "Invalid sleep duration"
1393 time.sleep(duration)
1397 def _CloseFDNoErr(fd, retries=5):
1398 """Close a file descriptor ignoring errors.
1401 @param fd: the file descriptor
1403 @param retries: how many retries to make, in case we get any
1404 other error than EBADF
1409 except OSError, err:
1410 if err.errno != errno.EBADF:
1412 _CloseFDNoErr(fd, retries - 1)
1413 # else either it's closed already or we're out of retries, so we
1414 # ignore this and go on
1417 def CloseFDs(noclose_fds=None):
1418 """Close file descriptors.
1420 This closes all file descriptors above 2 (i.e. except
1423 @type noclose_fds: list or None
1424 @param noclose_fds: if given, it denotes a list of file descriptor
1425 that should not be closed
1428 # Default maximum for the number of available file descriptors.
1429 if 'SC_OPEN_MAX' in os.sysconf_names:
1431 MAXFD = os.sysconf('SC_OPEN_MAX')
1438 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1439 if (maxfd == resource.RLIM_INFINITY):
1442 # Iterate through and close all file descriptors (except the standard ones)
1443 for fd in range(3, maxfd):
1444 if noclose_fds and fd in noclose_fds:
1449 def Daemonize(logfile):
1450 """Daemonize the current process.
1452 This detaches the current process from the controlling terminal and
1453 runs it in the background as a daemon.
1456 @param logfile: the logfile to which we should redirect stdout/stderr
1458 @return: the value zero
1466 if (pid == 0): # The first child.
1469 pid = os.fork() # Fork a second child.
1470 if (pid == 0): # The second child.
1474 # exit() or _exit()? See below.
1475 os._exit(0) # Exit parent (the first child) of the second child.
1477 os._exit(0) # Exit parent of the first child.
1481 i = os.open("/dev/null", os.O_RDONLY) # stdin
1482 assert i == 0, "Can't close/reopen stdin"
1483 i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1484 assert i == 1, "Can't close/reopen stdout"
1485 # Duplicate standard output to standard error.
1490 def DaemonPidFileName(name):
1491 """Compute a ganeti pid file absolute path
1494 @param name: the daemon name
1496 @return: the full path to the pidfile corresponding to the given
1500 return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1503 def WritePidFile(name):
1504 """Write the current process pidfile.
1506 The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1509 @param name: the daemon name to use
1510 @raise errors.GenericError: if the pid file already exists and
1511 points to a live process
1515 pidfilename = DaemonPidFileName(name)
1516 if IsProcessAlive(ReadPidFile(pidfilename)):
1517 raise errors.GenericError("%s contains a live process" % pidfilename)
1519 WriteFile(pidfilename, data="%d\n" % pid)
1522 def RemovePidFile(name):
1523 """Remove the current process pidfile.
1525 Any errors are ignored.
1528 @param name: the daemon name used to derive the pidfile name
1531 pidfilename = DaemonPidFileName(name)
1532 # TODO: we could check here that the file contains our pid
1534 RemoveFile(pidfilename)
1539 def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1541 """Kill a process given by its pid.
1544 @param pid: The PID to terminate.
1546 @param signal_: The signal to send, by default SIGTERM
1548 @param timeout: The timeout after which, if the process is still alive,
1549 a SIGKILL will be sent. If not positive, no such checking
1551 @type waitpid: boolean
1552 @param waitpid: If true, we should waitpid on this process after
1553 sending signals, since it's our own child and otherwise it
1554 would remain as zombie
1557 def _helper(pid, signal_, wait):
1558 """Simple helper to encapsulate the kill/waitpid sequence"""
1559 os.kill(pid, signal_)
1562 os.waitpid(pid, os.WNOHANG)
1567 # kill with pid=0 == suicide
1568 raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1570 if not IsProcessAlive(pid):
1572 _helper(pid, signal_, waitpid)
1576 # Wait up to $timeout seconds
1577 end = time.time() + timeout
1579 while time.time() < end and IsProcessAlive(pid):
1581 (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1587 # Make wait time longer for next try
1591 if IsProcessAlive(pid):
1592 # Kill process if it's still alive
1593 _helper(pid, signal.SIGKILL, waitpid)
1596 def FindFile(name, search_path, test=os.path.exists):
1597 """Look for a filesystem object in a given path.
1599 This is an abstract method to search for filesystem object (files,
1600 dirs) under a given search path.
1603 @param name: the name to look for
1604 @type search_path: str
1605 @param search_path: location to start at
1606 @type test: callable
1607 @param test: a function taking one argument that should return True
1608 if the a given object is valid; the default value is
1609 os.path.exists, causing only existing files to be returned
1611 @return: full path to the object if found, None otherwise
1614 for dir_name in search_path:
1615 item_name = os.path.sep.join([dir_name, name])
1621 def CheckVolumeGroupSize(vglist, vgname, minsize):
1622 """Checks if the volume group list is valid.
1624 The function will check if a given volume group is in the list of
1625 volume groups and has a minimum size.
1628 @param vglist: dictionary of volume group names and their size
1630 @param vgname: the volume group we should check
1632 @param minsize: the minimum size we accept
1634 @return: None for success, otherwise the error message
1637 vgsize = vglist.get(vgname, None)
1639 return "volume group '%s' missing" % vgname
1640 elif vgsize < minsize:
1641 return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1642 (vgname, minsize, vgsize))
1646 def SplitTime(value):
1647 """Splits time as floating point number into a tuple.
1649 @param value: Time in seconds
1650 @type value: int or float
1651 @return: Tuple containing (seconds, microseconds)
1654 (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1656 assert 0 <= seconds, \
1657 "Seconds must be larger than or equal to 0, but are %s" % seconds
1658 assert 0 <= microseconds <= 999999, \
1659 "Microseconds must be 0-999999, but are %s" % microseconds
1661 return (int(seconds), int(microseconds))
1664 def MergeTime(timetuple):
1665 """Merges a tuple into time as a floating point number.
1667 @param timetuple: Time as tuple, (seconds, microseconds)
1668 @type timetuple: tuple
1669 @return: Time as a floating point number expressed in seconds
1672 (seconds, microseconds) = timetuple
1674 assert 0 <= seconds, \
1675 "Seconds must be larger than or equal to 0, but are %s" % seconds
1676 assert 0 <= microseconds <= 999999, \
1677 "Microseconds must be 0-999999, but are %s" % microseconds
1679 return float(seconds) + (float(microseconds) * 0.000001)
1682 def GetNodeDaemonPort():
1683 """Get the node daemon port for this cluster.
1685 Note that this routine does not read a ganeti-specific file, but
1686 instead uses C{socket.getservbyname} to allow pre-customization of
1687 this parameter outside of Ganeti.
1693 port = socket.getservbyname("ganeti-noded", "tcp")
1694 except socket.error:
1695 port = constants.DEFAULT_NODED_PORT
1700 def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1701 multithreaded=False):
1702 """Configures the logging module.
1705 @param logfile: the filename to which we should log
1706 @type debug: boolean
1707 @param debug: whether to enable debug messages too or
1708 only those at C{INFO} and above level
1709 @type stderr_logging: boolean
1710 @param stderr_logging: whether we should also log to the standard error
1712 @param program: the name under which we should log messages
1713 @type multithreaded: boolean
1714 @param multithreaded: if True, will add the thread name to the log file
1715 @raise EnvironmentError: if we can't open the log file and
1716 stderr logging is disabled
1719 fmt = "%(asctime)s: " + program + " pid=%(process)d"
1721 fmt += "/%(threadName)s"
1723 fmt += " %(module)s:%(lineno)s"
1724 fmt += " %(levelname)s %(message)s"
1725 formatter = logging.Formatter(fmt)
1727 root_logger = logging.getLogger("")
1728 root_logger.setLevel(logging.NOTSET)
1730 # Remove all previously setup handlers
1731 for handler in root_logger.handlers:
1733 root_logger.removeHandler(handler)
1736 stderr_handler = logging.StreamHandler()
1737 stderr_handler.setFormatter(formatter)
1739 stderr_handler.setLevel(logging.NOTSET)
1741 stderr_handler.setLevel(logging.CRITICAL)
1742 root_logger.addHandler(stderr_handler)
1744 # this can fail, if the logging directories are not setup or we have
1745 # a permisssion problem; in this case, it's best to log but ignore
1746 # the error if stderr_logging is True, and if false we re-raise the
1747 # exception since otherwise we could run but without any logs at all
1749 logfile_handler = logging.FileHandler(logfile)
1750 logfile_handler.setFormatter(formatter)
1752 logfile_handler.setLevel(logging.DEBUG)
1754 logfile_handler.setLevel(logging.INFO)
1755 root_logger.addHandler(logfile_handler)
1756 except EnvironmentError:
1758 logging.exception("Failed to enable logging to file '%s'", logfile)
1760 # we need to re-raise the exception
1763 def IsNormAbsPath(path):
1764 """Check whether a path is absolute and also normalized
1766 This avoids things like /dir/../../other/path to be valid.
1769 return os.path.normpath(path) == path and os.path.isabs(path)
1771 def TailFile(fname, lines=20):
1772 """Return the last lines from a file.
1774 @note: this function will only read and parse the last 4KB of
1775 the file; if the lines are very long, it could be that less
1776 than the requested number of lines are returned
1778 @param fname: the file name
1780 @param lines: the (maximum) number of lines to return
1783 fd = open(fname, "r")
1787 pos = max(0, pos-4096)
1789 raw_data = fd.read()
1793 rows = raw_data.splitlines()
1794 return rows[-lines:]
1797 def SafeEncode(text):
1798 """Return a 'safe' version of a source string.
1800 This function mangles the input string and returns a version that
1801 should be safe to display/encode as ASCII. To this end, we first
1802 convert it to ASCII using the 'backslashreplace' encoding which
1803 should get rid of any non-ASCII chars, and then we process it
1804 through a loop copied from the string repr sources in the python; we
1805 don't use string_escape anymore since that escape single quotes and
1806 backslashes too, and that is too much; and that escaping is not
1807 stable, i.e. string_escape(string_escape(x)) != string_escape(x).
1809 @type text: str or unicode
1810 @param text: input data
1812 @return: a safe version of text
1815 if isinstance(text, unicode):
1816 # only if unicode; if str already, we handle it below
1817 text = text.encode('ascii', 'backslashreplace')
1827 elif c < 32 or c >= 127: # non-printable
1828 resu += "\\x%02x" % (c & 0xff)
1834 def CommaJoin(names):
1835 """Nicely join a set of identifiers.
1837 @param names: set, list or tuple
1838 @return: a string with the formatted results
1841 return ", ".join(["'%s'" % val for val in names])
1844 def LockedMethod(fn):
1845 """Synchronized object access decorator.
1847 This decorator is intended to protect access to an object using the
1848 object's own lock which is hardcoded to '_lock'.
1851 def _LockDebug(*args, **kwargs):
1853 logging.debug(*args, **kwargs)
1855 def wrapper(self, *args, **kwargs):
1856 assert hasattr(self, '_lock')
1858 _LockDebug("Waiting for %s", lock)
1861 _LockDebug("Acquired %s", lock)
1862 result = fn(self, *args, **kwargs)
1864 _LockDebug("Releasing %s", lock)
1866 _LockDebug("Released %s", lock)
1872 """Locks a file using POSIX locks.
1875 @param fd: the file descriptor we need to lock
1879 fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1880 except IOError, err:
1881 if err.errno == errno.EAGAIN:
1882 raise errors.LockError("File already locked")
1886 class FileLock(object):
1887 """Utility class for file locks.
1890 def __init__(self, filename):
1891 """Constructor for FileLock.
1893 This will open the file denoted by the I{filename} argument.
1896 @param filename: path to the file to be locked
1899 self.filename = filename
1900 self.fd = open(self.filename, "w")
1906 """Close the file and release the lock.
1913 def _flock(self, flag, blocking, timeout, errmsg):
1914 """Wrapper for fcntl.flock.
1917 @param flag: operation flag
1918 @type blocking: bool
1919 @param blocking: whether the operation should be done in blocking mode.
1920 @type timeout: None or float
1921 @param timeout: for how long the operation should be retried (implies
1923 @type errmsg: string
1924 @param errmsg: error message in case operation fails.
1927 assert self.fd, "Lock was closed"
1928 assert timeout is None or timeout >= 0, \
1929 "If specified, timeout must be positive"
1931 if timeout is not None:
1932 flag |= fcntl.LOCK_NB
1933 timeout_end = time.time() + timeout
1935 # Blocking doesn't have effect with timeout
1937 flag |= fcntl.LOCK_NB
1943 fcntl.flock(self.fd, flag)
1945 except IOError, err:
1946 if err.errno in (errno.EAGAIN, ):
1947 if timeout_end is not None and time.time() < timeout_end:
1948 # Wait before trying again
1949 time.sleep(max(0.1, min(1.0, timeout)))
1951 raise errors.LockError(errmsg)
1953 logging.exception("fcntl.flock failed")
1956 def Exclusive(self, blocking=False, timeout=None):
1957 """Locks the file in exclusive mode.
1959 @type blocking: boolean
1960 @param blocking: whether to block and wait until we
1961 can lock the file or return immediately
1962 @type timeout: int or None
1963 @param timeout: if not None, the duration to wait for the lock
1967 self._flock(fcntl.LOCK_EX, blocking, timeout,
1968 "Failed to lock %s in exclusive mode" % self.filename)
1970 def Shared(self, blocking=False, timeout=None):
1971 """Locks the file in shared mode.
1973 @type blocking: boolean
1974 @param blocking: whether to block and wait until we
1975 can lock the file or return immediately
1976 @type timeout: int or None
1977 @param timeout: if not None, the duration to wait for the lock
1981 self._flock(fcntl.LOCK_SH, blocking, timeout,
1982 "Failed to lock %s in shared mode" % self.filename)
1984 def Unlock(self, blocking=True, timeout=None):
1985 """Unlocks the file.
1987 According to C{flock(2)}, unlocking can also be a nonblocking
1990 To make a non-blocking request, include LOCK_NB with any of the above
1993 @type blocking: boolean
1994 @param blocking: whether to block and wait until we
1995 can lock the file or return immediately
1996 @type timeout: int or None
1997 @param timeout: if not None, the duration to wait for the lock
2001 self._flock(fcntl.LOCK_UN, blocking, timeout,
2002 "Failed to unlock %s" % self.filename)
2005 class SignalHandler(object):
2006 """Generic signal handler class.
2008 It automatically restores the original handler when deconstructed or
2009 when L{Reset} is called. You can either pass your own handler
2010 function in or query the L{called} attribute to detect whether the
2014 @ivar signum: the signals we handle
2015 @type called: boolean
2016 @ivar called: tracks whether any of the signals have been raised
2019 def __init__(self, signum):
2020 """Constructs a new SignalHandler instance.
2022 @type signum: int or list of ints
2023 @param signum: Single signal number or set of signal numbers
2026 if isinstance(signum, (int, long)):
2027 self.signum = set([signum])
2029 self.signum = set(signum)
2035 for signum in self.signum:
2037 prev_handler = signal.signal(signum, self._HandleSignal)
2039 self._previous[signum] = prev_handler
2041 # Restore previous handler
2042 signal.signal(signum, prev_handler)
2045 # Reset all handlers
2047 # Here we have a race condition: a handler may have already been called,
2048 # but there's not much we can do about it at this point.
2055 """Restore previous handler.
2057 This will reset all the signals to their previous handlers.
2060 for signum, prev_handler in self._previous.items():
2061 signal.signal(signum, prev_handler)
2062 # If successful, remove from dict
2063 del self._previous[signum]
2066 """Unsets the L{called} flag.
2068 This function can be used in case a signal may arrive several times.
2073 def _HandleSignal(self, signum, frame):
2074 """Actual signal handling function.
2077 # This is not nice and not absolutely atomic, but it appears to be the only
2078 # solution in Python -- there are no atomic types.
2082 class FieldSet(object):
2083 """A simple field set.
2085 Among the features are:
2086 - checking if a string is among a list of static string or regex objects
2087 - checking if a whole list of string matches
2088 - returning the matching groups from a regex match
2090 Internally, all fields are held as regular expression objects.
2093 def __init__(self, *items):
2094 self.items = [re.compile("^%s$" % value) for value in items]
2096 def Extend(self, other_set):
2097 """Extend the field set with the items from another one"""
2098 self.items.extend(other_set.items)
2100 def Matches(self, field):
2101 """Checks if a field matches the current set
2104 @param field: the string to match
2105 @return: either False or a regular expression match object
2108 for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2112 def NonMatching(self, items):
2113 """Returns the list of fields not matching the current set
2116 @param items: the list of fields to check
2118 @return: list of non-matching fields
2121 return [val for val in items if not self.Matches(val)]