4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti small utilities
42 from cStringIO import StringIO
44 from ganeti import logger
45 from ganeti import errors
46 from ganeti import constants
50 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
54 class RunResult(object):
55 """Simple class for holding the result of running external programs.
58 exit_code: the exit code of the program, or None (if the program
60 signal: numeric signal that caused the program to finish, or None
61 (if the program wasn't terminated by a signal)
62 stdout: the standard output of the program
63 stderr: the standard error of the program
64 failed: a Boolean value which is True in case the program was
65 terminated by a signal or exited with a non-zero exit code
66 fail_reason: a string detailing the termination reason
69 __slots__ = ["exit_code", "signal", "stdout", "stderr",
70 "failed", "fail_reason", "cmd"]
73 def __init__(self, exit_code, signal, stdout, stderr, cmd):
75 self.exit_code = exit_code
79 self.failed = (signal is not None or exit_code != 0)
81 if self.signal is not None:
82 self.fail_reason = "terminated by signal %s" % self.signal
83 elif self.exit_code is not None:
84 self.fail_reason = "exited with exit code %s" % self.exit_code
86 self.fail_reason = "unable to determine termination reason"
88 if debug and self.failed:
89 logger.Debug("Command '%s' failed (%s); output: %s" %
90 (self.cmd, self.fail_reason, self.output))
93 """Returns the combined stdout and stderr for easier usage.
96 return self.stdout + self.stderr
98 output = property(_GetOutput, None, None, "Return full output")
101 def _GetLockFile(subsystem):
102 """Compute the file name for a given lock name."""
103 return "%s/ganeti_lock_%s" % (constants.LOCK_DIR, subsystem)
106 def Lock(name, max_retries=None, debug=False, autoclean=True):
107 """Lock a given subsystem.
109 In case the lock is already held by an alive process, the function
110 will sleep indefintely and poll with a one second interval.
112 When the optional integer argument 'max_retries' is passed with a
113 non-zero value, the function will sleep only for this number of
114 times, and then it will will raise a LockError if the lock can't be
115 acquired. Passing in a negative number will cause only one try to
116 get the lock. Passing a positive number will make the function retry
117 for approximately that number of seconds.
120 lockfile = _GetLockFile(name)
122 if name in _locksheld:
123 raise errors.LockError('Lock "%s" already held!' % (name,))
131 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_SYNC)
133 except OSError, creat_err:
134 if creat_err.errno != errno.EEXIST:
135 raise errors.LockError("Can't create the lock file. Error '%s'." %
139 pf = open(lockfile, 'r')
140 except IOError, open_err:
143 raise errors.LockError("Lock file exists but cannot be opened."
144 " Error: '%s'." % str(open_err))
151 raise errors.LockError("Invalid pid string in %s" %
154 if not IsProcessAlive(pid):
157 if cleanupcount >= 5:
158 raise errors.LockError, ("Too many stale lock cleanups! Check"
159 " what process is dying.")
160 logger.Error('Stale lockfile %s for pid %d, autocleaned.' %
165 raise errors.LockError("Stale lockfile %s for pid %d?" %
168 if max_retries and max_retries <= retries:
169 raise errors.LockError("Can't acquire lock during the specified"
171 if retries == 5 and (debug or sys.stdin.isatty()):
172 logger.ToStderr("Waiting for '%s' lock from pid %d..." % (name, pid))
178 os.write(fd, '%d\n' % (os.getpid(),))
181 _locksheld.append(name)
185 """Unlock a given subsystem.
188 lockfile = _GetLockFile(name)
191 fd = os.open(lockfile, os.O_RDONLY)
193 raise errors.LockError('Lock "%s" not held.' % (name,))
195 f = os.fdopen(fd, 'r')
201 raise errors.LockError('Unable to determine PID of locking process.')
203 if pid != os.getpid():
204 raise errors.LockError('Lock not held by me (%d != %d)' %
208 _locksheld.remove(name)
215 for lock in _locksheld:
220 """Execute a (shell) command.
222 The command should not read from its standard input, as it will be
226 cmd: command to run. (str)
228 Returns: `RunResult` instance
231 if isinstance(cmd, list):
232 cmd = [str(val) for val in cmd]
233 strcmd = " ".join(cmd)
238 env = os.environ.copy()
240 poller = select.poll()
241 child = subprocess.Popen(cmd, shell=shell,
242 stderr=subprocess.PIPE,
243 stdout=subprocess.PIPE,
244 stdin=subprocess.PIPE,
245 close_fds=True, env=env)
248 poller.register(child.stdout, select.POLLIN)
249 poller.register(child.stderr, select.POLLIN)
253 child.stdout.fileno(): (out, child.stdout),
254 child.stderr.fileno(): (err, child.stderr),
257 status = fcntl.fcntl(fd, fcntl.F_GETFL)
258 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
261 for fd, event in poller.poll():
262 if event & select.POLLIN or event & select.POLLPRI:
263 data = fdmap[fd][1].read()
264 # no data from read signifies EOF (the same as POLLHUP)
266 poller.unregister(fd)
269 fdmap[fd][0].write(data)
270 if (event & select.POLLNVAL or event & select.POLLHUP or
271 event & select.POLLERR):
272 poller.unregister(fd)
278 status = child.wait()
286 return RunResult(exitcode, signal, out, err, strcmd)
289 def RunCmdUnlocked(cmd):
290 """Execute a shell command without the 'cmd' lock.
292 This variant of `RunCmd()` drops the 'cmd' lock before running the
293 command and re-aquires it afterwards, thus it can be used to call
294 other ganeti commands.
296 The argument and return values are the same as for the `RunCmd()`
300 cmd - command to run. (str)
313 def RemoveFile(filename):
314 """Remove a file ignoring some errors.
316 Remove a file, ignoring non-existing ones or directories. Other
323 if err.errno not in (errno.ENOENT, errno.EISDIR):
327 def _FingerprintFile(filename):
328 """Compute the fingerprint of a file.
330 If the file does not exist, a None will be returned
334 filename - Filename (str)
337 if not (os.path.exists(filename) and os.path.isfile(filename)):
350 return fp.hexdigest()
353 def FingerprintFiles(files):
354 """Compute fingerprints for a list of files.
357 files - array of filenames. ( [str, ...] )
360 dictionary of filename: fingerprint for the files that exist
365 for filename in files:
366 cksum = _FingerprintFile(filename)
368 ret[filename] = cksum
373 def CheckDict(target, template, logname=None):
374 """Ensure a dictionary has a required set of keys.
376 For the given dictionaries `target` and `template`, ensure target
377 has all the keys from template. Missing keys are added with values
381 target - the dictionary to check
382 template - template dictionary
383 logname - a caller-chosen string to identify the debug log
384 entry; if None, no logging will be done
394 target[k] = template[k]
396 if missing and logname:
397 logger.Debug('%s missing keys %s' %
398 (logname, ', '.join(missing)))
401 def IsProcessAlive(pid):
402 """Check if a given pid exists on the system.
404 Returns: true or false, depending on if the pid exists or not
406 Remarks: zombie processes treated as not alive
410 f = open("/proc/%d/status" % pid)
412 if err.errno in (errno.ENOENT, errno.ENOTDIR):
419 state = data[1].split()
420 if len(state) > 1 and state[1] == "Z":
428 def MatchNameComponent(key, name_list):
429 """Try to match a name against a list.
431 This function will try to match a name like test1 against a list
432 like ['test1.example.com', 'test2.example.com', ...]. Against this
433 list, 'test1' as well as 'test1.example' will match, but not
434 'test1.ex'. A multiple match will be considered as no match at all
435 (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).
438 key: the name to be searched
439 name_list: the list of strings against which to search the key
442 None if there is no match *or* if there are multiple matches
443 otherwise the element from the list which matches
446 mo = re.compile("^%s(\..*)?$" % re.escape(key))
447 names_filtered = [name for name in name_list if mo.match(name) is not None]
448 if len(names_filtered) != 1:
450 return names_filtered[0]
454 """Class implementing resolver and hostname functionality
457 def __init__(self, name=None):
458 """Initialize the host name object.
460 If the name argument is not passed, it will use this system's
465 name = self.SysName()
468 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
469 self.ip = self.ipaddrs[0]
472 """Returns the hostname without domain.
475 return self.name.split('.')[0]
479 """Return the current system's name.
481 This is simply a wrapper over socket.gethostname()
484 return socket.gethostname()
487 def LookupHostname(hostname):
491 hostname: hostname to look up
494 a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
495 in case of errors in resolving, we raise a ResolverError
499 result = socket.gethostbyname_ex(hostname)
500 except socket.gaierror, err:
501 # hostname not found in DNS
502 raise errors.ResolverError(hostname, err.args[0], err.args[1])
507 def ListVolumeGroups():
508 """List volume groups and their size
511 Dictionary with keys volume name and values the size of the volume
514 command = "vgs --noheadings --units m --nosuffix -o name,size"
515 result = RunCmd(command)
520 for line in result.stdout.splitlines():
522 name, size = line.split()
523 size = int(float(size))
524 except (IndexError, ValueError), err:
525 logger.Error("Invalid output from vgs (%s): %s" % (err, line))
533 def BridgeExists(bridge):
534 """Check whether the given bridge exists in the system
537 True if it does, false otherwise.
540 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
543 def NiceSort(name_list):
544 """Sort a list of strings based on digit and non-digit groupings.
546 Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
547 sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].
549 The sort algorithm breaks each name in groups of either only-digits
550 or no-digits. Only the first eight such groups are considered, and
551 after that we just use what's left of the string.
554 - a copy of the list sorted according to our algorithm
557 _SORTER_BASE = "(\D+|\d+)"
558 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
559 _SORTER_BASE, _SORTER_BASE,
560 _SORTER_BASE, _SORTER_BASE,
561 _SORTER_BASE, _SORTER_BASE)
562 _SORTER_RE = re.compile(_SORTER_FULL)
563 _SORTER_NODIGIT = re.compile("^\D*$")
565 """Attempts to convert a variable to integer."""
566 if val is None or _SORTER_NODIGIT.match(val):
571 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
572 for name in name_list]
574 return [tup[1] for tup in to_sort]
577 def CheckDaemonAlive(pid_file, process_string):
578 """Check wether the specified daemon is alive.
581 - pid_file: file to read the daemon pid from, the file is
582 expected to contain only a single line containing
584 - process_string: a substring that we expect to find in
585 the command line of the daemon process
588 - True if the daemon is judged to be alive (that is:
589 - the PID file exists, is readable and contains a number
590 - a process of the specified PID is running
591 - that process contains the specified string in its
593 - the process is not in state Z (zombie))
598 pid_file = file(pid_file, 'r')
600 pid = int(pid_file.readline())
604 cmdline_file_path = "/proc/%s/cmdline" % (pid)
605 cmdline_file = open(cmdline_file_path, 'r')
607 cmdline = cmdline_file.readline()
611 if not process_string in cmdline:
614 stat_file_path = "/proc/%s/stat" % (pid)
615 stat_file = open(stat_file_path, 'r')
617 process_state = stat_file.readline().split()[2]
621 if process_state == 'Z':
624 except (IndexError, IOError, ValueError):
630 def TryConvert(fn, val):
631 """Try to convert a value ignoring errors.
633 This function tries to apply function `fn` to `val`. If no
634 ValueError or TypeError exceptions are raised, it will return the
635 result, else it will return the original value. Any other exceptions
636 are propagated to the caller.
641 except (ValueError, TypeError), err:
647 """Verifies the syntax of an IP address.
649 This function checks if the ip address passes is valid or not based
650 on syntax (not ip range, class calculations or anything).
653 unit = "(0|[1-9]\d{0,2})"
654 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
657 def IsValidShellParam(word):
658 """Verifies is the given word is safe from the shell's p.o.v.
660 This means that we can pass this to a command via the shell and be
661 sure that it doesn't alter the command line and is passed as such to
664 Note that we are overly restrictive here, in order to be on the safe
668 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
671 def BuildShellCmd(template, *args):
672 """Build a safe shell command line from the given arguments.
674 This function will check all arguments in the args list so that they
675 are valid shell parameters (i.e. they don't contain shell
676 metacharaters). If everything is ok, it will return the result of
681 if not IsValidShellParam(word):
682 raise errors.ProgrammerError("Shell argument '%s' contains"
683 " invalid characters" % word)
684 return template % args
687 def FormatUnit(value):
688 """Formats an incoming number of MiB with the appropriate unit.
690 Value needs to be passed as a numeric type. Return value is always a string.
694 return "%dM" % round(value, 0)
696 elif value < (1024 * 1024):
697 return "%0.1fG" % round(float(value) / 1024, 1)
700 return "%0.1fT" % round(float(value) / 1024 / 1024, 1)
703 def ParseUnit(input_string):
704 """Tries to extract number and scale from the given string.
706 Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
707 is specified, it defaults to MiB. Return value is always an int in MiB.
710 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
712 raise errors.UnitParseError("Invalid format")
714 value = float(m.groups()[0])
718 lcunit = unit.lower()
722 if lcunit in ('m', 'mb', 'mib'):
723 # Value already in MiB
726 elif lcunit in ('g', 'gb', 'gib'):
729 elif lcunit in ('t', 'tb', 'tib'):
733 raise errors.UnitParseError("Unknown unit: %s" % unit)
735 # Make sure we round up
736 if int(value) < value:
739 # Round up to the next multiple of 4
742 value += 4 - value % 4
747 def AddAuthorizedKey(file_name, key):
748 """Adds an SSH public key to an authorized_keys file.
751 file_name: Path to authorized_keys file
752 key: String containing key
754 key_fields = key.split()
756 f = open(file_name, 'a+')
760 # Ignore whitespace changes
761 if line.split() == key_fields:
763 nl = line.endswith('\n')
767 f.write(key.rstrip('\r\n'))
774 def RemoveAuthorizedKey(file_name, key):
775 """Removes an SSH public key from an authorized_keys file.
778 file_name: Path to authorized_keys file
779 key: String containing key
781 key_fields = key.split()
783 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
785 out = os.fdopen(fd, 'w')
787 f = open(file_name, 'r')
790 # Ignore whitespace changes while comparing lines
791 if line.split() != key_fields:
795 os.rename(tmpname, file_name)
805 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
806 """Sets the name of an IP address and hostname in /etc/hosts.
809 # Ensure aliases are unique
810 aliases = UniqueSequence([hostname] + aliases)[1:]
812 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
814 out = os.fdopen(fd, 'w')
816 f = open(file_name, 'r')
820 fields = line.split()
821 if fields and not fields[0].startswith('#') and ip == fields[0]:
825 out.write("%s\t%s" % (ip, hostname))
827 out.write(" %s" % ' '.join(aliases))
832 os.rename(tmpname, file_name)
842 def RemoveEtcHostsEntry(file_name, hostname):
843 """Removes a hostname from /etc/hosts.
845 IP addresses without names are removed from the file.
847 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
849 out = os.fdopen(fd, 'w')
851 f = open(file_name, 'r')
854 fields = line.split()
855 if len(fields) > 1 and not fields[0].startswith('#'):
857 if hostname in names:
858 while hostname in names:
859 names.remove(hostname)
861 out.write("%s %s\n" % (fields[0], ' '.join(names)))
868 os.rename(tmpname, file_name)
878 def CreateBackup(file_name):
879 """Creates a backup of a file.
881 Returns: the path to the newly created backup file.
884 if not os.path.isfile(file_name):
885 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
888 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
889 dir_name = os.path.dirname(file_name)
891 fsrc = open(file_name, 'rb')
893 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
894 fdst = os.fdopen(fd, 'wb')
896 shutil.copyfileobj(fsrc, fdst)
905 def ShellQuote(value):
906 """Quotes shell argument according to POSIX.
909 if _re_shell_unquoted.match(value):
912 return "'%s'" % value.replace("'", "'\\''")
915 def ShellQuoteArgs(args):
916 """Quotes all given shell arguments and concatenates using spaces.
919 return ' '.join([ShellQuote(i) for i in args])
922 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
923 """Simple ping implementation using TCP connect(2).
925 Try to do a TCP connect(2) from an optional source IP to the
926 specified target IP and the specified target port. If the optional
927 parameter live_port_needed is set to true, requires the remote end
928 to accept the connection. The timeout is specified in seconds and
929 defaults to 10 seconds. If the source optional argument is not
930 passed, the source address selection is left to the kernel,
931 otherwise we try to connect using the passed address (failures to
932 bind other than EADDRNOTAVAIL will be ignored).
935 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
939 if source is not None:
941 sock.bind((source, 0))
942 except socket.error, (errcode, errstring):
943 if errcode == errno.EADDRNOTAVAIL:
946 sock.settimeout(timeout)
949 sock.connect((target, port))
952 except socket.timeout:
954 except socket.error, (errcode, errstring):
955 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
960 def ListVisibleFiles(path):
961 """Returns a list of all visible files in a directory.
964 files = [i for i in os.listdir(path) if not i.startswith(".")]
969 def GetHomeDir(user, default=None):
970 """Try to get the homedir of the given user.
972 The user can be passed either as a string (denoting the name) or as
973 an integer (denoting the user id). If the user is not found, the
974 'default' argument is returned, which defaults to None.
978 if isinstance(user, basestring):
979 result = pwd.getpwnam(user)
980 elif isinstance(user, (int, long)):
981 result = pwd.getpwuid(user)
983 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
991 """Returns a random UUID.
994 f = open("/proc/sys/kernel/random/uuid", "r")
996 return f.read(128).rstrip("\n")
1001 def WriteFile(file_name, fn=None, data=None,
1002 mode=None, uid=-1, gid=-1,
1003 atime=None, mtime=None):
1004 """(Over)write a file atomically.
1006 The file_name and either fn (a function taking one argument, the
1007 file descriptor, and which should write the data to it) or data (the
1008 contents of the file) must be passed. The other arguments are
1009 optional and allow setting the file mode, owner and group, and the
1010 mtime/atime of the file.
1012 If the function doesn't raise an exception, it has succeeded and the
1013 target file has the new contents. If the file has raised an
1014 exception, an existing target file should be unmodified and the
1015 temporary file should be removed.
1018 if not os.path.isabs(file_name):
1019 raise errors.ProgrammerError("Path passed to WriteFile is not"
1020 " absolute: '%s'" % file_name)
1022 if [fn, data].count(None) != 1:
1023 raise errors.ProgrammerError("fn or data required")
1025 if [atime, mtime].count(None) == 1:
1026 raise errors.ProgrammerError("Both atime and mtime must be either"
1030 dir_name, base_name = os.path.split(file_name)
1031 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1032 # here we need to make sure we remove the temp file, if any error
1033 # leaves it in place
1035 if uid != -1 or gid != -1:
1036 os.chown(new_name, uid, gid)
1038 os.chmod(new_name, mode)
1039 if data is not None:
1044 if atime is not None and mtime is not None:
1045 os.utime(new_name, (atime, mtime))
1046 os.rename(new_name, file_name)
1049 RemoveFile(new_name)
1052 def all(seq, pred=bool):
1053 "Returns True if pred(x) is True for every element in the iterable"
1054 for elem in itertools.ifilterfalse(pred, seq):
1059 def any(seq, pred=bool):
1060 "Returns True if pred(x) is True for at least one element in the iterable"
1061 for elem in itertools.ifilter(pred, seq):
1066 def UniqueSequence(seq):
1067 """Returns a list with unique elements.
1069 Element order is preserved.
1072 return [i for i in seq if i not in seen and not seen.add(i)]
1075 def IsValidMac(mac):
1076 """Predicate to check if a MAC address is valid.
1078 Checks wether the supplied MAC address is formally correct, only
1079 accepts colon separated format.
1081 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1082 return mac_check.match(mac) is not None
1085 def TestDelay(duration):
1086 """Sleep for a fixed amount of time.
1091 time.sleep(duration)
1095 def FindFile(name, search_path, test=os.path.exists):
1096 """Look for a filesystem object in a given path.
1098 This is an abstract method to search for filesystem object (files,
1099 dirs) under a given search path.
1102 - name: the name to look for
1103 - search_path: list of directory names
1104 - test: the test which the full path must satisfy
1105 (defaults to os.path.exists)
1108 - full path to the item if found
1112 for dir_name in search_path:
1113 item_name = os.path.sep.join([dir_name, name])