4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti small utilities
40 from ganeti import logger
41 from ganeti import errors
42 from ganeti import constants
46 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
50 class RunResult(object):
51 """Simple class for holding the result of running external programs.
54 exit_code: the exit code of the program, or None (if the program
56 signal: numeric signal that caused the program to finish, or None
57 (if the program wasn't terminated by a signal)
58 stdout: the standard output of the program
59 stderr: the standard error of the program
60 failed: a Boolean value which is True in case the program was
61 terminated by a signal or exited with a non-zero exit code
62 fail_reason: a string detailing the termination reason
65 __slots__ = ["exit_code", "signal", "stdout", "stderr",
66 "failed", "fail_reason", "cmd"]
69 def __init__(self, exit_code, signal, stdout, stderr, cmd):
71 self.exit_code = exit_code
75 self.failed = (signal is not None or exit_code != 0)
77 if self.signal is not None:
78 self.fail_reason = "terminated by signal %s" % self.signal
79 elif self.exit_code is not None:
80 self.fail_reason = "exited with exit code %s" % self.exit_code
82 self.fail_reason = "unable to determine termination reason"
84 if debug and self.failed:
85 logger.Debug("Command '%s' failed (%s); output: %s" %
86 (self.cmd, self.fail_reason, self.output))
89 """Returns the combined stdout and stderr for easier usage.
92 return self.stdout + self.stderr
94 output = property(_GetOutput, None, None, "Return full output")
97 def _GetLockFile(subsystem):
98 """Compute the file name for a given lock name."""
99 return "%s/ganeti_lock_%s" % (constants.LOCK_DIR, subsystem)
102 def Lock(name, max_retries=None, debug=False):
103 """Lock a given subsystem.
105 In case the lock is already held by an alive process, the function
106 will sleep indefintely and poll with a one second interval.
108 When the optional integer argument 'max_retries' is passed with a
109 non-zero value, the function will sleep only for this number of
110 times, and then it will will raise a LockError if the lock can't be
111 acquired. Passing in a negative number will cause only one try to
112 get the lock. Passing a positive number will make the function retry
113 for approximately that number of seconds.
116 lockfile = _GetLockFile(name)
118 if name in _locksheld:
119 raise errors.LockError('Lock "%s" already held!' % (name,))
126 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_SYNC)
128 except OSError, creat_err:
129 if creat_err.errno != errno.EEXIST:
130 raise errors.LockError("Can't create the lock file. Error '%s'." %
134 pf = open(lockfile, 'r')
135 except IOError, open_err:
138 raise errors.LockError("Lock file exists but cannot be opened."
139 " Error: '%s'." % str(open_err))
146 raise errors.LockError("Invalid pid string in %s" %
149 if not IsProcessAlive(pid):
150 raise errors.LockError("Stale lockfile %s for pid %d?" %
153 if max_retries and max_retries <= retries:
154 raise errors.LockError("Can't acquire lock during the specified"
156 if retries == 5 and (debug or sys.stdin.isatty()):
157 logger.ToStderr("Waiting for '%s' lock from pid %d..." % (name, pid))
163 os.write(fd, '%d\n' % (os.getpid(),))
166 _locksheld.append(name)
170 """Unlock a given subsystem.
173 lockfile = _GetLockFile(name)
176 fd = os.open(lockfile, os.O_RDONLY)
178 raise errors.LockError('Lock "%s" not held.' % (name,))
180 f = os.fdopen(fd, 'r')
186 raise errors.LockError('Unable to determine PID of locking process.')
188 if pid != os.getpid():
189 raise errors.LockError('Lock not held by me (%d != %d)' %
193 _locksheld.remove(name)
200 for lock in _locksheld:
205 """Execute a (shell) command.
207 The command should not read from its standard input, as it will be
211 cmd: command to run. (str)
213 Returns: `RunResult` instance
216 if isinstance(cmd, list):
217 cmd = [str(val) for val in cmd]
218 strcmd = " ".join(cmd)
223 env = os.environ.copy()
225 child = subprocess.Popen(cmd, shell=shell,
226 stderr=subprocess.PIPE,
227 stdout=subprocess.PIPE,
228 stdin=subprocess.PIPE,
229 close_fds=True, env=env)
232 out = child.stdout.read()
233 err = child.stderr.read()
235 status = child.wait()
243 return RunResult(exitcode, signal, out, err, strcmd)
246 def RunCmdUnlocked(cmd):
247 """Execute a shell command without the 'cmd' lock.
249 This variant of `RunCmd()` drops the 'cmd' lock before running the
250 command and re-aquires it afterwards, thus it can be used to call
251 other ganeti commands.
253 The argument and return values are the same as for the `RunCmd()`
257 cmd - command to run. (str)
270 def RemoveFile(filename):
271 """Remove a file ignoring some errors.
273 Remove a file, ignoring non-existing ones or directories. Other
280 if err.errno not in (errno.ENOENT, errno.EISDIR):
284 def _FingerprintFile(filename):
285 """Compute the fingerprint of a file.
287 If the file does not exist, a None will be returned
291 filename - Filename (str)
294 if not (os.path.exists(filename) and os.path.isfile(filename)):
307 return fp.hexdigest()
310 def FingerprintFiles(files):
311 """Compute fingerprints for a list of files.
314 files - array of filenames. ( [str, ...] )
317 dictionary of filename: fingerprint for the files that exist
322 for filename in files:
323 cksum = _FingerprintFile(filename)
325 ret[filename] = cksum
330 def CheckDict(target, template, logname=None):
331 """Ensure a dictionary has a required set of keys.
333 For the given dictionaries `target` and `template`, ensure target
334 has all the keys from template. Missing keys are added with values
338 target - the dictionary to check
339 template - template dictionary
340 logname - a caller-chosen string to identify the debug log
341 entry; if None, no logging will be done
351 target[k] = template[k]
353 if missing and logname:
354 logger.Debug('%s missing keys %s' %
355 (logname, ', '.join(missing)))
358 def IsProcessAlive(pid):
359 """Check if a given pid exists on the system.
361 Returns: true or false, depending on if the pid exists or not
363 Remarks: zombie processes treated as not alive
367 f = open("/proc/%d/status" % pid)
369 if err.errno in (errno.ENOENT, errno.ENOTDIR):
376 state = data[1].split()
377 if len(state) > 1 and state[1] == "Z":
385 def MatchNameComponent(key, name_list):
386 """Try to match a name against a list.
388 This function will try to match a name like test1 against a list
389 like ['test1.example.com', 'test2.example.com', ...]. Against this
390 list, 'test1' as well as 'test1.example' will match, but not
391 'test1.ex'. A multiple match will be considered as no match at all
392 (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).
395 key: the name to be searched
396 name_list: the list of strings against which to search the key
399 None if there is no match *or* if there are multiple matches
400 otherwise the element from the list which matches
403 mo = re.compile("^%s(\..*)?$" % re.escape(key))
404 names_filtered = [name for name in name_list if mo.match(name) is not None]
405 if len(names_filtered) != 1:
407 return names_filtered[0]
411 """Class implementing resolver and hostname functionality
414 def __init__(self, name=None):
415 """Initialize the host name object.
417 If the name argument is not passed, it will use this system's
422 name = self.SysName()
425 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
426 self.ip = self.ipaddrs[0]
429 """Returns the hostname without domain.
432 return self.name.split('.')[0]
436 """Return the current system's name.
438 This is simply a wrapper over socket.gethostname()
441 return socket.gethostname()
444 def LookupHostname(hostname):
448 hostname: hostname to look up
451 a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
452 in case of errors in resolving, we raise a ResolverError
456 result = socket.gethostbyname_ex(hostname)
457 except socket.gaierror, err:
458 # hostname not found in DNS
459 raise errors.ResolverError(hostname, err.args[0], err.args[1])
464 def ListVolumeGroups():
465 """List volume groups and their size
468 Dictionary with keys volume name and values the size of the volume
471 command = "vgs --noheadings --units m --nosuffix -o name,size"
472 result = RunCmd(command)
477 for line in result.stdout.splitlines():
479 name, size = line.split()
480 size = int(float(size))
481 except (IndexError, ValueError), err:
482 logger.Error("Invalid output from vgs (%s): %s" % (err, line))
490 def BridgeExists(bridge):
491 """Check whether the given bridge exists in the system
494 True if it does, false otherwise.
497 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
500 def NiceSort(name_list):
501 """Sort a list of strings based on digit and non-digit groupings.
503 Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
504 sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].
506 The sort algorithm breaks each name in groups of either only-digits
507 or no-digits. Only the first eight such groups are considered, and
508 after that we just use what's left of the string.
511 - a copy of the list sorted according to our algorithm
514 _SORTER_BASE = "(\D+|\d+)"
515 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
516 _SORTER_BASE, _SORTER_BASE,
517 _SORTER_BASE, _SORTER_BASE,
518 _SORTER_BASE, _SORTER_BASE)
519 _SORTER_RE = re.compile(_SORTER_FULL)
520 _SORTER_NODIGIT = re.compile("^\D*$")
522 """Attempts to convert a variable to integer."""
523 if val is None or _SORTER_NODIGIT.match(val):
528 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
529 for name in name_list]
531 return [tup[1] for tup in to_sort]
534 def CheckDaemonAlive(pid_file, process_string):
535 """Check wether the specified daemon is alive.
538 - pid_file: file to read the daemon pid from, the file is
539 expected to contain only a single line containing
541 - process_string: a substring that we expect to find in
542 the command line of the daemon process
545 - True if the daemon is judged to be alive (that is:
546 - the PID file exists, is readable and contains a number
547 - a process of the specified PID is running
548 - that process contains the specified string in its
550 - the process is not in state Z (zombie))
555 pid_file = file(pid_file, 'r')
557 pid = int(pid_file.readline())
561 cmdline_file_path = "/proc/%s/cmdline" % (pid)
562 cmdline_file = open(cmdline_file_path, 'r')
564 cmdline = cmdline_file.readline()
568 if not process_string in cmdline:
571 stat_file_path = "/proc/%s/stat" % (pid)
572 stat_file = open(stat_file_path, 'r')
574 process_state = stat_file.readline().split()[2]
578 if process_state == 'Z':
581 except (IndexError, IOError, ValueError):
587 def TryConvert(fn, val):
588 """Try to convert a value ignoring errors.
590 This function tries to apply function `fn` to `val`. If no
591 ValueError or TypeError exceptions are raised, it will return the
592 result, else it will return the original value. Any other exceptions
593 are propagated to the caller.
598 except (ValueError, TypeError), err:
604 """Verifies the syntax of an IP address.
606 This function checks if the ip address passes is valid or not based
607 on syntax (not ip range, class calculations or anything).
610 unit = "(0|[1-9]\d{0,2})"
611 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
614 def IsValidShellParam(word):
615 """Verifies is the given word is safe from the shell's p.o.v.
617 This means that we can pass this to a command via the shell and be
618 sure that it doesn't alter the command line and is passed as such to
621 Note that we are overly restrictive here, in order to be on the safe
625 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
628 def BuildShellCmd(template, *args):
629 """Build a safe shell command line from the given arguments.
631 This function will check all arguments in the args list so that they
632 are valid shell parameters (i.e. they don't contain shell
633 metacharaters). If everything is ok, it will return the result of
638 if not IsValidShellParam(word):
639 raise errors.ProgrammerError("Shell argument '%s' contains"
640 " invalid characters" % word)
641 return template % args
644 def FormatUnit(value):
645 """Formats an incoming number of MiB with the appropriate unit.
647 Value needs to be passed as a numeric type. Return value is always a string.
651 return "%dM" % round(value, 0)
653 elif value < (1024 * 1024):
654 return "%0.1fG" % round(float(value) / 1024, 1)
657 return "%0.1fT" % round(float(value) / 1024 / 1024, 1)
660 def ParseUnit(input_string):
661 """Tries to extract number and scale from the given string.
663 Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
664 is specified, it defaults to MiB. Return value is always an int in MiB.
667 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
669 raise errors.UnitParseError("Invalid format")
671 value = float(m.groups()[0])
675 lcunit = unit.lower()
679 if lcunit in ('m', 'mb', 'mib'):
680 # Value already in MiB
683 elif lcunit in ('g', 'gb', 'gib'):
686 elif lcunit in ('t', 'tb', 'tib'):
690 raise errors.UnitParseError("Unknown unit: %s" % unit)
692 # Make sure we round up
693 if int(value) < value:
696 # Round up to the next multiple of 4
699 value += 4 - value % 4
704 def AddAuthorizedKey(file_name, key):
705 """Adds an SSH public key to an authorized_keys file.
708 file_name: Path to authorized_keys file
709 key: String containing key
711 key_fields = key.split()
713 f = open(file_name, 'a+')
717 # Ignore whitespace changes
718 if line.split() == key_fields:
720 nl = line.endswith('\n')
724 f.write(key.rstrip('\r\n'))
731 def RemoveAuthorizedKey(file_name, key):
732 """Removes an SSH public key from an authorized_keys file.
735 file_name: Path to authorized_keys file
736 key: String containing key
738 key_fields = key.split()
740 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
742 out = os.fdopen(fd, 'w')
744 f = open(file_name, 'r')
747 # Ignore whitespace changes while comparing lines
748 if line.split() != key_fields:
752 os.rename(tmpname, file_name)
762 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
763 """Sets the name of an IP address and hostname in /etc/hosts.
766 # Ensure aliases are unique
767 aliases = UniqueSequence([hostname] + aliases)[1:]
769 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
771 out = os.fdopen(fd, 'w')
773 f = open(file_name, 'r')
777 fields = line.split()
778 if fields and not fields[0].startswith('#') and ip == fields[0]:
782 out.write("%s\t%s" % (ip, hostname))
784 out.write(" %s" % ' '.join(aliases))
789 os.rename(tmpname, file_name)
799 def RemoveEtcHostsEntry(file_name, hostname):
800 """Removes a hostname from /etc/hosts.
802 IP addresses without names are removed from the file.
804 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
806 out = os.fdopen(fd, 'w')
808 f = open(file_name, 'r')
811 fields = line.split()
812 if len(fields) > 1 and not fields[0].startswith('#'):
814 if hostname in names:
815 while hostname in names:
816 names.remove(hostname)
818 out.write("%s %s\n" % (fields[0], ' '.join(names)))
825 os.rename(tmpname, file_name)
835 def CreateBackup(file_name):
836 """Creates a backup of a file.
838 Returns: the path to the newly created backup file.
841 if not os.path.isfile(file_name):
842 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
845 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
846 dir_name = os.path.dirname(file_name)
848 fsrc = open(file_name, 'rb')
850 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
851 fdst = os.fdopen(fd, 'wb')
853 shutil.copyfileobj(fsrc, fdst)
862 def ShellQuote(value):
863 """Quotes shell argument according to POSIX.
866 if _re_shell_unquoted.match(value):
869 return "'%s'" % value.replace("'", "'\\''")
872 def ShellQuoteArgs(args):
873 """Quotes all given shell arguments and concatenates using spaces.
876 return ' '.join([ShellQuote(i) for i in args])
880 def TcpPing(source, target, port, timeout=10, live_port_needed=False):
881 """Simple ping implementation using TCP connect(2).
883 Try to do a TCP connect(2) from the specified source IP to the specified
884 target IP and the specified target port. If live_port_needed is set to true,
885 requires the remote end to accept the connection. The timeout is specified
886 in seconds and defaults to 10 seconds
889 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
894 sock.bind((source, 0))
895 except socket.error, (errcode, errstring):
896 if errcode == errno.EADDRNOTAVAIL:
899 sock.settimeout(timeout)
902 sock.connect((target, port))
905 except socket.timeout:
907 except socket.error, (errcode, errstring):
908 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
913 def ListVisibleFiles(path):
914 """Returns a list of all visible files in a directory.
917 files = [i for i in os.listdir(path) if not i.startswith(".")]
922 def GetHomeDir(user, default=None):
923 """Try to get the homedir of the given user.
925 The user can be passed either as a string (denoting the name) or as
926 an integer (denoting the user id). If the user is not found, the
927 'default' argument is returned, which defaults to None.
931 if isinstance(user, basestring):
932 result = pwd.getpwnam(user)
933 elif isinstance(user, (int, long)):
934 result = pwd.getpwuid(user)
936 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
944 """Returns a random UUID.
947 f = open("/proc/sys/kernel/random/uuid", "r")
949 return f.read(128).rstrip("\n")
954 def WriteFile(file_name, fn=None, data=None,
955 mode=None, uid=-1, gid=-1,
956 atime=None, mtime=None):
957 """(Over)write a file atomically.
959 The file_name and either fn (a function taking one argument, the
960 file descriptor, and which should write the data to it) or data (the
961 contents of the file) must be passed. The other arguments are
962 optional and allow setting the file mode, owner and group, and the
963 mtime/atime of the file.
965 If the function doesn't raise an exception, it has succeeded and the
966 target file has the new contents. If the file has raised an
967 exception, an existing target file should be unmodified and the
968 temporary file should be removed.
971 if not os.path.isabs(file_name):
972 raise errors.ProgrammerError("Path passed to WriteFile is not"
973 " absolute: '%s'" % file_name)
975 if [fn, data].count(None) != 1:
976 raise errors.ProgrammerError("fn or data required")
978 if [atime, mtime].count(None) == 1:
979 raise errors.ProgrammerError("Both atime and mtime must be either"
983 dir_name, base_name = os.path.split(file_name)
984 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
985 # here we need to make sure we remove the temp file, if any error
988 if uid != -1 or gid != -1:
989 os.chown(new_name, uid, gid)
991 os.chmod(new_name, mode)
997 if atime is not None and mtime is not None:
998 os.utime(new_name, (atime, mtime))
999 os.rename(new_name, file_name)
1002 RemoveFile(new_name)
1005 def all(seq, pred=bool):
1006 "Returns True if pred(x) is True for every element in the iterable"
1007 for elem in itertools.ifilterfalse(pred, seq):
1012 def any(seq, pred=bool):
1013 "Returns True if pred(x) is True for at least one element in the iterable"
1014 for elem in itertools.ifilter(pred, seq):
1019 def UniqueSequence(seq):
1020 """Returns a list with unique elements.
1022 Element order is preserved.
1025 return [i for i in seq if i not in seen and not seen.add(i)]