4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti small utilities
42 from cStringIO import StringIO
44 from ganeti import logger
45 from ganeti import errors
46 from ganeti import constants
50 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
54 class RunResult(object):
55 """Simple class for holding the result of running external programs.
58 exit_code: the exit code of the program, or None (if the program
60 signal: numeric signal that caused the program to finish, or None
61 (if the program wasn't terminated by a signal)
62 stdout: the standard output of the program
63 stderr: the standard error of the program
64 failed: a Boolean value which is True in case the program was
65 terminated by a signal or exited with a non-zero exit code
66 fail_reason: a string detailing the termination reason
69 __slots__ = ["exit_code", "signal", "stdout", "stderr",
70 "failed", "fail_reason", "cmd"]
73 def __init__(self, exit_code, signal, stdout, stderr, cmd):
75 self.exit_code = exit_code
79 self.failed = (signal is not None or exit_code != 0)
81 if self.signal is not None:
82 self.fail_reason = "terminated by signal %s" % self.signal
83 elif self.exit_code is not None:
84 self.fail_reason = "exited with exit code %s" % self.exit_code
86 self.fail_reason = "unable to determine termination reason"
88 if debug and self.failed:
89 logger.Debug("Command '%s' failed (%s); output: %s" %
90 (self.cmd, self.fail_reason, self.output))
93 """Returns the combined stdout and stderr for easier usage.
96 return self.stdout + self.stderr
98 output = property(_GetOutput, None, None, "Return full output")
101 def _GetLockFile(subsystem):
102 """Compute the file name for a given lock name."""
103 return "%s/ganeti_lock_%s" % (constants.LOCK_DIR, subsystem)
106 def Lock(name, max_retries=None, debug=False):
107 """Lock a given subsystem.
109 In case the lock is already held by an alive process, the function
110 will sleep indefintely and poll with a one second interval.
112 When the optional integer argument 'max_retries' is passed with a
113 non-zero value, the function will sleep only for this number of
114 times, and then it will will raise a LockError if the lock can't be
115 acquired. Passing in a negative number will cause only one try to
116 get the lock. Passing a positive number will make the function retry
117 for approximately that number of seconds.
120 lockfile = _GetLockFile(name)
122 if name in _locksheld:
123 raise errors.LockError('Lock "%s" already held!' % (name,))
130 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_SYNC)
132 except OSError, creat_err:
133 if creat_err.errno != errno.EEXIST:
134 raise errors.LockError("Can't create the lock file. Error '%s'." %
138 pf = open(lockfile, 'r')
139 except IOError, open_err:
142 raise errors.LockError("Lock file exists but cannot be opened."
143 " Error: '%s'." % str(open_err))
150 raise errors.LockError("Invalid pid string in %s" %
153 if not IsProcessAlive(pid):
154 raise errors.LockError("Stale lockfile %s for pid %d?" %
157 if max_retries and max_retries <= retries:
158 raise errors.LockError("Can't acquire lock during the specified"
160 if retries == 5 and (debug or sys.stdin.isatty()):
161 logger.ToStderr("Waiting for '%s' lock from pid %d..." % (name, pid))
167 os.write(fd, '%d\n' % (os.getpid(),))
170 _locksheld.append(name)
174 """Unlock a given subsystem.
177 lockfile = _GetLockFile(name)
180 fd = os.open(lockfile, os.O_RDONLY)
182 raise errors.LockError('Lock "%s" not held.' % (name,))
184 f = os.fdopen(fd, 'r')
190 raise errors.LockError('Unable to determine PID of locking process.')
192 if pid != os.getpid():
193 raise errors.LockError('Lock not held by me (%d != %d)' %
197 _locksheld.remove(name)
204 for lock in _locksheld:
209 """Execute a (shell) command.
211 The command should not read from its standard input, as it will be
215 cmd: command to run. (str)
217 Returns: `RunResult` instance
220 if isinstance(cmd, list):
221 cmd = [str(val) for val in cmd]
222 strcmd = " ".join(cmd)
227 env = os.environ.copy()
229 poller = select.poll()
230 child = subprocess.Popen(cmd, shell=shell,
231 stderr=subprocess.PIPE,
232 stdout=subprocess.PIPE,
233 stdin=subprocess.PIPE,
234 close_fds=True, env=env)
237 poller.register(child.stdout, select.POLLIN)
238 poller.register(child.stderr, select.POLLIN)
242 child.stdout.fileno(): (out, child.stdout),
243 child.stderr.fileno(): (err, child.stderr),
246 status = fcntl.fcntl(fd, fcntl.F_GETFL)
247 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
250 for fd, event in poller.poll():
251 if event & select.POLLIN or event & select.POLLPRI:
252 data = fdmap[fd][1].read()
253 # no data from read signifies EOF (the same as POLLHUP)
255 poller.unregister(fd)
258 fdmap[fd][0].write(data)
259 if (event & select.POLLNVAL or event & select.POLLHUP or
260 event & select.POLLERR):
261 poller.unregister(fd)
267 status = child.wait()
275 return RunResult(exitcode, signal, out, err, strcmd)
278 def RunCmdUnlocked(cmd):
279 """Execute a shell command without the 'cmd' lock.
281 This variant of `RunCmd()` drops the 'cmd' lock before running the
282 command and re-aquires it afterwards, thus it can be used to call
283 other ganeti commands.
285 The argument and return values are the same as for the `RunCmd()`
289 cmd - command to run. (str)
302 def RemoveFile(filename):
303 """Remove a file ignoring some errors.
305 Remove a file, ignoring non-existing ones or directories. Other
312 if err.errno not in (errno.ENOENT, errno.EISDIR):
316 def _FingerprintFile(filename):
317 """Compute the fingerprint of a file.
319 If the file does not exist, a None will be returned
323 filename - Filename (str)
326 if not (os.path.exists(filename) and os.path.isfile(filename)):
339 return fp.hexdigest()
342 def FingerprintFiles(files):
343 """Compute fingerprints for a list of files.
346 files - array of filenames. ( [str, ...] )
349 dictionary of filename: fingerprint for the files that exist
354 for filename in files:
355 cksum = _FingerprintFile(filename)
357 ret[filename] = cksum
362 def CheckDict(target, template, logname=None):
363 """Ensure a dictionary has a required set of keys.
365 For the given dictionaries `target` and `template`, ensure target
366 has all the keys from template. Missing keys are added with values
370 target - the dictionary to check
371 template - template dictionary
372 logname - a caller-chosen string to identify the debug log
373 entry; if None, no logging will be done
383 target[k] = template[k]
385 if missing and logname:
386 logger.Debug('%s missing keys %s' %
387 (logname, ', '.join(missing)))
390 def IsProcessAlive(pid):
391 """Check if a given pid exists on the system.
393 Returns: true or false, depending on if the pid exists or not
395 Remarks: zombie processes treated as not alive
399 f = open("/proc/%d/status" % pid)
401 if err.errno in (errno.ENOENT, errno.ENOTDIR):
408 state = data[1].split()
409 if len(state) > 1 and state[1] == "Z":
417 def MatchNameComponent(key, name_list):
418 """Try to match a name against a list.
420 This function will try to match a name like test1 against a list
421 like ['test1.example.com', 'test2.example.com', ...]. Against this
422 list, 'test1' as well as 'test1.example' will match, but not
423 'test1.ex'. A multiple match will be considered as no match at all
424 (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).
427 key: the name to be searched
428 name_list: the list of strings against which to search the key
431 None if there is no match *or* if there are multiple matches
432 otherwise the element from the list which matches
435 mo = re.compile("^%s(\..*)?$" % re.escape(key))
436 names_filtered = [name for name in name_list if mo.match(name) is not None]
437 if len(names_filtered) != 1:
439 return names_filtered[0]
443 """Class implementing resolver and hostname functionality
446 def __init__(self, name=None):
447 """Initialize the host name object.
449 If the name argument is not passed, it will use this system's
454 name = self.SysName()
457 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
458 self.ip = self.ipaddrs[0]
461 """Returns the hostname without domain.
464 return self.name.split('.')[0]
468 """Return the current system's name.
470 This is simply a wrapper over socket.gethostname()
473 return socket.gethostname()
476 def LookupHostname(hostname):
480 hostname: hostname to look up
483 a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
484 in case of errors in resolving, we raise a ResolverError
488 result = socket.gethostbyname_ex(hostname)
489 except socket.gaierror, err:
490 # hostname not found in DNS
491 raise errors.ResolverError(hostname, err.args[0], err.args[1])
496 def ListVolumeGroups():
497 """List volume groups and their size
500 Dictionary with keys volume name and values the size of the volume
503 command = "vgs --noheadings --units m --nosuffix -o name,size"
504 result = RunCmd(command)
509 for line in result.stdout.splitlines():
511 name, size = line.split()
512 size = int(float(size))
513 except (IndexError, ValueError), err:
514 logger.Error("Invalid output from vgs (%s): %s" % (err, line))
522 def BridgeExists(bridge):
523 """Check whether the given bridge exists in the system
526 True if it does, false otherwise.
529 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
532 def NiceSort(name_list):
533 """Sort a list of strings based on digit and non-digit groupings.
535 Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
536 sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].
538 The sort algorithm breaks each name in groups of either only-digits
539 or no-digits. Only the first eight such groups are considered, and
540 after that we just use what's left of the string.
543 - a copy of the list sorted according to our algorithm
546 _SORTER_BASE = "(\D+|\d+)"
547 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
548 _SORTER_BASE, _SORTER_BASE,
549 _SORTER_BASE, _SORTER_BASE,
550 _SORTER_BASE, _SORTER_BASE)
551 _SORTER_RE = re.compile(_SORTER_FULL)
552 _SORTER_NODIGIT = re.compile("^\D*$")
554 """Attempts to convert a variable to integer."""
555 if val is None or _SORTER_NODIGIT.match(val):
560 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
561 for name in name_list]
563 return [tup[1] for tup in to_sort]
566 def CheckDaemonAlive(pid_file, process_string):
567 """Check wether the specified daemon is alive.
570 - pid_file: file to read the daemon pid from, the file is
571 expected to contain only a single line containing
573 - process_string: a substring that we expect to find in
574 the command line of the daemon process
577 - True if the daemon is judged to be alive (that is:
578 - the PID file exists, is readable and contains a number
579 - a process of the specified PID is running
580 - that process contains the specified string in its
582 - the process is not in state Z (zombie))
587 pid_file = file(pid_file, 'r')
589 pid = int(pid_file.readline())
593 cmdline_file_path = "/proc/%s/cmdline" % (pid)
594 cmdline_file = open(cmdline_file_path, 'r')
596 cmdline = cmdline_file.readline()
600 if not process_string in cmdline:
603 stat_file_path = "/proc/%s/stat" % (pid)
604 stat_file = open(stat_file_path, 'r')
606 process_state = stat_file.readline().split()[2]
610 if process_state == 'Z':
613 except (IndexError, IOError, ValueError):
619 def TryConvert(fn, val):
620 """Try to convert a value ignoring errors.
622 This function tries to apply function `fn` to `val`. If no
623 ValueError or TypeError exceptions are raised, it will return the
624 result, else it will return the original value. Any other exceptions
625 are propagated to the caller.
630 except (ValueError, TypeError), err:
636 """Verifies the syntax of an IP address.
638 This function checks if the ip address passes is valid or not based
639 on syntax (not ip range, class calculations or anything).
642 unit = "(0|[1-9]\d{0,2})"
643 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
646 def IsValidShellParam(word):
647 """Verifies is the given word is safe from the shell's p.o.v.
649 This means that we can pass this to a command via the shell and be
650 sure that it doesn't alter the command line and is passed as such to
653 Note that we are overly restrictive here, in order to be on the safe
657 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
660 def BuildShellCmd(template, *args):
661 """Build a safe shell command line from the given arguments.
663 This function will check all arguments in the args list so that they
664 are valid shell parameters (i.e. they don't contain shell
665 metacharaters). If everything is ok, it will return the result of
670 if not IsValidShellParam(word):
671 raise errors.ProgrammerError("Shell argument '%s' contains"
672 " invalid characters" % word)
673 return template % args
676 def FormatUnit(value):
677 """Formats an incoming number of MiB with the appropriate unit.
679 Value needs to be passed as a numeric type. Return value is always a string.
683 return "%dM" % round(value, 0)
685 elif value < (1024 * 1024):
686 return "%0.1fG" % round(float(value) / 1024, 1)
689 return "%0.1fT" % round(float(value) / 1024 / 1024, 1)
692 def ParseUnit(input_string):
693 """Tries to extract number and scale from the given string.
695 Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
696 is specified, it defaults to MiB. Return value is always an int in MiB.
699 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
701 raise errors.UnitParseError("Invalid format")
703 value = float(m.groups()[0])
707 lcunit = unit.lower()
711 if lcunit in ('m', 'mb', 'mib'):
712 # Value already in MiB
715 elif lcunit in ('g', 'gb', 'gib'):
718 elif lcunit in ('t', 'tb', 'tib'):
722 raise errors.UnitParseError("Unknown unit: %s" % unit)
724 # Make sure we round up
725 if int(value) < value:
728 # Round up to the next multiple of 4
731 value += 4 - value % 4
736 def AddAuthorizedKey(file_name, key):
737 """Adds an SSH public key to an authorized_keys file.
740 file_name: Path to authorized_keys file
741 key: String containing key
743 key_fields = key.split()
745 f = open(file_name, 'a+')
749 # Ignore whitespace changes
750 if line.split() == key_fields:
752 nl = line.endswith('\n')
756 f.write(key.rstrip('\r\n'))
763 def RemoveAuthorizedKey(file_name, key):
764 """Removes an SSH public key from an authorized_keys file.
767 file_name: Path to authorized_keys file
768 key: String containing key
770 key_fields = key.split()
772 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
774 out = os.fdopen(fd, 'w')
776 f = open(file_name, 'r')
779 # Ignore whitespace changes while comparing lines
780 if line.split() != key_fields:
784 os.rename(tmpname, file_name)
794 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
795 """Sets the name of an IP address and hostname in /etc/hosts.
798 # Ensure aliases are unique
799 aliases = UniqueSequence([hostname] + aliases)[1:]
801 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
803 out = os.fdopen(fd, 'w')
805 f = open(file_name, 'r')
809 fields = line.split()
810 if fields and not fields[0].startswith('#') and ip == fields[0]:
814 out.write("%s\t%s" % (ip, hostname))
816 out.write(" %s" % ' '.join(aliases))
821 os.rename(tmpname, file_name)
831 def RemoveEtcHostsEntry(file_name, hostname):
832 """Removes a hostname from /etc/hosts.
834 IP addresses without names are removed from the file.
836 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
838 out = os.fdopen(fd, 'w')
840 f = open(file_name, 'r')
843 fields = line.split()
844 if len(fields) > 1 and not fields[0].startswith('#'):
846 if hostname in names:
847 while hostname in names:
848 names.remove(hostname)
850 out.write("%s %s\n" % (fields[0], ' '.join(names)))
857 os.rename(tmpname, file_name)
867 def CreateBackup(file_name):
868 """Creates a backup of a file.
870 Returns: the path to the newly created backup file.
873 if not os.path.isfile(file_name):
874 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
877 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
878 dir_name = os.path.dirname(file_name)
880 fsrc = open(file_name, 'rb')
882 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
883 fdst = os.fdopen(fd, 'wb')
885 shutil.copyfileobj(fsrc, fdst)
894 def ShellQuote(value):
895 """Quotes shell argument according to POSIX.
898 if _re_shell_unquoted.match(value):
901 return "'%s'" % value.replace("'", "'\\''")
904 def ShellQuoteArgs(args):
905 """Quotes all given shell arguments and concatenates using spaces.
908 return ' '.join([ShellQuote(i) for i in args])
911 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
912 """Simple ping implementation using TCP connect(2).
914 Try to do a TCP connect(2) from an optional source IP to the
915 specified target IP and the specified target port. If the optional
916 parameter live_port_needed is set to true, requires the remote end
917 to accept the connection. The timeout is specified in seconds and
918 defaults to 10 seconds. If the source optional argument is not
919 passed, the source address selection is left to the kernel,
920 otherwise we try to connect using the passed address (failures to
921 bind other than EADDRNOTAVAIL will be ignored).
924 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
928 if source is not None:
930 sock.bind((source, 0))
931 except socket.error, (errcode, errstring):
932 if errcode == errno.EADDRNOTAVAIL:
935 sock.settimeout(timeout)
938 sock.connect((target, port))
941 except socket.timeout:
943 except socket.error, (errcode, errstring):
944 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
949 def ListVisibleFiles(path):
950 """Returns a list of all visible files in a directory.
953 files = [i for i in os.listdir(path) if not i.startswith(".")]
958 def GetHomeDir(user, default=None):
959 """Try to get the homedir of the given user.
961 The user can be passed either as a string (denoting the name) or as
962 an integer (denoting the user id). If the user is not found, the
963 'default' argument is returned, which defaults to None.
967 if isinstance(user, basestring):
968 result = pwd.getpwnam(user)
969 elif isinstance(user, (int, long)):
970 result = pwd.getpwuid(user)
972 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
980 """Returns a random UUID.
983 f = open("/proc/sys/kernel/random/uuid", "r")
985 return f.read(128).rstrip("\n")
990 def WriteFile(file_name, fn=None, data=None,
991 mode=None, uid=-1, gid=-1,
992 atime=None, mtime=None):
993 """(Over)write a file atomically.
995 The file_name and either fn (a function taking one argument, the
996 file descriptor, and which should write the data to it) or data (the
997 contents of the file) must be passed. The other arguments are
998 optional and allow setting the file mode, owner and group, and the
999 mtime/atime of the file.
1001 If the function doesn't raise an exception, it has succeeded and the
1002 target file has the new contents. If the file has raised an
1003 exception, an existing target file should be unmodified and the
1004 temporary file should be removed.
1007 if not os.path.isabs(file_name):
1008 raise errors.ProgrammerError("Path passed to WriteFile is not"
1009 " absolute: '%s'" % file_name)
1011 if [fn, data].count(None) != 1:
1012 raise errors.ProgrammerError("fn or data required")
1014 if [atime, mtime].count(None) == 1:
1015 raise errors.ProgrammerError("Both atime and mtime must be either"
1019 dir_name, base_name = os.path.split(file_name)
1020 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1021 # here we need to make sure we remove the temp file, if any error
1022 # leaves it in place
1024 if uid != -1 or gid != -1:
1025 os.chown(new_name, uid, gid)
1027 os.chmod(new_name, mode)
1028 if data is not None:
1033 if atime is not None and mtime is not None:
1034 os.utime(new_name, (atime, mtime))
1035 os.rename(new_name, file_name)
1038 RemoveFile(new_name)
1041 def all(seq, pred=bool):
1042 "Returns True if pred(x) is True for every element in the iterable"
1043 for elem in itertools.ifilterfalse(pred, seq):
1048 def any(seq, pred=bool):
1049 "Returns True if pred(x) is True for at least one element in the iterable"
1050 for elem in itertools.ifilter(pred, seq):
1055 def UniqueSequence(seq):
1056 """Returns a list with unique elements.
1058 Element order is preserved.
1061 return [i for i in seq if i not in seen and not seen.add(i)]
1064 def IsValidMac(mac):
1065 """Predicate to check if a MAC address is valid.
1067 Checks wether the supplied MAC address is formally correct, only
1068 accepts colon separated format.
1070 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1071 return mac_check.match(mac) is not None
1074 def TestDelay(duration):
1075 """Sleep for a fixed amount of time.
1080 time.sleep(duration)