4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti small utilities
43 from cStringIO import StringIO
45 from ganeti import logger
46 from ganeti import errors
47 from ganeti import constants
51 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
56 class RunResult(object):
57 """Simple class for holding the result of running external programs.
60 exit_code: the exit code of the program, or None (if the program
62 signal: numeric signal that caused the program to finish, or None
63 (if the program wasn't terminated by a signal)
64 stdout: the standard output of the program
65 stderr: the standard error of the program
66 failed: a Boolean value which is True in case the program was
67 terminated by a signal or exited with a non-zero exit code
68 fail_reason: a string detailing the termination reason
71 __slots__ = ["exit_code", "signal", "stdout", "stderr",
72 "failed", "fail_reason", "cmd"]
75 def __init__(self, exit_code, signal, stdout, stderr, cmd):
77 self.exit_code = exit_code
81 self.failed = (signal is not None or exit_code != 0)
83 if self.signal is not None:
84 self.fail_reason = "terminated by signal %s" % self.signal
85 elif self.exit_code is not None:
86 self.fail_reason = "exited with exit code %s" % self.exit_code
88 self.fail_reason = "unable to determine termination reason"
90 if debug and self.failed:
91 logger.Debug("Command '%s' failed (%s); output: %s" %
92 (self.cmd, self.fail_reason, self.output))
95 """Returns the combined stdout and stderr for easier usage.
98 return self.stdout + self.stderr
100 output = property(_GetOutput, None, None, "Return full output")
103 def _GetLockFile(subsystem):
104 """Compute the file name for a given lock name."""
105 return "%s/ganeti_lock_%s" % (constants.LOCK_DIR, subsystem)
108 def Lock(name, max_retries=None, debug=False):
109 """Lock a given subsystem.
111 In case the lock is already held by an alive process, the function
112 will sleep indefintely and poll with a one second interval.
114 When the optional integer argument 'max_retries' is passed with a
115 non-zero value, the function will sleep only for this number of
116 times, and then it will will raise a LockError if the lock can't be
117 acquired. Passing in a negative number will cause only one try to
118 get the lock. Passing a positive number will make the function retry
119 for approximately that number of seconds.
122 lockfile = _GetLockFile(name)
124 if name in _locksheld:
125 raise errors.LockError('Lock "%s" already held!' % (name,))
132 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_SYNC)
134 except OSError, creat_err:
135 if creat_err.errno != errno.EEXIST:
136 raise errors.LockError("Can't create the lock file. Error '%s'." %
140 pf = open(lockfile, 'r')
141 except IOError, open_err:
144 raise errors.LockError("Lock file exists but cannot be opened."
145 " Error: '%s'." % str(open_err))
152 raise errors.LockError("Invalid pid string in %s" %
155 if not IsProcessAlive(pid):
156 raise errors.LockError("Stale lockfile %s for pid %d?" %
159 if max_retries and max_retries <= retries:
160 raise errors.LockError("Can't acquire lock during the specified"
162 if retries == 5 and (debug or sys.stdin.isatty()):
163 logger.ToStderr("Waiting for '%s' lock from pid %d..." % (name, pid))
169 os.write(fd, '%d\n' % (os.getpid(),))
172 _locksheld.append(name)
176 """Unlock a given subsystem.
179 lockfile = _GetLockFile(name)
182 fd = os.open(lockfile, os.O_RDONLY)
184 raise errors.LockError('Lock "%s" not held.' % (name,))
186 f = os.fdopen(fd, 'r')
192 raise errors.LockError('Unable to determine PID of locking process.')
194 if pid != os.getpid():
195 raise errors.LockError('Lock not held by me (%d != %d)' %
199 _locksheld.remove(name)
206 for lock in _locksheld:
211 """Execute a (shell) command.
213 The command should not read from its standard input, as it will be
217 cmd: command to run. (str)
219 Returns: `RunResult` instance
222 if isinstance(cmd, list):
223 cmd = [str(val) for val in cmd]
224 strcmd = " ".join(cmd)
229 env = os.environ.copy()
231 poller = select.poll()
232 child = subprocess.Popen(cmd, shell=shell,
233 stderr=subprocess.PIPE,
234 stdout=subprocess.PIPE,
235 stdin=subprocess.PIPE,
236 close_fds=True, env=env)
239 poller.register(child.stdout, select.POLLIN)
240 poller.register(child.stderr, select.POLLIN)
244 child.stdout.fileno(): (out, child.stdout),
245 child.stderr.fileno(): (err, child.stderr),
248 status = fcntl.fcntl(fd, fcntl.F_GETFL)
249 fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
252 for fd, event in poller.poll():
253 if event & select.POLLIN or event & select.POLLPRI:
254 data = fdmap[fd][1].read()
255 # no data from read signifies EOF (the same as POLLHUP)
257 poller.unregister(fd)
260 fdmap[fd][0].write(data)
261 if (event & select.POLLNVAL or event & select.POLLHUP or
262 event & select.POLLERR):
263 poller.unregister(fd)
269 status = child.wait()
277 return RunResult(exitcode, signal, out, err, strcmd)
280 def RunCmdUnlocked(cmd):
281 """Execute a shell command without the 'cmd' lock.
283 This variant of `RunCmd()` drops the 'cmd' lock before running the
284 command and re-aquires it afterwards, thus it can be used to call
285 other ganeti commands.
287 The argument and return values are the same as for the `RunCmd()`
291 cmd - command to run. (str)
304 def RemoveFile(filename):
305 """Remove a file ignoring some errors.
307 Remove a file, ignoring non-existing ones or directories. Other
314 if err.errno not in (errno.ENOENT, errno.EISDIR):
318 def _FingerprintFile(filename):
319 """Compute the fingerprint of a file.
321 If the file does not exist, a None will be returned
325 filename - Filename (str)
328 if not (os.path.exists(filename) and os.path.isfile(filename)):
341 return fp.hexdigest()
344 def FingerprintFiles(files):
345 """Compute fingerprints for a list of files.
348 files - array of filenames. ( [str, ...] )
351 dictionary of filename: fingerprint for the files that exist
356 for filename in files:
357 cksum = _FingerprintFile(filename)
359 ret[filename] = cksum
364 def CheckDict(target, template, logname=None):
365 """Ensure a dictionary has a required set of keys.
367 For the given dictionaries `target` and `template`, ensure target
368 has all the keys from template. Missing keys are added with values
372 target - the dictionary to check
373 template - template dictionary
374 logname - a caller-chosen string to identify the debug log
375 entry; if None, no logging will be done
385 target[k] = template[k]
387 if missing and logname:
388 logger.Debug('%s missing keys %s' %
389 (logname, ', '.join(missing)))
392 def IsProcessAlive(pid):
393 """Check if a given pid exists on the system.
395 Returns: true or false, depending on if the pid exists or not
397 Remarks: zombie processes treated as not alive
401 f = open("/proc/%d/status" % pid)
403 if err.errno in (errno.ENOENT, errno.ENOTDIR):
410 state = data[1].split()
411 if len(state) > 1 and state[1] == "Z":
419 def MatchNameComponent(key, name_list):
420 """Try to match a name against a list.
422 This function will try to match a name like test1 against a list
423 like ['test1.example.com', 'test2.example.com', ...]. Against this
424 list, 'test1' as well as 'test1.example' will match, but not
425 'test1.ex'. A multiple match will be considered as no match at all
426 (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).
429 key: the name to be searched
430 name_list: the list of strings against which to search the key
433 None if there is no match *or* if there are multiple matches
434 otherwise the element from the list which matches
437 mo = re.compile("^%s(\..*)?$" % re.escape(key))
438 names_filtered = [name for name in name_list if mo.match(name) is not None]
439 if len(names_filtered) != 1:
441 return names_filtered[0]
445 """Class implementing resolver and hostname functionality
448 def __init__(self, name=None):
449 """Initialize the host name object.
451 If the name argument is not passed, it will use this system's
456 name = self.SysName()
459 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
460 self.ip = self.ipaddrs[0]
463 """Returns the hostname without domain.
466 return self.name.split('.')[0]
470 """Return the current system's name.
472 This is simply a wrapper over socket.gethostname()
475 return socket.gethostname()
478 def LookupHostname(hostname):
482 hostname: hostname to look up
485 a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
486 in case of errors in resolving, we raise a ResolverError
490 result = socket.gethostbyname_ex(hostname)
491 except socket.gaierror, err:
492 # hostname not found in DNS
493 raise errors.ResolverError(hostname, err.args[0], err.args[1])
498 def ListVolumeGroups():
499 """List volume groups and their size
502 Dictionary with keys volume name and values the size of the volume
505 command = "vgs --noheadings --units m --nosuffix -o name,size"
506 result = RunCmd(command)
511 for line in result.stdout.splitlines():
513 name, size = line.split()
514 size = int(float(size))
515 except (IndexError, ValueError), err:
516 logger.Error("Invalid output from vgs (%s): %s" % (err, line))
524 def BridgeExists(bridge):
525 """Check whether the given bridge exists in the system
528 True if it does, false otherwise.
531 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
534 def NiceSort(name_list):
535 """Sort a list of strings based on digit and non-digit groupings.
537 Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
538 sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].
540 The sort algorithm breaks each name in groups of either only-digits
541 or no-digits. Only the first eight such groups are considered, and
542 after that we just use what's left of the string.
545 - a copy of the list sorted according to our algorithm
548 _SORTER_BASE = "(\D+|\d+)"
549 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
550 _SORTER_BASE, _SORTER_BASE,
551 _SORTER_BASE, _SORTER_BASE,
552 _SORTER_BASE, _SORTER_BASE)
553 _SORTER_RE = re.compile(_SORTER_FULL)
554 _SORTER_NODIGIT = re.compile("^\D*$")
556 """Attempts to convert a variable to integer."""
557 if val is None or _SORTER_NODIGIT.match(val):
562 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
563 for name in name_list]
565 return [tup[1] for tup in to_sort]
568 def CheckDaemonAlive(pid_file, process_string):
569 """Check wether the specified daemon is alive.
572 - pid_file: file to read the daemon pid from, the file is
573 expected to contain only a single line containing
575 - process_string: a substring that we expect to find in
576 the command line of the daemon process
579 - True if the daemon is judged to be alive (that is:
580 - the PID file exists, is readable and contains a number
581 - a process of the specified PID is running
582 - that process contains the specified string in its
584 - the process is not in state Z (zombie))
589 pid_file = file(pid_file, 'r')
591 pid = int(pid_file.readline())
595 cmdline_file_path = "/proc/%s/cmdline" % (pid)
596 cmdline_file = open(cmdline_file_path, 'r')
598 cmdline = cmdline_file.readline()
602 if not process_string in cmdline:
605 stat_file_path = "/proc/%s/stat" % (pid)
606 stat_file = open(stat_file_path, 'r')
608 process_state = stat_file.readline().split()[2]
612 if process_state == 'Z':
615 except (IndexError, IOError, ValueError):
621 def TryConvert(fn, val):
622 """Try to convert a value ignoring errors.
624 This function tries to apply function `fn` to `val`. If no
625 ValueError or TypeError exceptions are raised, it will return the
626 result, else it will return the original value. Any other exceptions
627 are propagated to the caller.
632 except (ValueError, TypeError), err:
638 """Verifies the syntax of an IP address.
640 This function checks if the ip address passes is valid or not based
641 on syntax (not ip range, class calculations or anything).
644 unit = "(0|[1-9]\d{0,2})"
645 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
648 def IsValidShellParam(word):
649 """Verifies is the given word is safe from the shell's p.o.v.
651 This means that we can pass this to a command via the shell and be
652 sure that it doesn't alter the command line and is passed as such to
655 Note that we are overly restrictive here, in order to be on the safe
659 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
662 def BuildShellCmd(template, *args):
663 """Build a safe shell command line from the given arguments.
665 This function will check all arguments in the args list so that they
666 are valid shell parameters (i.e. they don't contain shell
667 metacharaters). If everything is ok, it will return the result of
672 if not IsValidShellParam(word):
673 raise errors.ProgrammerError("Shell argument '%s' contains"
674 " invalid characters" % word)
675 return template % args
678 def FormatUnit(value):
679 """Formats an incoming number of MiB with the appropriate unit.
681 Value needs to be passed as a numeric type. Return value is always a string.
685 return "%dM" % round(value, 0)
687 elif value < (1024 * 1024):
688 return "%0.1fG" % round(float(value) / 1024, 1)
691 return "%0.1fT" % round(float(value) / 1024 / 1024, 1)
694 def ParseUnit(input_string):
695 """Tries to extract number and scale from the given string.
697 Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
698 is specified, it defaults to MiB. Return value is always an int in MiB.
701 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
703 raise errors.UnitParseError("Invalid format")
705 value = float(m.groups()[0])
709 lcunit = unit.lower()
713 if lcunit in ('m', 'mb', 'mib'):
714 # Value already in MiB
717 elif lcunit in ('g', 'gb', 'gib'):
720 elif lcunit in ('t', 'tb', 'tib'):
724 raise errors.UnitParseError("Unknown unit: %s" % unit)
726 # Make sure we round up
727 if int(value) < value:
730 # Round up to the next multiple of 4
733 value += 4 - value % 4
738 def AddAuthorizedKey(file_name, key):
739 """Adds an SSH public key to an authorized_keys file.
742 file_name: Path to authorized_keys file
743 key: String containing key
745 key_fields = key.split()
747 f = open(file_name, 'a+')
751 # Ignore whitespace changes
752 if line.split() == key_fields:
754 nl = line.endswith('\n')
758 f.write(key.rstrip('\r\n'))
765 def RemoveAuthorizedKey(file_name, key):
766 """Removes an SSH public key from an authorized_keys file.
769 file_name: Path to authorized_keys file
770 key: String containing key
772 key_fields = key.split()
774 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
776 out = os.fdopen(fd, 'w')
778 f = open(file_name, 'r')
781 # Ignore whitespace changes while comparing lines
782 if line.split() != key_fields:
786 os.rename(tmpname, file_name)
796 def SetEtcHostsEntry(file_name, ip, hostname, aliases):
797 """Sets the name of an IP address and hostname in /etc/hosts.
800 # Ensure aliases are unique
801 aliases = UniqueSequence([hostname] + aliases)[1:]
803 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
805 out = os.fdopen(fd, 'w')
807 f = open(file_name, 'r')
811 fields = line.split()
812 if fields and not fields[0].startswith('#') and ip == fields[0]:
816 out.write("%s\t%s" % (ip, hostname))
818 out.write(" %s" % ' '.join(aliases))
823 os.rename(tmpname, file_name)
833 def RemoveEtcHostsEntry(file_name, hostname):
834 """Removes a hostname from /etc/hosts.
836 IP addresses without names are removed from the file.
838 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
840 out = os.fdopen(fd, 'w')
842 f = open(file_name, 'r')
845 fields = line.split()
846 if len(fields) > 1 and not fields[0].startswith('#'):
848 if hostname in names:
849 while hostname in names:
850 names.remove(hostname)
852 out.write("%s %s\n" % (fields[0], ' '.join(names)))
859 os.rename(tmpname, file_name)
869 def CreateBackup(file_name):
870 """Creates a backup of a file.
872 Returns: the path to the newly created backup file.
875 if not os.path.isfile(file_name):
876 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
879 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
880 dir_name = os.path.dirname(file_name)
882 fsrc = open(file_name, 'rb')
884 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
885 fdst = os.fdopen(fd, 'wb')
887 shutil.copyfileobj(fsrc, fdst)
896 def ShellQuote(value):
897 """Quotes shell argument according to POSIX.
900 if _re_shell_unquoted.match(value):
903 return "'%s'" % value.replace("'", "'\\''")
906 def ShellQuoteArgs(args):
907 """Quotes all given shell arguments and concatenates using spaces.
910 return ' '.join([ShellQuote(i) for i in args])
913 def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
914 """Simple ping implementation using TCP connect(2).
916 Try to do a TCP connect(2) from an optional source IP to the
917 specified target IP and the specified target port. If the optional
918 parameter live_port_needed is set to true, requires the remote end
919 to accept the connection. The timeout is specified in seconds and
920 defaults to 10 seconds. If the source optional argument is not
921 passed, the source address selection is left to the kernel,
922 otherwise we try to connect using the passed address (failures to
923 bind other than EADDRNOTAVAIL will be ignored).
926 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
930 if source is not None:
932 sock.bind((source, 0))
933 except socket.error, (errcode, errstring):
934 if errcode == errno.EADDRNOTAVAIL:
937 sock.settimeout(timeout)
940 sock.connect((target, port))
943 except socket.timeout:
945 except socket.error, (errcode, errstring):
946 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
951 def ListVisibleFiles(path):
952 """Returns a list of all visible files in a directory.
955 files = [i for i in os.listdir(path) if not i.startswith(".")]
960 def GetHomeDir(user, default=None):
961 """Try to get the homedir of the given user.
963 The user can be passed either as a string (denoting the name) or as
964 an integer (denoting the user id). If the user is not found, the
965 'default' argument is returned, which defaults to None.
969 if isinstance(user, basestring):
970 result = pwd.getpwnam(user)
971 elif isinstance(user, (int, long)):
972 result = pwd.getpwuid(user)
974 raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
982 """Returns a random UUID.
985 f = open("/proc/sys/kernel/random/uuid", "r")
987 return f.read(128).rstrip("\n")
992 def WriteFile(file_name, fn=None, data=None,
993 mode=None, uid=-1, gid=-1,
994 atime=None, mtime=None):
995 """(Over)write a file atomically.
997 The file_name and either fn (a function taking one argument, the
998 file descriptor, and which should write the data to it) or data (the
999 contents of the file) must be passed. The other arguments are
1000 optional and allow setting the file mode, owner and group, and the
1001 mtime/atime of the file.
1003 If the function doesn't raise an exception, it has succeeded and the
1004 target file has the new contents. If the file has raised an
1005 exception, an existing target file should be unmodified and the
1006 temporary file should be removed.
1009 if not os.path.isabs(file_name):
1010 raise errors.ProgrammerError("Path passed to WriteFile is not"
1011 " absolute: '%s'" % file_name)
1013 if [fn, data].count(None) != 1:
1014 raise errors.ProgrammerError("fn or data required")
1016 if [atime, mtime].count(None) == 1:
1017 raise errors.ProgrammerError("Both atime and mtime must be either"
1021 dir_name, base_name = os.path.split(file_name)
1022 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1023 # here we need to make sure we remove the temp file, if any error
1024 # leaves it in place
1026 if uid != -1 or gid != -1:
1027 os.chown(new_name, uid, gid)
1029 os.chmod(new_name, mode)
1030 if data is not None:
1035 if atime is not None and mtime is not None:
1036 os.utime(new_name, (atime, mtime))
1037 os.rename(new_name, file_name)
1040 RemoveFile(new_name)
1043 def all(seq, pred=bool):
1044 "Returns True if pred(x) is True for every element in the iterable"
1045 for elem in itertools.ifilterfalse(pred, seq):
1050 def any(seq, pred=bool):
1051 "Returns True if pred(x) is True for at least one element in the iterable"
1052 for elem in itertools.ifilter(pred, seq):
1057 def UniqueSequence(seq):
1058 """Returns a list with unique elements.
1060 Element order is preserved.
1063 return [i for i in seq if i not in seen and not seen.add(i)]
1066 def IsValidMac(mac):
1067 """Predicate to check if a MAC address is valid.
1069 Checks wether the supplied MAC address is formally correct, only
1070 accepts colon separated format.
1072 mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1073 return mac_check.match(mac) is not None
1076 def TestDelay(duration):
1077 """Sleep for a fixed amount of time.
1082 time.sleep(duration)
1086 def Daemonize(logfile, noclose_fds=None):
1087 """Daemonize the current process.
1089 This detaches the current process from the controlling terminal and
1090 runs it in the background as a daemon.
1095 # Default maximum for the number of available file descriptors.
1096 if 'SC_OPEN_MAX' in os.sysconf_names:
1098 MAXFD = os.sysconf('SC_OPEN_MAX')
1108 if (pid == 0): # The first child.
1111 pid = os.fork() # Fork a second child.
1112 if (pid == 0): # The second child.
1116 # exit() or _exit()? See below.
1117 os._exit(0) # Exit parent (the first child) of the second child.
1119 os._exit(0) # Exit parent of the first child.
1120 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1121 if (maxfd == resource.RLIM_INFINITY):
1124 # Iterate through and close all file descriptors.
1125 for fd in range(0, maxfd):
1126 if noclose_fds and fd in noclose_fds:
1130 except OSError: # ERROR, fd wasn't open to begin with (ignored)
1132 os.open(logfile, os.O_RDWR|os.O_CREAT|os.O_APPEND, 0600)
1133 # Duplicate standard input to standard output and standard error.
1134 os.dup2(0, 1) # standard output (1)
1135 os.dup2(0, 2) # standard error (2)