4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Ganeti small utilities
37 from ganeti import logger
38 from ganeti import errors
41 _re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
43 class RunResult(object):
44 """Simple class for holding the result of running external programs.
47 exit_code: the exit code of the program, or None (if the program
49 signal: numeric signal that caused the program to finish, or None
50 (if the program wasn't terminated by a signal)
51 stdout: the standard output of the program
52 stderr: the standard error of the program
53 failed: a Boolean value which is True in case the program was
54 terminated by a signal or exited with a non-zero exit code
55 fail_reason: a string detailing the termination reason
58 __slots__ = ["exit_code", "signal", "stdout", "stderr",
59 "failed", "fail_reason", "cmd"]
62 def __init__(self, exit_code, signal, stdout, stderr, cmd):
64 self.exit_code = exit_code
68 self.failed = (signal is not None or exit_code != 0)
70 if self.signal is not None:
71 self.fail_reason = "terminated by signal %s" % self.signal
72 elif self.exit_code is not None:
73 self.fail_reason = "exited with exit code %s" % self.exit_code
75 self.fail_reason = "unable to determine termination reason"
78 """Returns the combined stdout and stderr for easier usage.
81 return self.stdout + self.stderr
83 output = property(_GetOutput, None, None, "Return full output")
86 def _GetLockFile(subsystem):
87 """Compute the file name for a given lock name."""
88 return "/var/lock/ganeti_lock_%s" % subsystem
91 def Lock(name, max_retries=None, debug=False):
92 """Lock a given subsystem.
94 In case the lock is already held by an alive process, the function
95 will sleep indefintely and poll with a one second interval.
97 When the optional integer argument 'max_retries' is passed with a
98 non-zero value, the function will sleep only for this number of
99 times, and then it will will raise a LockError if the lock can't be
100 acquired. Passing in a negative number will cause only one try to
101 get the lock. Passing a positive number will make the function retry
102 for approximately that number of seconds.
105 lockfile = _GetLockFile(name)
107 if name in _locksheld:
108 raise errors.LockError('Lock "%s" already held!' % (name,))
115 fd = os.open(lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_SYNC)
117 except OSError, creat_err:
118 if creat_err.errno != errno.EEXIST:
119 raise errors.LockError("Can't create the lock file. Error '%s'." %
123 pf = open(lockfile, 'r')
124 except IOError, open_err:
127 raise errors.LockError("Lock file exists but cannot be opened."
128 " Error: '%s'." % str(open_err))
135 raise errors.LockError("Invalid pid string in %s" %
138 if not IsProcessAlive(pid):
139 raise errors.LockError("Stale lockfile %s for pid %d?" %
142 if max_retries and max_retries <= retries:
143 raise errors.LockError("Can't acquire lock during the specified"
145 if retries == 5 and (debug or sys.stdin.isatty()):
146 logger.ToStderr("Waiting for '%s' lock from pid %d..." % (name, pid))
152 os.write(fd, '%d\n' % (os.getpid(),))
155 _locksheld.append(name)
159 """Unlock a given subsystem.
162 lockfile = _GetLockFile(name)
165 fd = os.open(lockfile, os.O_RDONLY)
167 raise errors.LockError('Lock "%s" not held.' % (name,))
169 f = os.fdopen(fd, 'r')
175 raise errors.LockError('Unable to determine PID of locking process.')
177 if pid != os.getpid():
178 raise errors.LockError('Lock not held by me (%d != %d)' %
182 _locksheld.remove(name)
189 for lock in _locksheld:
194 """Execute a (shell) command.
196 The command should not read from its standard input, as it will be
200 cmd: command to run. (str)
202 Returns: `RunResult` instance
205 if isinstance(cmd, list):
206 cmd = [str(val) for val in cmd]
207 strcmd = " ".join(cmd)
212 env = os.environ.copy()
214 child = subprocess.Popen(cmd, shell=shell,
215 stderr=subprocess.PIPE,
216 stdout=subprocess.PIPE,
217 stdin=subprocess.PIPE,
218 close_fds=True, env=env)
221 out = child.stdout.read()
222 err = child.stderr.read()
224 status = child.wait()
232 return RunResult(exitcode, signal, out, err, strcmd)
235 def RunCmdUnlocked(cmd):
236 """Execute a shell command without the 'cmd' lock.
238 This variant of `RunCmd()` drops the 'cmd' lock before running the
239 command and re-aquires it afterwards, thus it can be used to call
240 other ganeti commands.
242 The argument and return values are the same as for the `RunCmd()`
246 cmd - command to run. (str)
259 def RemoveFile(filename):
260 """Remove a file ignoring some errors.
262 Remove a file, ignoring non-existing ones or directories. Other
269 if err.errno not in (errno.ENOENT, errno.EISDIR):
273 def _FingerprintFile(filename):
274 """Compute the fingerprint of a file.
276 If the file does not exist, a None will be returned
280 filename - Filename (str)
283 if not (os.path.exists(filename) and os.path.isfile(filename)):
296 return fp.hexdigest()
299 def FingerprintFiles(files):
300 """Compute fingerprints for a list of files.
303 files - array of filenames. ( [str, ...] )
306 dictionary of filename: fingerprint for the files that exist
311 for filename in files:
312 cksum = _FingerprintFile(filename)
314 ret[filename] = cksum
319 def CheckDict(target, template, logname=None):
320 """Ensure a dictionary has a required set of keys.
322 For the given dictionaries `target` and `template`, ensure target
323 has all the keys from template. Missing keys are added with values
327 target - the dictionary to check
328 template - template dictionary
329 logname - a caller-chosen string to identify the debug log
330 entry; if None, no logging will be done
340 target[k] = template[k]
342 if missing and logname:
343 logger.Debug('%s missing keys %s' %
344 (logname, ', '.join(missing)))
347 def IsProcessAlive(pid):
348 """Check if a given pid exists on the system.
350 Returns: true or false, depending on if the pid exists or not
352 Remarks: zombie processes treated as not alive
356 f = open("/proc/%d/status" % pid)
358 if err.errno in (errno.ENOENT, errno.ENOTDIR):
365 state = data[1].split()
366 if len(state) > 1 and state[1] == "Z":
374 def MatchNameComponent(key, name_list):
375 """Try to match a name against a list.
377 This function will try to match a name like test1 against a list
378 like ['test1.example.com', 'test2.example.com', ...]. Against this
379 list, 'test1' as well as 'test1.example' will match, but not
380 'test1.ex'. A multiple match will be considered as no match at all
381 (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).
384 key: the name to be searched
385 name_list: the list of strings against which to search the key
388 None if there is no match *or* if there are multiple matches
389 otherwise the element from the list which matches
392 mo = re.compile("^%s(\..*)?$" % re.escape(key))
393 names_filtered = [name for name in name_list if mo.match(name) is not None]
394 if len(names_filtered) != 1:
396 return names_filtered[0]
400 """Class implementing resolver and hostname functionality
403 def __init__(self, name=None):
404 """Initialize the host name object.
406 If the name argument is not passed, it will use this system's
411 name = self.SysName()
414 self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
415 self.ip = self.ipaddrs[0]
419 """Return the current system's name.
421 This is simply a wrapper over socket.gethostname()
424 return socket.gethostname()
427 def LookupHostname(hostname):
431 hostname: hostname to look up
434 a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
435 in case of errors in resolving, we raise a ResolverError
439 result = socket.gethostbyname_ex(hostname)
440 except socket.gaierror, err:
441 # hostname not found in DNS
442 raise errors.ResolverError(hostname, err.args[0], err.args[1])
447 def ListVolumeGroups():
448 """List volume groups and their size
451 Dictionary with keys volume name and values the size of the volume
454 command = "vgs --noheadings --units m --nosuffix -o name,size"
455 result = RunCmd(command)
460 for line in result.stdout.splitlines():
462 name, size = line.split()
463 size = int(float(size))
464 except (IndexError, ValueError), err:
465 logger.Error("Invalid output from vgs (%s): %s" % (err, line))
473 def BridgeExists(bridge):
474 """Check whether the given bridge exists in the system
477 True if it does, false otherwise.
480 return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
483 def NiceSort(name_list):
484 """Sort a list of strings based on digit and non-digit groupings.
486 Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
487 sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].
489 The sort algorithm breaks each name in groups of either only-digits
490 or no-digits. Only the first eight such groups are considered, and
491 after that we just use what's left of the string.
494 - a copy of the list sorted according to our algorithm
497 _SORTER_BASE = "(\D+|\d+)"
498 _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
499 _SORTER_BASE, _SORTER_BASE,
500 _SORTER_BASE, _SORTER_BASE,
501 _SORTER_BASE, _SORTER_BASE)
502 _SORTER_RE = re.compile(_SORTER_FULL)
503 _SORTER_NODIGIT = re.compile("^\D*$")
505 """Attempts to convert a variable to integer."""
506 if val is None or _SORTER_NODIGIT.match(val):
511 to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
512 for name in name_list]
514 return [tup[1] for tup in to_sort]
517 def CheckDaemonAlive(pid_file, process_string):
518 """Check wether the specified daemon is alive.
521 - pid_file: file to read the daemon pid from, the file is
522 expected to contain only a single line containing
524 - process_string: a substring that we expect to find in
525 the command line of the daemon process
528 - True if the daemon is judged to be alive (that is:
529 - the PID file exists, is readable and contains a number
530 - a process of the specified PID is running
531 - that process contains the specified string in its
533 - the process is not in state Z (zombie))
538 pid_file = file(pid_file, 'r')
540 pid = int(pid_file.readline())
544 cmdline_file_path = "/proc/%s/cmdline" % (pid)
545 cmdline_file = open(cmdline_file_path, 'r')
547 cmdline = cmdline_file.readline()
551 if not process_string in cmdline:
554 stat_file_path = "/proc/%s/stat" % (pid)
555 stat_file = open(stat_file_path, 'r')
557 process_state = stat_file.readline().split()[2]
561 if process_state == 'Z':
564 except (IndexError, IOError, ValueError):
570 def TryConvert(fn, val):
571 """Try to convert a value ignoring errors.
573 This function tries to apply function `fn` to `val`. If no
574 ValueError or TypeError exceptions are raised, it will return the
575 result, else it will return the original value. Any other exceptions
576 are propagated to the caller.
581 except (ValueError, TypeError), err:
587 """Verifies the syntax of an IP address.
589 This function checks if the ip address passes is valid or not based
590 on syntax (not ip range, class calculations or anything).
593 unit = "(0|[1-9]\d{0,2})"
594 return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
597 def IsValidShellParam(word):
598 """Verifies is the given word is safe from the shell's p.o.v.
600 This means that we can pass this to a command via the shell and be
601 sure that it doesn't alter the command line and is passed as such to
604 Note that we are overly restrictive here, in order to be on the safe
608 return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
611 def BuildShellCmd(template, *args):
612 """Build a safe shell command line from the given arguments.
614 This function will check all arguments in the args list so that they
615 are valid shell parameters (i.e. they don't contain shell
616 metacharaters). If everything is ok, it will return the result of
621 if not IsValidShellParam(word):
622 raise errors.ProgrammerError("Shell argument '%s' contains"
623 " invalid characters" % word)
624 return template % args
627 def FormatUnit(value):
628 """Formats an incoming number of MiB with the appropriate unit.
630 Value needs to be passed as a numeric type. Return value is always a string.
634 return "%dM" % round(value, 0)
636 elif value < (1024 * 1024):
637 return "%0.1fG" % round(float(value) / 1024, 1)
640 return "%0.1fT" % round(float(value) / 1024 / 1024, 1)
643 def ParseUnit(input_string):
644 """Tries to extract number and scale from the given string.
646 Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
647 is specified, it defaults to MiB. Return value is always an int in MiB.
650 m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
652 raise errors.UnitParseError("Invalid format")
654 value = float(m.groups()[0])
658 lcunit = unit.lower()
662 if lcunit in ('m', 'mb', 'mib'):
663 # Value already in MiB
666 elif lcunit in ('g', 'gb', 'gib'):
669 elif lcunit in ('t', 'tb', 'tib'):
673 raise errors.UnitParseError("Unknown unit: %s" % unit)
675 # Make sure we round up
676 if int(value) < value:
679 # Round up to the next multiple of 4
682 value += 4 - value % 4
687 def AddAuthorizedKey(file_name, key):
688 """Adds an SSH public key to an authorized_keys file.
691 file_name: Path to authorized_keys file
692 key: String containing key
694 key_fields = key.split()
696 f = open(file_name, 'a+')
700 # Ignore whitespace changes
701 if line.split() == key_fields:
703 nl = line.endswith('\n')
707 f.write(key.rstrip('\r\n'))
714 def RemoveAuthorizedKey(file_name, key):
715 """Removes an SSH public key from an authorized_keys file.
718 file_name: Path to authorized_keys file
719 key: String containing key
721 key_fields = key.split()
723 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
724 out = os.fdopen(fd, 'w')
726 f = open(file_name, 'r')
729 # Ignore whitespace changes while comparing lines
730 if line.split() != key_fields:
734 os.rename(tmpname, file_name)
741 def CreateBackup(file_name):
742 """Creates a backup of a file.
744 Returns: the path to the newly created backup file.
747 if not os.path.isfile(file_name):
748 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
751 prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
752 dir = os.path.dirname(file_name)
754 fsrc = open(file_name, 'rb')
756 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir)
757 fdst = os.fdopen(fd, 'wb')
759 shutil.copyfileobj(fsrc, fdst)
768 def ShellQuote(value):
769 """Quotes shell argument according to POSIX.
772 if _re_shell_unquoted.match(value):
775 return "'%s'" % value.replace("'", "'\\''")
778 def ShellQuoteArgs(args):
779 """Quotes all given shell arguments and concatenates using spaces.
782 return ' '.join([ShellQuote(i) for i in args])
785 def _ParseIpOutput(output):
786 """Parsing code for GetLocalIPAddresses().
788 This function is split out, so we can unit test it.
791 re_ip = re.compile('^(\d+\.\d+\.\d+\.\d+)(?:/\d+)$')
794 for line in output.splitlines(False):
795 fields = line.split()
798 m = re_ip.match(fields[3])
800 ips.append(m.group(1))
805 def GetLocalIPAddresses():
806 """Gets a list of all local IP addresses.
808 Should this break one day, a small Python module written in C could
809 use the API call getifaddrs().
812 result = RunCmd(["ip", "-family", "inet", "-oneline", "addr", "show"])
814 raise errors.OpExecError("Command '%s' failed, error: %s,"
815 " output: %s" % (result.cmd, result.fail_reason, result.output))
817 return _ParseIpOutput(result.output)
820 def TcpPing(source, target, port, timeout=10, live_port_needed=True):
821 """Simple ping implementation using TCP connect(2).
823 Try to do a TCP connect(2) from the specified source IP to the specified
824 target IP and the specified target port. If live_port_needed is set to true,
825 requires the remote end to accept the connection. The timeout is specified
826 in seconds and defaults to 10 seconds
829 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
834 sock.bind((source, 0))
835 except socket.error, (errcode, errstring):
836 if errcode == errno.EADDRNOTAVAIL:
839 sock.settimeout(timeout)
842 sock.connect((target, port))
845 except socket.timeout:
847 except socket.error, (errcode, errstring):
848 success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
853 def ListVisibleFiles(path):
854 """Returns a list of all visible files in a directory.
857 return [i for i in os.listdir(path) if not i.startswith(".")]