4 # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for I/O.
33 from ganeti import errors
34 from ganeti import constants
35 from ganeti import pathutils
36 from ganeti.utils import filelock
38 #: Directory used by fsck(8) to store recovered data, usually at a file
39 #: system's root directory
40 _LOST_AND_FOUND = "lost+found"
42 # Possible values for keep_perms in WriteFile()
55 """Format an EnvironmentError exception.
57 If the L{err} argument has an errno attribute, it will be looked up
58 and converted into a textual C{E...} description. Otherwise the
59 string representation of the error will be returned.
61 @type err: L{EnvironmentError}
62 @param err: the exception to format
65 if hasattr(err, "errno"):
66 detail = errno.errorcode[err.errno]
73 """Helper to store file handle's C{fstat}.
75 Useful in combination with L{ReadFile}'s C{preread} parameter.
79 """Initializes this class.
84 def __call__(self, fh):
85 """Calls C{fstat} on file handle.
88 self.st = os.fstat(fh.fileno())
91 def ReadFile(file_name, size=-1, preread=None):
95 @param size: Read at most size bytes (if negative, entire file)
96 @type preread: callable receiving file handle as single parameter
97 @param preread: Function called before file is read
99 @return: the (possibly partial) content of the file
102 f = open(file_name, "r")
112 def WriteFile(file_name, fn=None, data=None,
113 mode=None, uid=-1, gid=-1,
114 atime=None, mtime=None, close=True,
115 dry_run=False, backup=False,
116 prewrite=None, postwrite=None, keep_perms=KP_NEVER):
117 """(Over)write a file atomically.
119 The file_name and either fn (a function taking one argument, the
120 file descriptor, and which should write the data to it) or data (the
121 contents of the file) must be passed. The other arguments are
122 optional and allow setting the file mode, owner and group, and the
123 mtime/atime of the file.
125 If the function doesn't raise an exception, it has succeeded and the
126 target file has the new contents. If the function has raised an
127 exception, an existing target file should be unmodified and the
128 temporary file should be removed.
131 @param file_name: the target filename
133 @param fn: content writing function, called with
134 file descriptor as parameter
136 @param data: contents of the file
138 @param mode: file mode
140 @param uid: the owner of the file
142 @param gid: the group of the file
144 @param atime: a custom access time to be set on the file
146 @param mtime: a custom modification time to be set on the file
148 @param close: whether to close file after writing it
149 @type prewrite: callable
150 @param prewrite: function to be called before writing content
151 @type postwrite: callable
152 @param postwrite: function to be called after writing content
153 @type keep_perms: members of L{KEEP_PERMS_VALUES}
154 @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are
155 taken from the other parameters; if L{KP_ALWAYS}, owner, group, and
156 mode are copied from the existing file; if L{KP_IF_EXISTS}, owner,
157 group, and mode are taken from the file, and if the file doesn't
158 exist, they are taken from the other parameters. It is an error to
159 pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid},
160 or C{mode} are set to non-default values.
163 @return: None if the 'close' parameter evaluates to True,
164 otherwise the file descriptor
166 @raise errors.ProgrammerError: if any of the arguments are not valid
169 if not os.path.isabs(file_name):
170 raise errors.ProgrammerError("Path passed to WriteFile is not"
171 " absolute: '%s'" % file_name)
173 if [fn, data].count(None) != 1:
174 raise errors.ProgrammerError("fn or data required")
176 if [atime, mtime].count(None) == 1:
177 raise errors.ProgrammerError("Both atime and mtime must be either"
180 if not keep_perms in KEEP_PERMS_VALUES:
181 raise errors.ProgrammerError("Invalid value for keep_perms: %s" %
183 if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None):
184 raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid',"
185 " and 'mode' cannot be set")
187 if backup and not dry_run and os.path.isfile(file_name):
188 CreateBackup(file_name)
190 if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS:
191 # os.stat() raises an exception if the file doesn't exist
193 file_stat = os.stat(file_name)
194 mode = stat.S_IMODE(file_stat.st_mode)
195 uid = file_stat.st_uid
196 gid = file_stat.st_gid
198 if keep_perms == KP_ALWAYS:
200 # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist
202 # Whether temporary file needs to be removed (e.g. if any error occurs)
208 (dir_name, base_name) = os.path.split(file_name)
209 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
213 if uid != -1 or gid != -1:
214 os.chown(new_name, uid, gid)
216 os.chmod(new_name, mode)
217 if callable(prewrite):
220 if isinstance(data, unicode):
222 assert isinstance(data, str)
225 while offset < to_write:
226 written = os.write(fd, buffer(data, offset))
228 assert written <= to_write - offset
230 assert offset == to_write
233 if callable(postwrite):
236 if atime is not None and mtime is not None:
237 os.utime(new_name, (atime, mtime))
239 # Close file unless the file descriptor should be returned
245 # Rename file to destination name
247 os.rename(new_name, file_name)
248 # Successful, no need to remove anymore
257 def GetFileID(path=None, fd=None):
258 """Returns the file 'id', i.e. the dev/inode and mtime information.
260 Either the path to the file or the fd must be given.
262 @param path: the file path
263 @param fd: a file descriptor
264 @return: a tuple of (device number, inode number, mtime)
267 if [path, fd].count(None) != 1:
268 raise errors.ProgrammerError("One and only one of fd/path must be given")
275 return (st.st_dev, st.st_ino, st.st_mtime)
278 def VerifyFileID(fi_disk, fi_ours):
279 """Verifies that two file IDs are matching.
281 Differences in the inode/device are not accepted, but and older
282 timestamp for fi_disk is accepted.
284 @param fi_disk: tuple (dev, inode, mtime) representing the actual
286 @param fi_ours: tuple (dev, inode, mtime) representing the last
291 (d1, i1, m1) = fi_disk
292 (d2, i2, m2) = fi_ours
294 return (d1, i1) == (d2, i2) and m1 <= m2
297 def SafeWriteFile(file_name, file_id, **kwargs):
298 """Wraper over L{WriteFile} that locks the target file.
300 By keeping the target file locked during WriteFile, we ensure that
301 cooperating writers will safely serialise access to the file.
304 @param file_name: the target filename
306 @param file_id: a result from L{GetFileID}
309 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
311 filelock.LockFile(fd)
312 if file_id is not None:
313 disk_id = GetFileID(fd=fd)
314 if not VerifyFileID(disk_id, file_id):
315 raise errors.LockError("Cannot overwrite file %s, it has been modified"
316 " since last written" % file_name)
317 return WriteFile(file_name, **kwargs)
322 def ReadOneLineFile(file_name, strict=False):
323 """Return the first non-empty line from a file.
325 @type strict: boolean
326 @param strict: if True, abort if the file has more than one
330 file_lines = ReadFile(file_name).splitlines()
331 full_lines = filter(bool, file_lines)
332 if not file_lines or not full_lines:
333 raise errors.GenericError("No data in one-liner file %s" % file_name)
334 elif strict and len(full_lines) > 1:
335 raise errors.GenericError("Too many lines in one-liner file %s" %
340 def RemoveFile(filename):
341 """Remove a file ignoring some errors.
343 Remove a file, ignoring non-existing ones or directories. Other
347 @param filename: the file to be removed
353 if err.errno not in (errno.ENOENT, errno.EISDIR):
357 def RemoveDir(dirname):
358 """Remove an empty directory.
360 Remove a directory, ignoring non-existing ones.
361 Other errors are passed. This includes the case,
362 where the directory is not empty, so it can't be removed.
365 @param dirname: the empty directory to be removed
371 if err.errno != errno.ENOENT:
375 def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None,
379 This just creates the very least directory if it does not exist and C{mkdir}
383 @param old: Original path
387 @param mkdir: Whether to create target directory if it doesn't exist
388 @type mkdir_mode: int
389 @param mkdir_mode: Mode for newly created directories
391 @param dir_uid: The uid for the (if fresh created) dir
393 @param dir_gid: The gid for the (if fresh created) dir
397 return os.rename(old, new)
399 # In at least one use case of this function, the job queue, directory
400 # creation is very rare. Checking for the directory before renaming is not
402 if mkdir and err.errno == errno.ENOENT:
403 # Create directory and try again
404 dir_path = os.path.dirname(new)
405 MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid)
407 return os.rename(old, new)
412 def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True,
413 _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
414 """Enforces that given path has given permissions.
416 @param path: The path to the file
417 @param mode: The mode of the file
418 @param uid: The uid of the owner of this file
419 @param gid: The gid of the owner of this file
420 @param must_exist: Specifies if non-existance of path will be an error
421 @param _chmod_fn: chmod function to use (unittest only)
422 @param _chown_fn: chown function to use (unittest only)
425 logging.debug("Checking %s", path)
427 # chown takes -1 if you want to keep one part of the ownership, however
428 # None is Python standard for that. So we remap them here.
437 fmode = stat.S_IMODE(st[stat.ST_MODE])
439 logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode)
440 _chmod_fn(path, mode)
442 if max(uid, gid) > -1:
443 fuid = st[stat.ST_UID]
444 fgid = st[stat.ST_GID]
445 if fuid != uid or fgid != gid:
446 logging.debug("Changing owner of %s from UID %s/GID %s to"
447 " UID %s/GID %s", path, fuid, fgid, uid, gid)
448 _chown_fn(path, uid, gid)
449 except EnvironmentError, err:
450 if err.errno == errno.ENOENT:
452 raise errors.GenericError("Path %s should exist, but does not" % path)
454 raise errors.GenericError("Error while changing permissions on %s: %s" %
458 def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat,
459 _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
460 """Enforces that given path is a dir and has given mode, uid and gid set.
462 @param path: The path to the file
463 @param mode: The mode of the file
464 @param uid: The uid of the owner of this file
465 @param gid: The gid of the owner of this file
466 @param _lstat_fn: Stat function to use (unittest only)
467 @param _mkdir_fn: mkdir function to use (unittest only)
468 @param _perm_fn: permission setter function to use (unittest only)
471 logging.debug("Checking directory %s", path)
473 # We don't want to follow symlinks
475 except EnvironmentError, err:
476 if err.errno != errno.ENOENT:
477 raise errors.GenericError("stat(2) on %s failed: %s" % (path, err))
480 if not stat.S_ISDIR(st[stat.ST_MODE]):
481 raise errors.GenericError(("Path %s is expected to be a directory, but "
484 _perm_fn(path, mode, uid=uid, gid=gid)
487 def Makedirs(path, mode=0750):
488 """Super-mkdir; create a leaf directory and all intermediate ones.
490 This is a wrapper around C{os.makedirs} adding error handling not implemented
495 os.makedirs(path, mode)
497 # Ignore EEXIST. This is only handled in os.makedirs as included in
498 # Python 2.5 and above.
499 if err.errno != errno.EEXIST or not os.path.exists(path):
503 def TimestampForFilename():
504 """Returns the current time formatted for filenames.
506 The format doesn't contain colons as some shells and applications treat them
507 as separators. Uses the local timezone.
510 return time.strftime("%Y-%m-%d_%H_%M_%S")
513 def CreateBackup(file_name):
514 """Creates a backup of a file.
517 @param file_name: file to be backed up
519 @return: the path to the newly created backup
520 @raise errors.ProgrammerError: for invalid file names
523 if not os.path.isfile(file_name):
524 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
527 prefix = ("%s.backup-%s." %
528 (os.path.basename(file_name), TimestampForFilename()))
529 dir_name = os.path.dirname(file_name)
531 fsrc = open(file_name, "rb")
533 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
534 fdst = os.fdopen(fd, "wb")
536 logging.debug("Backing up %s at %s", file_name, backup_name)
537 shutil.copyfileobj(fsrc, fdst)
546 def ListVisibleFiles(path, _is_mountpoint=os.path.ismount):
547 """Returns a list of visible files in a directory.
550 @param path: the directory to enumerate
552 @return: the list of all files not starting with a dot
553 @raise ProgrammerError: if L{path} is not an absolue and normalized path
556 if not IsNormAbsPath(path):
557 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
558 " absolute/normalized: '%s'" % path)
560 mountpoint = _is_mountpoint(path)
565 Ignores files starting with a dot (".") as by Unix convention they're
566 considered hidden. The "lost+found" directory found at the root of some
567 filesystems is also hidden.
570 return not (name.startswith(".") or
571 (mountpoint and name == _LOST_AND_FOUND and
572 os.path.isdir(os.path.join(path, name))))
574 return filter(fn, os.listdir(path))
577 def EnsureDirs(dirs):
578 """Make required directories, if they don't exist.
580 @param dirs: list of tuples (dir_name, dir_mode)
581 @type dirs: list of (string, integer)
584 for dir_name, dir_mode in dirs:
586 os.mkdir(dir_name, dir_mode)
587 except EnvironmentError, err:
588 if err.errno != errno.EEXIST:
589 raise errors.GenericError("Cannot create needed directory"
590 " '%s': %s" % (dir_name, err))
592 os.chmod(dir_name, dir_mode)
593 except EnvironmentError, err:
594 raise errors.GenericError("Cannot change directory permissions on"
595 " '%s': %s" % (dir_name, err))
596 if not os.path.isdir(dir_name):
597 raise errors.GenericError("%s is not a directory" % dir_name)
600 def FindFile(name, search_path, test=os.path.exists):
601 """Look for a filesystem object in a given path.
603 This is an abstract method to search for filesystem object (files,
604 dirs) under a given search path.
607 @param name: the name to look for
608 @type search_path: str
609 @param search_path: location to start at
611 @param test: a function taking one argument that should return True
612 if the a given object is valid; the default value is
613 os.path.exists, causing only existing files to be returned
615 @return: full path to the object if found, None otherwise
618 # validate the filename mask
619 if constants.EXT_PLUGIN_MASK.match(name) is None:
620 logging.critical("Invalid value passed for external script name: '%s'",
624 for dir_name in search_path:
625 # FIXME: investigate switch to PathJoin
626 item_name = os.path.sep.join([dir_name, name])
627 # check the user test and that we're indeed resolving to the given
629 if test(item_name) and os.path.basename(item_name) == name:
634 def IsNormAbsPath(path):
635 """Check whether a path is absolute and also normalized
637 This avoids things like /dir/../../other/path to be valid.
640 return os.path.normpath(path) == path and os.path.isabs(path)
643 def IsBelowDir(root, other_path):
644 """Check whether a path is below a root dir.
646 This works around the nasty byte-byte comparison of commonprefix.
649 if not (os.path.isabs(root) and os.path.isabs(other_path)):
650 raise ValueError("Provided paths '%s' and '%s' are not absolute" %
653 norm_other = os.path.normpath(other_path)
655 if norm_other == os.sep:
656 # The root directory can never be below another path
659 norm_root = os.path.normpath(root)
661 if norm_root == os.sep:
662 # This is the root directory, no need to add another slash
663 prepared_root = norm_root
665 prepared_root = "%s%s" % (norm_root, os.sep)
667 return os.path.commonprefix([prepared_root, norm_other]) == prepared_root
671 """Safe-join a list of path components.
674 - the first argument must be an absolute path
675 - no component in the path must have backtracking (e.g. /../),
676 since we check for normalization at the end
678 @param args: the path components to be joined
679 @raise ValueError: for invalid paths
682 # ensure we're having at least one path passed in
684 # ensure the first component is an absolute and normalized path name
686 if not IsNormAbsPath(root):
687 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
688 result = os.path.join(*args)
689 # ensure that the whole path is normalized
690 if not IsNormAbsPath(result):
691 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
692 # check that we're still under the original prefix
693 if not IsBelowDir(root, result):
694 raise ValueError("Error: path joining resulted in different prefix"
695 " (%s != %s)" % (result, root))
699 def TailFile(fname, lines=20):
700 """Return the last lines from a file.
702 @note: this function will only read and parse the last 4KB of
703 the file; if the lines are very long, it could be that less
704 than the requested number of lines are returned
706 @param fname: the file name
708 @param lines: the (maximum) number of lines to return
711 fd = open(fname, "r")
715 pos = max(0, pos - 4096)
721 rows = raw_data.splitlines()
725 def BytesToMebibyte(value):
726 """Converts bytes to mebibytes.
729 @param value: Value in bytes
731 @return: Value in mebibytes
734 return int(round(value / (1024.0 * 1024.0), 0))
737 def CalculateDirectorySize(path):
738 """Calculates the size of a directory recursively.
741 @param path: Path to directory
743 @return: Size in mebibytes
748 for (curpath, _, files) in os.walk(path):
749 for filename in files:
750 st = os.lstat(PathJoin(curpath, filename))
753 return BytesToMebibyte(size)
756 def GetFilesystemStats(path):
757 """Returns the total and free space on a filesystem.
760 @param path: Path on filesystem to be examined
762 @return: tuple of (Total space, Free space) in mebibytes
765 st = os.statvfs(path)
767 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
768 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
769 return (tsize, fsize)
772 def ReadPidFile(pidfile):
773 """Read a pid from a file.
775 @type pidfile: string
776 @param pidfile: path to the file containing the pid
778 @return: The process id, if the file exists and contains a valid PID,
783 raw_data = ReadOneLineFile(pidfile)
784 except EnvironmentError, err:
785 if err.errno != errno.ENOENT:
786 logging.exception("Can't read pid file")
789 return _ParsePidFileContents(raw_data)
792 def _ParsePidFileContents(data):
793 """Tries to extract a process ID from a PID file's content.
797 @return: Zero if nothing could be read, PID otherwise
802 except (TypeError, ValueError):
803 logging.info("Can't parse pid file contents", exc_info=True)
809 def ReadLockedPidFile(path):
810 """Reads a locked PID file.
812 This can be used together with L{utils.process.StartDaemon}.
815 @param path: Path to PID file
816 @return: PID as integer or, if file was unlocked or couldn't be opened, None
820 fd = os.open(path, os.O_RDONLY)
821 except EnvironmentError, err:
822 if err.errno == errno.ENOENT:
823 # PID file doesn't exist
829 # Try to acquire lock
830 filelock.LockFile(fd)
831 except errors.LockError:
832 # Couldn't lock, daemon is running
833 return int(os.read(fd, 100))
840 def _SplitSshKey(key):
841 """Splits a line for SSH's C{authorized_keys} file.
843 If the line has no options (e.g. no C{command="..."}), only the significant
844 parts, the key type and its hash, are used. Otherwise the whole line is used
845 (split at whitespace).
854 if parts and parts[0] in constants.SSHAK_ALL:
855 # If the key has no options in front of it, we only want the significant
857 return (False, parts[:2])
859 # Can't properly split the line, so use everything
863 def AddAuthorizedKey(file_obj, key):
864 """Adds an SSH public key to an authorized_keys file.
866 @type file_obj: str or file handle
867 @param file_obj: path to authorized_keys file
869 @param key: string containing key
872 key_fields = _SplitSshKey(key)
874 if isinstance(file_obj, basestring):
875 f = open(file_obj, "a+")
882 # Ignore whitespace changes
883 if _SplitSshKey(line) == key_fields:
885 nl = line.endswith("\n")
889 f.write(key.rstrip("\r\n"))
896 def RemoveAuthorizedKey(file_name, key):
897 """Removes an SSH public key from an authorized_keys file.
900 @param file_name: path to authorized_keys file
902 @param key: string containing key
905 key_fields = _SplitSshKey(key)
907 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
909 out = os.fdopen(fd, "w")
911 f = open(file_name, "r")
914 # Ignore whitespace changes while comparing lines
915 if _SplitSshKey(line) != key_fields:
919 os.rename(tmpname, file_name)
929 def DaemonPidFileName(name):
930 """Compute a ganeti pid file absolute path
933 @param name: the daemon name
935 @return: the full path to the pidfile corresponding to the given
939 return PathJoin(pathutils.RUN_DIR, "%s.pid" % name)
942 def WritePidFile(pidfile):
943 """Write the current process pidfile.
945 @type pidfile: string
946 @param pidfile: the path to the file to be written
947 @raise errors.LockError: if the pid file already exists and
948 points to a live process
950 @return: the file descriptor of the lock file; do not close this unless
951 you want to unlock the pid file
954 # We don't rename nor truncate the file to not drop locks under
956 fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0600)
958 # Lock the PID file (and fail if not possible to do so). Any code
959 # wanting to send a signal to the daemon should try to lock the PID
960 # file before reading it. If acquiring the lock succeeds, the daemon is
961 # no longer running and the signal should not be sent.
963 filelock.LockFile(fd_pidfile)
964 except errors.LockError:
965 msg = ["PID file '%s' is already locked by another process" % pidfile]
966 # Try to read PID file
967 pid = _ParsePidFileContents(os.read(fd_pidfile, 100))
969 msg.append(", PID read from file is %s" % pid)
970 raise errors.PidFileLockError("".join(msg))
972 os.write(fd_pidfile, "%d\n" % os.getpid())
977 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
978 """Reads the watcher pause file.
980 @type filename: string
981 @param filename: Path to watcher pause file
982 @type now: None, float or int
983 @param now: Current time as Unix timestamp
984 @type remove_after: int
985 @param remove_after: Remove watcher pause file after specified amount of
986 seconds past the pause end time
993 value = ReadFile(filename)
995 if err.errno != errno.ENOENT:
999 if value is not None:
1003 logging.warning(("Watcher pause file (%s) contains invalid value,"
1004 " removing it"), filename)
1005 RemoveFile(filename)
1008 if value is not None:
1009 # Remove file if it's outdated
1010 if now > (value + remove_after):
1011 RemoveFile(filename)
1021 """Returns a random UUID.
1023 @note: This is a Linux-specific method as it uses the /proc
1028 return ReadFile(constants.RANDOM_UUID_FILE, size=128).rstrip("\n")
1031 class TemporaryFileManager(object):
1032 """Stores the list of files to be deleted and removes them on demand.
1042 def Add(self, filename):
1043 """Add file to list of files to be deleted.
1045 @type filename: string
1046 @param filename: path to filename to be added
1049 self._files.append(filename)
1051 def Remove(self, filename):
1052 """Remove file from list of files to be deleted.
1054 @type filename: string
1055 @param filename: path to filename to be deleted
1058 self._files.remove(filename)
1061 """Delete all files marked for deletion
1065 RemoveFile(self._files.pop())