4 # Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for I/O.
33 from ganeti import errors
34 from ganeti import constants
35 from ganeti import pathutils
36 from ganeti.utils import filelock
39 #: Path generating random UUID
40 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
42 #: Directory used by fsck(8) to store recovered data, usually at a file
43 #: system's root directory
44 _LOST_AND_FOUND = "lost+found"
46 # Possible values for keep_perms in WriteFile()
59 """Format an EnvironmentError exception.
61 If the L{err} argument has an errno attribute, it will be looked up
62 and converted into a textual C{E...} description. Otherwise the
63 string representation of the error will be returned.
65 @type err: L{EnvironmentError}
66 @param err: the exception to format
69 if hasattr(err, "errno"):
70 detail = errno.errorcode[err.errno]
77 """Helper to store file handle's C{fstat}.
79 Useful in combination with L{ReadFile}'s C{preread} parameter.
83 """Initializes this class.
88 def __call__(self, fh):
89 """Calls C{fstat} on file handle.
92 self.st = os.fstat(fh.fileno())
95 def ReadFile(file_name, size=-1, preread=None):
99 @param size: Read at most size bytes (if negative, entire file)
100 @type preread: callable receiving file handle as single parameter
101 @param preread: Function called before file is read
103 @return: the (possibly partial) content of the file
106 f = open(file_name, "r")
116 def WriteFile(file_name, fn=None, data=None,
117 mode=None, uid=-1, gid=-1,
118 atime=None, mtime=None, close=True,
119 dry_run=False, backup=False,
120 prewrite=None, postwrite=None, keep_perms=KP_NEVER):
121 """(Over)write a file atomically.
123 The file_name and either fn (a function taking one argument, the
124 file descriptor, and which should write the data to it) or data (the
125 contents of the file) must be passed. The other arguments are
126 optional and allow setting the file mode, owner and group, and the
127 mtime/atime of the file.
129 If the function doesn't raise an exception, it has succeeded and the
130 target file has the new contents. If the function has raised an
131 exception, an existing target file should be unmodified and the
132 temporary file should be removed.
135 @param file_name: the target filename
137 @param fn: content writing function, called with
138 file descriptor as parameter
140 @param data: contents of the file
142 @param mode: file mode
144 @param uid: the owner of the file
146 @param gid: the group of the file
148 @param atime: a custom access time to be set on the file
150 @param mtime: a custom modification time to be set on the file
152 @param close: whether to close file after writing it
153 @type prewrite: callable
154 @param prewrite: function to be called before writing content
155 @type postwrite: callable
156 @param postwrite: function to be called after writing content
157 @type keep_perms: members of L{KEEP_PERMS_VALUES}
158 @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are
159 taken from the other parameters; if L{KP_ALWAYS}, owner, group, and
160 mode are copied from the existing file; if L{KP_IF_EXISTS}, owner,
161 group, and mode are taken from the file, and if the file doesn't
162 exist, they are taken from the other parameters. It is an error to
163 pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid},
164 or C{mode} are set to non-default values.
167 @return: None if the 'close' parameter evaluates to True,
168 otherwise the file descriptor
170 @raise errors.ProgrammerError: if any of the arguments are not valid
173 if not os.path.isabs(file_name):
174 raise errors.ProgrammerError("Path passed to WriteFile is not"
175 " absolute: '%s'" % file_name)
177 if [fn, data].count(None) != 1:
178 raise errors.ProgrammerError("fn or data required")
180 if [atime, mtime].count(None) == 1:
181 raise errors.ProgrammerError("Both atime and mtime must be either"
184 if not keep_perms in KEEP_PERMS_VALUES:
185 raise errors.ProgrammerError("Invalid value for keep_perms: %s" %
187 if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None):
188 raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid',"
189 " and 'mode' cannot be set")
191 if backup and not dry_run and os.path.isfile(file_name):
192 CreateBackup(file_name)
194 if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS:
195 # os.stat() raises an exception if the file doesn't exist
197 file_stat = os.stat(file_name)
198 mode = stat.S_IMODE(file_stat.st_mode)
199 uid = file_stat.st_uid
200 gid = file_stat.st_gid
202 if keep_perms == KP_ALWAYS:
204 # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist
206 # Whether temporary file needs to be removed (e.g. if any error occurs)
212 (dir_name, base_name) = os.path.split(file_name)
213 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
217 if uid != -1 or gid != -1:
218 os.chown(new_name, uid, gid)
220 os.chmod(new_name, mode)
221 if callable(prewrite):
224 if isinstance(data, unicode):
226 assert isinstance(data, str)
229 while offset < to_write:
230 written = os.write(fd, buffer(data, offset))
232 assert written <= to_write - offset
234 assert offset == to_write
237 if callable(postwrite):
240 if atime is not None and mtime is not None:
241 os.utime(new_name, (atime, mtime))
243 # Close file unless the file descriptor should be returned
249 # Rename file to destination name
251 os.rename(new_name, file_name)
252 # Successful, no need to remove anymore
261 def GetFileID(path=None, fd=None):
262 """Returns the file 'id', i.e. the dev/inode and mtime information.
264 Either the path to the file or the fd must be given.
266 @param path: the file path
267 @param fd: a file descriptor
268 @return: a tuple of (device number, inode number, mtime)
271 if [path, fd].count(None) != 1:
272 raise errors.ProgrammerError("One and only one of fd/path must be given")
279 return (st.st_dev, st.st_ino, st.st_mtime)
282 def VerifyFileID(fi_disk, fi_ours):
283 """Verifies that two file IDs are matching.
285 Differences in the inode/device are not accepted, but and older
286 timestamp for fi_disk is accepted.
288 @param fi_disk: tuple (dev, inode, mtime) representing the actual
290 @param fi_ours: tuple (dev, inode, mtime) representing the last
295 (d1, i1, m1) = fi_disk
296 (d2, i2, m2) = fi_ours
298 return (d1, i1) == (d2, i2) and m1 <= m2
301 def SafeWriteFile(file_name, file_id, **kwargs):
302 """Wraper over L{WriteFile} that locks the target file.
304 By keeping the target file locked during WriteFile, we ensure that
305 cooperating writers will safely serialise access to the file.
308 @param file_name: the target filename
310 @param file_id: a result from L{GetFileID}
313 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
315 filelock.LockFile(fd)
316 if file_id is not None:
317 disk_id = GetFileID(fd=fd)
318 if not VerifyFileID(disk_id, file_id):
319 raise errors.LockError("Cannot overwrite file %s, it has been modified"
320 " since last written" % file_name)
321 return WriteFile(file_name, **kwargs)
326 def ReadOneLineFile(file_name, strict=False):
327 """Return the first non-empty line from a file.
329 @type strict: boolean
330 @param strict: if True, abort if the file has more than one
334 file_lines = ReadFile(file_name).splitlines()
335 full_lines = filter(bool, file_lines)
336 if not file_lines or not full_lines:
337 raise errors.GenericError("No data in one-liner file %s" % file_name)
338 elif strict and len(full_lines) > 1:
339 raise errors.GenericError("Too many lines in one-liner file %s" %
344 def RemoveFile(filename):
345 """Remove a file ignoring some errors.
347 Remove a file, ignoring non-existing ones or directories. Other
351 @param filename: the file to be removed
357 if err.errno not in (errno.ENOENT, errno.EISDIR):
361 def RemoveDir(dirname):
362 """Remove an empty directory.
364 Remove a directory, ignoring non-existing ones.
365 Other errors are passed. This includes the case,
366 where the directory is not empty, so it can't be removed.
369 @param dirname: the empty directory to be removed
375 if err.errno != errno.ENOENT:
379 def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None,
383 This just creates the very least directory if it does not exist and C{mkdir}
387 @param old: Original path
391 @param mkdir: Whether to create target directory if it doesn't exist
392 @type mkdir_mode: int
393 @param mkdir_mode: Mode for newly created directories
395 @param dir_uid: The uid for the (if fresh created) dir
397 @param dir_gid: The gid for the (if fresh created) dir
401 return os.rename(old, new)
403 # In at least one use case of this function, the job queue, directory
404 # creation is very rare. Checking for the directory before renaming is not
406 if mkdir and err.errno == errno.ENOENT:
407 # Create directory and try again
408 dir_path = os.path.dirname(new)
409 MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid)
411 return os.rename(old, new)
416 def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True,
417 _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
418 """Enforces that given path has given permissions.
420 @param path: The path to the file
421 @param mode: The mode of the file
422 @param uid: The uid of the owner of this file
423 @param gid: The gid of the owner of this file
424 @param must_exist: Specifies if non-existance of path will be an error
425 @param _chmod_fn: chmod function to use (unittest only)
426 @param _chown_fn: chown function to use (unittest only)
429 logging.debug("Checking %s", path)
431 # chown takes -1 if you want to keep one part of the ownership, however
432 # None is Python standard for that. So we remap them here.
441 fmode = stat.S_IMODE(st[stat.ST_MODE])
443 logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode)
444 _chmod_fn(path, mode)
446 if max(uid, gid) > -1:
447 fuid = st[stat.ST_UID]
448 fgid = st[stat.ST_GID]
449 if fuid != uid or fgid != gid:
450 logging.debug("Changing owner of %s from UID %s/GID %s to"
451 " UID %s/GID %s", path, fuid, fgid, uid, gid)
452 _chown_fn(path, uid, gid)
453 except EnvironmentError, err:
454 if err.errno == errno.ENOENT:
456 raise errors.GenericError("Path %s should exist, but does not" % path)
458 raise errors.GenericError("Error while changing permissions on %s: %s" %
462 def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat,
463 _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
464 """Enforces that given path is a dir and has given mode, uid and gid set.
466 @param path: The path to the file
467 @param mode: The mode of the file
468 @param uid: The uid of the owner of this file
469 @param gid: The gid of the owner of this file
470 @param _lstat_fn: Stat function to use (unittest only)
471 @param _mkdir_fn: mkdir function to use (unittest only)
472 @param _perm_fn: permission setter function to use (unittest only)
475 logging.debug("Checking directory %s", path)
477 # We don't want to follow symlinks
479 except EnvironmentError, err:
480 if err.errno != errno.ENOENT:
481 raise errors.GenericError("stat(2) on %s failed: %s" % (path, err))
484 if not stat.S_ISDIR(st[stat.ST_MODE]):
485 raise errors.GenericError(("Path %s is expected to be a directory, but "
488 _perm_fn(path, mode, uid=uid, gid=gid)
491 def Makedirs(path, mode=0750):
492 """Super-mkdir; create a leaf directory and all intermediate ones.
494 This is a wrapper around C{os.makedirs} adding error handling not implemented
499 os.makedirs(path, mode)
501 # Ignore EEXIST. This is only handled in os.makedirs as included in
502 # Python 2.5 and above.
503 if err.errno != errno.EEXIST or not os.path.exists(path):
507 def TimestampForFilename():
508 """Returns the current time formatted for filenames.
510 The format doesn't contain colons as some shells and applications treat them
511 as separators. Uses the local timezone.
514 return time.strftime("%Y-%m-%d_%H_%M_%S")
517 def CreateBackup(file_name):
518 """Creates a backup of a file.
521 @param file_name: file to be backed up
523 @return: the path to the newly created backup
524 @raise errors.ProgrammerError: for invalid file names
527 if not os.path.isfile(file_name):
528 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
531 prefix = ("%s.backup-%s." %
532 (os.path.basename(file_name), TimestampForFilename()))
533 dir_name = os.path.dirname(file_name)
535 fsrc = open(file_name, "rb")
537 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
538 fdst = os.fdopen(fd, "wb")
540 logging.debug("Backing up %s at %s", file_name, backup_name)
541 shutil.copyfileobj(fsrc, fdst)
550 def ListVisibleFiles(path, _is_mountpoint=os.path.ismount):
551 """Returns a list of visible files in a directory.
554 @param path: the directory to enumerate
556 @return: the list of all files not starting with a dot
557 @raise ProgrammerError: if L{path} is not an absolue and normalized path
560 if not IsNormAbsPath(path):
561 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
562 " absolute/normalized: '%s'" % path)
564 mountpoint = _is_mountpoint(path)
569 Ignores files starting with a dot (".") as by Unix convention they're
570 considered hidden. The "lost+found" directory found at the root of some
571 filesystems is also hidden.
574 return not (name.startswith(".") or
575 (mountpoint and name == _LOST_AND_FOUND and
576 os.path.isdir(os.path.join(path, name))))
578 return filter(fn, os.listdir(path))
581 def EnsureDirs(dirs):
582 """Make required directories, if they don't exist.
584 @param dirs: list of tuples (dir_name, dir_mode)
585 @type dirs: list of (string, integer)
588 for dir_name, dir_mode in dirs:
590 os.mkdir(dir_name, dir_mode)
591 except EnvironmentError, err:
592 if err.errno != errno.EEXIST:
593 raise errors.GenericError("Cannot create needed directory"
594 " '%s': %s" % (dir_name, err))
596 os.chmod(dir_name, dir_mode)
597 except EnvironmentError, err:
598 raise errors.GenericError("Cannot change directory permissions on"
599 " '%s': %s" % (dir_name, err))
600 if not os.path.isdir(dir_name):
601 raise errors.GenericError("%s is not a directory" % dir_name)
604 def FindFile(name, search_path, test=os.path.exists):
605 """Look for a filesystem object in a given path.
607 This is an abstract method to search for filesystem object (files,
608 dirs) under a given search path.
611 @param name: the name to look for
612 @type search_path: str
613 @param search_path: location to start at
615 @param test: a function taking one argument that should return True
616 if the a given object is valid; the default value is
617 os.path.exists, causing only existing files to be returned
619 @return: full path to the object if found, None otherwise
622 # validate the filename mask
623 if constants.EXT_PLUGIN_MASK.match(name) is None:
624 logging.critical("Invalid value passed for external script name: '%s'",
628 for dir_name in search_path:
629 # FIXME: investigate switch to PathJoin
630 item_name = os.path.sep.join([dir_name, name])
631 # check the user test and that we're indeed resolving to the given
633 if test(item_name) and os.path.basename(item_name) == name:
638 def IsNormAbsPath(path):
639 """Check whether a path is absolute and also normalized
641 This avoids things like /dir/../../other/path to be valid.
644 return os.path.normpath(path) == path and os.path.isabs(path)
647 def IsBelowDir(root, other_path):
648 """Check whether a path is below a root dir.
650 This works around the nasty byte-byte comparisation of commonprefix.
653 if not (os.path.isabs(root) and os.path.isabs(other_path)):
654 raise ValueError("Provided paths '%s' and '%s' are not absolute" %
656 prepared_root = "%s%s" % (os.path.normpath(root), os.sep)
657 return os.path.commonprefix([prepared_root,
658 os.path.normpath(other_path)]) == prepared_root
662 """Safe-join a list of path components.
665 - the first argument must be an absolute path
666 - no component in the path must have backtracking (e.g. /../),
667 since we check for normalization at the end
669 @param args: the path components to be joined
670 @raise ValueError: for invalid paths
673 # ensure we're having at least one path passed in
675 # ensure the first component is an absolute and normalized path name
677 if not IsNormAbsPath(root):
678 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
679 result = os.path.join(*args)
680 # ensure that the whole path is normalized
681 if not IsNormAbsPath(result):
682 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
683 # check that we're still under the original prefix
684 if not IsBelowDir(root, result):
685 raise ValueError("Error: path joining resulted in different prefix"
686 " (%s != %s)" % (result, root))
690 def TailFile(fname, lines=20):
691 """Return the last lines from a file.
693 @note: this function will only read and parse the last 4KB of
694 the file; if the lines are very long, it could be that less
695 than the requested number of lines are returned
697 @param fname: the file name
699 @param lines: the (maximum) number of lines to return
702 fd = open(fname, "r")
706 pos = max(0, pos - 4096)
712 rows = raw_data.splitlines()
716 def BytesToMebibyte(value):
717 """Converts bytes to mebibytes.
720 @param value: Value in bytes
722 @return: Value in mebibytes
725 return int(round(value / (1024.0 * 1024.0), 0))
728 def CalculateDirectorySize(path):
729 """Calculates the size of a directory recursively.
732 @param path: Path to directory
734 @return: Size in mebibytes
739 for (curpath, _, files) in os.walk(path):
740 for filename in files:
741 st = os.lstat(PathJoin(curpath, filename))
744 return BytesToMebibyte(size)
747 def GetFilesystemStats(path):
748 """Returns the total and free space on a filesystem.
751 @param path: Path on filesystem to be examined
753 @return: tuple of (Total space, Free space) in mebibytes
756 st = os.statvfs(path)
758 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
759 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
760 return (tsize, fsize)
763 def ReadPidFile(pidfile):
764 """Read a pid from a file.
766 @type pidfile: string
767 @param pidfile: path to the file containing the pid
769 @return: The process id, if the file exists and contains a valid PID,
774 raw_data = ReadOneLineFile(pidfile)
775 except EnvironmentError, err:
776 if err.errno != errno.ENOENT:
777 logging.exception("Can't read pid file")
780 return _ParsePidFileContents(raw_data)
783 def _ParsePidFileContents(data):
784 """Tries to extract a process ID from a PID file's content.
788 @return: Zero if nothing could be read, PID otherwise
793 except (TypeError, ValueError):
794 logging.info("Can't parse pid file contents", exc_info=True)
800 def ReadLockedPidFile(path):
801 """Reads a locked PID file.
803 This can be used together with L{utils.process.StartDaemon}.
806 @param path: Path to PID file
807 @return: PID as integer or, if file was unlocked or couldn't be opened, None
811 fd = os.open(path, os.O_RDONLY)
812 except EnvironmentError, err:
813 if err.errno == errno.ENOENT:
814 # PID file doesn't exist
820 # Try to acquire lock
821 filelock.LockFile(fd)
822 except errors.LockError:
823 # Couldn't lock, daemon is running
824 return int(os.read(fd, 100))
831 def AddAuthorizedKey(file_obj, key):
832 """Adds an SSH public key to an authorized_keys file.
834 @type file_obj: str or file handle
835 @param file_obj: path to authorized_keys file
837 @param key: string containing key
840 key_fields = key.split()
842 if isinstance(file_obj, basestring):
843 f = open(file_obj, "a+")
850 # Ignore whitespace changes
851 if line.split() == key_fields:
853 nl = line.endswith("\n")
857 f.write(key.rstrip("\r\n"))
864 def RemoveAuthorizedKey(file_name, key):
865 """Removes an SSH public key from an authorized_keys file.
868 @param file_name: path to authorized_keys file
870 @param key: string containing key
873 key_fields = key.split()
875 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
877 out = os.fdopen(fd, "w")
879 f = open(file_name, "r")
882 # Ignore whitespace changes while comparing lines
883 if line.split() != key_fields:
887 os.rename(tmpname, file_name)
897 def DaemonPidFileName(name):
898 """Compute a ganeti pid file absolute path
901 @param name: the daemon name
903 @return: the full path to the pidfile corresponding to the given
907 return PathJoin(pathutils.RUN_DIR, "%s.pid" % name)
910 def WritePidFile(pidfile):
911 """Write the current process pidfile.
913 @type pidfile: string
914 @param pidfile: the path to the file to be written
915 @raise errors.LockError: if the pid file already exists and
916 points to a live process
918 @return: the file descriptor of the lock file; do not close this unless
919 you want to unlock the pid file
922 # We don't rename nor truncate the file to not drop locks under
924 fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0600)
926 # Lock the PID file (and fail if not possible to do so). Any code
927 # wanting to send a signal to the daemon should try to lock the PID
928 # file before reading it. If acquiring the lock succeeds, the daemon is
929 # no longer running and the signal should not be sent.
931 filelock.LockFile(fd_pidfile)
932 except errors.LockError:
933 msg = ["PID file '%s' is already locked by another process" % pidfile]
934 # Try to read PID file
935 pid = _ParsePidFileContents(os.read(fd_pidfile, 100))
937 msg.append(", PID read from file is %s" % pid)
938 raise errors.PidFileLockError("".join(msg))
940 os.write(fd_pidfile, "%d\n" % os.getpid())
945 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
946 """Reads the watcher pause file.
948 @type filename: string
949 @param filename: Path to watcher pause file
950 @type now: None, float or int
951 @param now: Current time as Unix timestamp
952 @type remove_after: int
953 @param remove_after: Remove watcher pause file after specified amount of
954 seconds past the pause end time
961 value = ReadFile(filename)
963 if err.errno != errno.ENOENT:
967 if value is not None:
971 logging.warning(("Watcher pause file (%s) contains invalid value,"
972 " removing it"), filename)
976 if value is not None:
977 # Remove file if it's outdated
978 if now > (value + remove_after):
989 """Returns a random UUID.
991 @note: This is a Linux-specific method as it uses the /proc
996 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
999 class TemporaryFileManager(object):
1000 """Stores the list of files to be deleted and removes them on demand.
1010 def Add(self, filename):
1011 """Add file to list of files to be deleted.
1013 @type filename: string
1014 @param filename: path to filename to be added
1017 self._files.append(filename)
1019 def Remove(self, filename):
1020 """Remove file from list of files to be deleted.
1022 @type filename: string
1023 @param filename: path to filename to be deleted
1026 self._files.remove(filename)
1029 """Delete all files marked for deletion
1033 RemoveFile(self._files.pop())