4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for I/O.
33 from ganeti import errors
34 from ganeti import constants
35 from ganeti.utils import filelock
38 #: Path generating random UUID
39 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
41 #: Directory used by fsck(8) to store recovered data, usually at a file
42 #: system's root directory
43 _LOST_AND_FOUND = "lost+found"
45 # Possible values for keep_perms in WriteFile()
58 """Format an EnvironmentError exception.
60 If the L{err} argument has an errno attribute, it will be looked up
61 and converted into a textual C{E...} description. Otherwise the
62 string representation of the error will be returned.
64 @type err: L{EnvironmentError}
65 @param err: the exception to format
68 if hasattr(err, "errno"):
69 detail = errno.errorcode[err.errno]
75 def ReadFile(file_name, size=-1, preread=None):
79 @param size: Read at most size bytes (if negative, entire file)
80 @type preread: callable receiving file handle as single parameter
81 @param preread: Function called before file is read
83 @return: the (possibly partial) content of the file
86 f = open(file_name, "r")
96 def WriteFile(file_name, fn=None, data=None,
97 mode=None, uid=-1, gid=-1,
98 atime=None, mtime=None, close=True,
99 dry_run=False, backup=False,
100 prewrite=None, postwrite=None, keep_perms=KP_NEVER):
101 """(Over)write a file atomically.
103 The file_name and either fn (a function taking one argument, the
104 file descriptor, and which should write the data to it) or data (the
105 contents of the file) must be passed. The other arguments are
106 optional and allow setting the file mode, owner and group, and the
107 mtime/atime of the file.
109 If the function doesn't raise an exception, it has succeeded and the
110 target file has the new contents. If the function has raised an
111 exception, an existing target file should be unmodified and the
112 temporary file should be removed.
115 @param file_name: the target filename
117 @param fn: content writing function, called with
118 file descriptor as parameter
120 @param data: contents of the file
122 @param mode: file mode
124 @param uid: the owner of the file
126 @param gid: the group of the file
128 @param atime: a custom access time to be set on the file
130 @param mtime: a custom modification time to be set on the file
132 @param close: whether to close file after writing it
133 @type prewrite: callable
134 @param prewrite: function to be called before writing content
135 @type postwrite: callable
136 @param postwrite: function to be called after writing content
137 @type keep_perms: members of L{KEEP_PERMS_VALUES}
138 @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are
139 taken from the other parameters; if L{KP_ALWAYS}, owner, group, and
140 mode are copied from the existing file; if L{KP_IF_EXISTS}, owner,
141 group, and mode are taken from the file, and if the file doesn't
142 exist, they are taken from the other parameters. It is an error to
143 pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid},
144 or C{mode} are set to non-default values.
147 @return: None if the 'close' parameter evaluates to True,
148 otherwise the file descriptor
150 @raise errors.ProgrammerError: if any of the arguments are not valid
153 if not os.path.isabs(file_name):
154 raise errors.ProgrammerError("Path passed to WriteFile is not"
155 " absolute: '%s'" % file_name)
157 if [fn, data].count(None) != 1:
158 raise errors.ProgrammerError("fn or data required")
160 if [atime, mtime].count(None) == 1:
161 raise errors.ProgrammerError("Both atime and mtime must be either"
164 if not keep_perms in KEEP_PERMS_VALUES:
165 raise errors.ProgrammerError("Invalid value for keep_perms: %s" %
167 if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None):
168 raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid',"
169 " and 'mode' cannot be set")
171 if backup and not dry_run and os.path.isfile(file_name):
172 CreateBackup(file_name)
174 if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS:
175 # os.stat() raises an exception if the file doesn't exist
177 file_stat = os.stat(file_name)
178 mode = stat.S_IMODE(file_stat.st_mode)
179 uid = file_stat.st_uid
180 gid = file_stat.st_gid
182 if keep_perms == KP_ALWAYS:
184 # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist
186 # Whether temporary file needs to be removed (e.g. if any error occurs)
192 (dir_name, base_name) = os.path.split(file_name)
193 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
197 if uid != -1 or gid != -1:
198 os.chown(new_name, uid, gid)
200 os.chmod(new_name, mode)
201 if callable(prewrite):
204 if isinstance(data, unicode):
206 assert isinstance(data, str)
209 while offset < to_write:
210 written = os.write(fd, buffer(data, offset))
212 assert written <= to_write - offset
214 assert offset == to_write
217 if callable(postwrite):
220 if atime is not None and mtime is not None:
221 os.utime(new_name, (atime, mtime))
223 # Close file unless the file descriptor should be returned
229 # Rename file to destination name
231 os.rename(new_name, file_name)
232 # Successful, no need to remove anymore
241 def GetFileID(path=None, fd=None):
242 """Returns the file 'id', i.e. the dev/inode and mtime information.
244 Either the path to the file or the fd must be given.
246 @param path: the file path
247 @param fd: a file descriptor
248 @return: a tuple of (device number, inode number, mtime)
251 if [path, fd].count(None) != 1:
252 raise errors.ProgrammerError("One and only one of fd/path must be given")
259 return (st.st_dev, st.st_ino, st.st_mtime)
262 def VerifyFileID(fi_disk, fi_ours):
263 """Verifies that two file IDs are matching.
265 Differences in the inode/device are not accepted, but and older
266 timestamp for fi_disk is accepted.
268 @param fi_disk: tuple (dev, inode, mtime) representing the actual
270 @param fi_ours: tuple (dev, inode, mtime) representing the last
275 (d1, i1, m1) = fi_disk
276 (d2, i2, m2) = fi_ours
278 return (d1, i1) == (d2, i2) and m1 <= m2
281 def SafeWriteFile(file_name, file_id, **kwargs):
282 """Wraper over L{WriteFile} that locks the target file.
284 By keeping the target file locked during WriteFile, we ensure that
285 cooperating writers will safely serialise access to the file.
288 @param file_name: the target filename
290 @param file_id: a result from L{GetFileID}
293 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
295 filelock.LockFile(fd)
296 if file_id is not None:
297 disk_id = GetFileID(fd=fd)
298 if not VerifyFileID(disk_id, file_id):
299 raise errors.LockError("Cannot overwrite file %s, it has been modified"
300 " since last written" % file_name)
301 return WriteFile(file_name, **kwargs)
306 def ReadOneLineFile(file_name, strict=False):
307 """Return the first non-empty line from a file.
309 @type strict: boolean
310 @param strict: if True, abort if the file has more than one
314 file_lines = ReadFile(file_name).splitlines()
315 full_lines = filter(bool, file_lines)
316 if not file_lines or not full_lines:
317 raise errors.GenericError("No data in one-liner file %s" % file_name)
318 elif strict and len(full_lines) > 1:
319 raise errors.GenericError("Too many lines in one-liner file %s" %
324 def RemoveFile(filename):
325 """Remove a file ignoring some errors.
327 Remove a file, ignoring non-existing ones or directories. Other
331 @param filename: the file to be removed
337 if err.errno not in (errno.ENOENT, errno.EISDIR):
341 def RemoveDir(dirname):
342 """Remove an empty directory.
344 Remove a directory, ignoring non-existing ones.
345 Other errors are passed. This includes the case,
346 where the directory is not empty, so it can't be removed.
349 @param dirname: the empty directory to be removed
355 if err.errno != errno.ENOENT:
359 def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None,
363 This just creates the very least directory if it does not exist and C{mkdir}
367 @param old: Original path
371 @param mkdir: Whether to create target directory if it doesn't exist
372 @type mkdir_mode: int
373 @param mkdir_mode: Mode for newly created directories
375 @param dir_uid: The uid for the (if fresh created) dir
377 @param dir_gid: The gid for the (if fresh created) dir
381 return os.rename(old, new)
383 # In at least one use case of this function, the job queue, directory
384 # creation is very rare. Checking for the directory before renaming is not
386 if mkdir and err.errno == errno.ENOENT:
387 # Create directory and try again
388 dir_path = os.path.dirname(new)
389 MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid)
391 return os.rename(old, new)
396 def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True,
397 _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
398 """Enforces that given path has given permissions.
400 @param path: The path to the file
401 @param mode: The mode of the file
402 @param uid: The uid of the owner of this file
403 @param gid: The gid of the owner of this file
404 @param must_exist: Specifies if non-existance of path will be an error
405 @param _chmod_fn: chmod function to use (unittest only)
406 @param _chown_fn: chown function to use (unittest only)
409 logging.debug("Checking %s", path)
411 # chown takes -1 if you want to keep one part of the ownership, however
412 # None is Python standard for that. So we remap them here.
421 fmode = stat.S_IMODE(st[stat.ST_MODE])
423 logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode)
424 _chmod_fn(path, mode)
426 if max(uid, gid) > -1:
427 fuid = st[stat.ST_UID]
428 fgid = st[stat.ST_GID]
429 if fuid != uid or fgid != gid:
430 logging.debug("Changing owner of %s from UID %s/GID %s to"
431 " UID %s/GID %s", path, fuid, fgid, uid, gid)
432 _chown_fn(path, uid, gid)
433 except EnvironmentError, err:
434 if err.errno == errno.ENOENT:
436 raise errors.GenericError("Path %s should exist, but does not" % path)
438 raise errors.GenericError("Error while changing permissions on %s: %s" %
442 def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat,
443 _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
444 """Enforces that given path is a dir and has given mode, uid and gid set.
446 @param path: The path to the file
447 @param mode: The mode of the file
448 @param uid: The uid of the owner of this file
449 @param gid: The gid of the owner of this file
450 @param _lstat_fn: Stat function to use (unittest only)
451 @param _mkdir_fn: mkdir function to use (unittest only)
452 @param _perm_fn: permission setter function to use (unittest only)
455 logging.debug("Checking directory %s", path)
457 # We don't want to follow symlinks
459 except EnvironmentError, err:
460 if err.errno != errno.ENOENT:
461 raise errors.GenericError("stat(2) on %s failed: %s" % (path, err))
464 if not stat.S_ISDIR(st[stat.ST_MODE]):
465 raise errors.GenericError(("Path %s is expected to be a directory, but "
468 _perm_fn(path, mode, uid=uid, gid=gid)
471 def Makedirs(path, mode=0750):
472 """Super-mkdir; create a leaf directory and all intermediate ones.
474 This is a wrapper around C{os.makedirs} adding error handling not implemented
479 os.makedirs(path, mode)
481 # Ignore EEXIST. This is only handled in os.makedirs as included in
482 # Python 2.5 and above.
483 if err.errno != errno.EEXIST or not os.path.exists(path):
487 def TimestampForFilename():
488 """Returns the current time formatted for filenames.
490 The format doesn't contain colons as some shells and applications treat them
491 as separators. Uses the local timezone.
494 return time.strftime("%Y-%m-%d_%H_%M_%S")
497 def CreateBackup(file_name):
498 """Creates a backup of a file.
501 @param file_name: file to be backed up
503 @return: the path to the newly created backup
504 @raise errors.ProgrammerError: for invalid file names
507 if not os.path.isfile(file_name):
508 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
511 prefix = ("%s.backup-%s." %
512 (os.path.basename(file_name), TimestampForFilename()))
513 dir_name = os.path.dirname(file_name)
515 fsrc = open(file_name, "rb")
517 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
518 fdst = os.fdopen(fd, "wb")
520 logging.debug("Backing up %s at %s", file_name, backup_name)
521 shutil.copyfileobj(fsrc, fdst)
530 def ListVisibleFiles(path, _is_mountpoint=os.path.ismount):
531 """Returns a list of visible files in a directory.
534 @param path: the directory to enumerate
536 @return: the list of all files not starting with a dot
537 @raise ProgrammerError: if L{path} is not an absolue and normalized path
540 if not IsNormAbsPath(path):
541 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
542 " absolute/normalized: '%s'" % path)
544 mountpoint = _is_mountpoint(path)
549 Ignores files starting with a dot (".") as by Unix convention they're
550 considered hidden. The "lost+found" directory found at the root of some
551 filesystems is also hidden.
554 return not (name.startswith(".") or
555 (mountpoint and name == _LOST_AND_FOUND and
556 os.path.isdir(os.path.join(path, name))))
558 return filter(fn, os.listdir(path))
561 def EnsureDirs(dirs):
562 """Make required directories, if they don't exist.
564 @param dirs: list of tuples (dir_name, dir_mode)
565 @type dirs: list of (string, integer)
568 for dir_name, dir_mode in dirs:
570 os.mkdir(dir_name, dir_mode)
571 except EnvironmentError, err:
572 if err.errno != errno.EEXIST:
573 raise errors.GenericError("Cannot create needed directory"
574 " '%s': %s" % (dir_name, err))
576 os.chmod(dir_name, dir_mode)
577 except EnvironmentError, err:
578 raise errors.GenericError("Cannot change directory permissions on"
579 " '%s': %s" % (dir_name, err))
580 if not os.path.isdir(dir_name):
581 raise errors.GenericError("%s is not a directory" % dir_name)
584 def FindFile(name, search_path, test=os.path.exists):
585 """Look for a filesystem object in a given path.
587 This is an abstract method to search for filesystem object (files,
588 dirs) under a given search path.
591 @param name: the name to look for
592 @type search_path: str
593 @param search_path: location to start at
595 @param test: a function taking one argument that should return True
596 if the a given object is valid; the default value is
597 os.path.exists, causing only existing files to be returned
599 @return: full path to the object if found, None otherwise
602 # validate the filename mask
603 if constants.EXT_PLUGIN_MASK.match(name) is None:
604 logging.critical("Invalid value passed for external script name: '%s'",
608 for dir_name in search_path:
609 # FIXME: investigate switch to PathJoin
610 item_name = os.path.sep.join([dir_name, name])
611 # check the user test and that we're indeed resolving to the given
613 if test(item_name) and os.path.basename(item_name) == name:
618 def IsNormAbsPath(path):
619 """Check whether a path is absolute and also normalized
621 This avoids things like /dir/../../other/path to be valid.
624 return os.path.normpath(path) == path and os.path.isabs(path)
627 def IsBelowDir(root, other_path):
628 """Check whether a path is below a root dir.
630 This works around the nasty byte-byte comparisation of commonprefix.
633 if not (os.path.isabs(root) and os.path.isabs(other_path)):
634 raise ValueError("Provided paths '%s' and '%s' are not absolute" %
636 prepared_root = "%s%s" % (os.path.normpath(root), os.sep)
637 return os.path.commonprefix([prepared_root,
638 os.path.normpath(other_path)]) == prepared_root
642 """Safe-join a list of path components.
645 - the first argument must be an absolute path
646 - no component in the path must have backtracking (e.g. /../),
647 since we check for normalization at the end
649 @param args: the path components to be joined
650 @raise ValueError: for invalid paths
653 # ensure we're having at least one path passed in
655 # ensure the first component is an absolute and normalized path name
657 if not IsNormAbsPath(root):
658 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
659 result = os.path.join(*args)
660 # ensure that the whole path is normalized
661 if not IsNormAbsPath(result):
662 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
663 # check that we're still under the original prefix
664 if not IsBelowDir(root, result):
665 raise ValueError("Error: path joining resulted in different prefix"
666 " (%s != %s)" % (result, root))
670 def TailFile(fname, lines=20):
671 """Return the last lines from a file.
673 @note: this function will only read and parse the last 4KB of
674 the file; if the lines are very long, it could be that less
675 than the requested number of lines are returned
677 @param fname: the file name
679 @param lines: the (maximum) number of lines to return
682 fd = open(fname, "r")
686 pos = max(0, pos - 4096)
692 rows = raw_data.splitlines()
696 def BytesToMebibyte(value):
697 """Converts bytes to mebibytes.
700 @param value: Value in bytes
702 @return: Value in mebibytes
705 return int(round(value / (1024.0 * 1024.0), 0))
708 def CalculateDirectorySize(path):
709 """Calculates the size of a directory recursively.
712 @param path: Path to directory
714 @return: Size in mebibytes
719 for (curpath, _, files) in os.walk(path):
720 for filename in files:
721 st = os.lstat(PathJoin(curpath, filename))
724 return BytesToMebibyte(size)
727 def GetFilesystemStats(path):
728 """Returns the total and free space on a filesystem.
731 @param path: Path on filesystem to be examined
733 @return: tuple of (Total space, Free space) in mebibytes
736 st = os.statvfs(path)
738 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
739 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
740 return (tsize, fsize)
743 def ReadPidFile(pidfile):
744 """Read a pid from a file.
746 @type pidfile: string
747 @param pidfile: path to the file containing the pid
749 @return: The process id, if the file exists and contains a valid PID,
754 raw_data = ReadOneLineFile(pidfile)
755 except EnvironmentError, err:
756 if err.errno != errno.ENOENT:
757 logging.exception("Can't read pid file")
760 return _ParsePidFileContents(raw_data)
763 def _ParsePidFileContents(data):
764 """Tries to extract a process ID from a PID file's content.
768 @return: Zero if nothing could be read, PID otherwise
773 except (TypeError, ValueError):
774 logging.info("Can't parse pid file contents", exc_info=True)
780 def ReadLockedPidFile(path):
781 """Reads a locked PID file.
783 This can be used together with L{utils.process.StartDaemon}.
786 @param path: Path to PID file
787 @return: PID as integer or, if file was unlocked or couldn't be opened, None
791 fd = os.open(path, os.O_RDONLY)
792 except EnvironmentError, err:
793 if err.errno == errno.ENOENT:
794 # PID file doesn't exist
800 # Try to acquire lock
801 filelock.LockFile(fd)
802 except errors.LockError:
803 # Couldn't lock, daemon is running
804 return int(os.read(fd, 100))
811 def AddAuthorizedKey(file_obj, key):
812 """Adds an SSH public key to an authorized_keys file.
814 @type file_obj: str or file handle
815 @param file_obj: path to authorized_keys file
817 @param key: string containing key
820 key_fields = key.split()
822 if isinstance(file_obj, basestring):
823 f = open(file_obj, "a+")
830 # Ignore whitespace changes
831 if line.split() == key_fields:
833 nl = line.endswith("\n")
837 f.write(key.rstrip("\r\n"))
844 def RemoveAuthorizedKey(file_name, key):
845 """Removes an SSH public key from an authorized_keys file.
848 @param file_name: path to authorized_keys file
850 @param key: string containing key
853 key_fields = key.split()
855 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
857 out = os.fdopen(fd, "w")
859 f = open(file_name, "r")
862 # Ignore whitespace changes while comparing lines
863 if line.split() != key_fields:
867 os.rename(tmpname, file_name)
877 def DaemonPidFileName(name):
878 """Compute a ganeti pid file absolute path
881 @param name: the daemon name
883 @return: the full path to the pidfile corresponding to the given
887 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
890 def WritePidFile(pidfile):
891 """Write the current process pidfile.
893 @type pidfile: string
894 @param pidfile: the path to the file to be written
895 @raise errors.LockError: if the pid file already exists and
896 points to a live process
898 @return: the file descriptor of the lock file; do not close this unless
899 you want to unlock the pid file
902 # We don't rename nor truncate the file to not drop locks under
904 fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0600)
906 # Lock the PID file (and fail if not possible to do so). Any code
907 # wanting to send a signal to the daemon should try to lock the PID
908 # file before reading it. If acquiring the lock succeeds, the daemon is
909 # no longer running and the signal should not be sent.
911 filelock.LockFile(fd_pidfile)
912 except errors.LockError:
913 msg = ["PID file '%s' is already locked by another process" % pidfile]
914 # Try to read PID file
915 pid = _ParsePidFileContents(os.read(fd_pidfile, 100))
917 msg.append(", PID read from file is %s" % pid)
918 raise errors.PidFileLockError("".join(msg))
920 os.write(fd_pidfile, "%d\n" % os.getpid())
925 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
926 """Reads the watcher pause file.
928 @type filename: string
929 @param filename: Path to watcher pause file
930 @type now: None, float or int
931 @param now: Current time as Unix timestamp
932 @type remove_after: int
933 @param remove_after: Remove watcher pause file after specified amount of
934 seconds past the pause end time
941 value = ReadFile(filename)
943 if err.errno != errno.ENOENT:
947 if value is not None:
951 logging.warning(("Watcher pause file (%s) contains invalid value,"
952 " removing it"), filename)
956 if value is not None:
957 # Remove file if it's outdated
958 if now > (value + remove_after):
969 """Returns a random UUID.
971 @note: This is a Linux-specific method as it uses the /proc
976 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
979 class TemporaryFileManager(object):
980 """Stores the list of files to be deleted and removes them on demand.
990 def Add(self, filename):
991 """Add file to list of files to be deleted.
993 @type filename: string
994 @param filename: path to filename to be added
997 self._files.append(filename)
999 def Remove(self, filename):
1000 """Remove file from list of files to be deleted.
1002 @type filename: string
1003 @param filename: path to filename to be deleted
1006 self._files.remove(filename)
1009 """Delete all files marked for deletion
1013 RemoveFile(self._files.pop())