4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for I/O.
32 from ganeti import errors
33 from ganeti import constants
34 from ganeti.utils import filelock
37 #: Path generating random UUID
38 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
41 def ReadFile(file_name, size=-1, preread=None):
45 @param size: Read at most size bytes (if negative, entire file)
46 @type preread: callable receiving file handle as single parameter
47 @param preread: Function called before file is read
49 @return: the (possibly partial) content of the file
52 f = open(file_name, "r")
62 def WriteFile(file_name, fn=None, data=None,
63 mode=None, uid=-1, gid=-1,
64 atime=None, mtime=None, close=True,
65 dry_run=False, backup=False,
66 prewrite=None, postwrite=None):
67 """(Over)write a file atomically.
69 The file_name and either fn (a function taking one argument, the
70 file descriptor, and which should write the data to it) or data (the
71 contents of the file) must be passed. The other arguments are
72 optional and allow setting the file mode, owner and group, and the
73 mtime/atime of the file.
75 If the function doesn't raise an exception, it has succeeded and the
76 target file has the new contents. If the function has raised an
77 exception, an existing target file should be unmodified and the
78 temporary file should be removed.
81 @param file_name: the target filename
83 @param fn: content writing function, called with
84 file descriptor as parameter
86 @param data: contents of the file
88 @param mode: file mode
90 @param uid: the owner of the file
92 @param gid: the group of the file
94 @param atime: a custom access time to be set on the file
96 @param mtime: a custom modification time to be set on the file
98 @param close: whether to close file after writing it
99 @type prewrite: callable
100 @param prewrite: function to be called before writing content
101 @type postwrite: callable
102 @param postwrite: function to be called after writing content
105 @return: None if the 'close' parameter evaluates to True,
106 otherwise the file descriptor
108 @raise errors.ProgrammerError: if any of the arguments are not valid
111 if not os.path.isabs(file_name):
112 raise errors.ProgrammerError("Path passed to WriteFile is not"
113 " absolute: '%s'" % file_name)
115 if [fn, data].count(None) != 1:
116 raise errors.ProgrammerError("fn or data required")
118 if [atime, mtime].count(None) == 1:
119 raise errors.ProgrammerError("Both atime and mtime must be either"
122 if backup and not dry_run and os.path.isfile(file_name):
123 CreateBackup(file_name)
125 # Whether temporary file needs to be removed (e.g. if any error occurs)
131 (dir_name, base_name) = os.path.split(file_name)
132 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
136 if uid != -1 or gid != -1:
137 os.chown(new_name, uid, gid)
139 os.chmod(new_name, mode)
140 if callable(prewrite):
143 if isinstance(data, unicode):
145 assert isinstance(data, str)
148 while offset < to_write:
149 written = os.write(fd, buffer(data, offset))
151 assert written <= to_write - offset
153 assert offset == to_write
156 if callable(postwrite):
159 if atime is not None and mtime is not None:
160 os.utime(new_name, (atime, mtime))
162 # Close file unless the file descriptor should be returned
168 # Rename file to destination name
170 os.rename(new_name, file_name)
171 # Successful, no need to remove anymore
180 def GetFileID(path=None, fd=None):
181 """Returns the file 'id', i.e. the dev/inode and mtime information.
183 Either the path to the file or the fd must be given.
185 @param path: the file path
186 @param fd: a file descriptor
187 @return: a tuple of (device number, inode number, mtime)
190 if [path, fd].count(None) != 1:
191 raise errors.ProgrammerError("One and only one of fd/path must be given")
198 return (st.st_dev, st.st_ino, st.st_mtime)
201 def VerifyFileID(fi_disk, fi_ours):
202 """Verifies that two file IDs are matching.
204 Differences in the inode/device are not accepted, but and older
205 timestamp for fi_disk is accepted.
207 @param fi_disk: tuple (dev, inode, mtime) representing the actual
209 @param fi_ours: tuple (dev, inode, mtime) representing the last
214 (d1, i1, m1) = fi_disk
215 (d2, i2, m2) = fi_ours
217 return (d1, i1) == (d2, i2) and m1 <= m2
220 def SafeWriteFile(file_name, file_id, **kwargs):
221 """Wraper over L{WriteFile} that locks the target file.
223 By keeping the target file locked during WriteFile, we ensure that
224 cooperating writers will safely serialise access to the file.
227 @param file_name: the target filename
229 @param file_id: a result from L{GetFileID}
232 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
234 filelock.LockFile(fd)
235 if file_id is not None:
236 disk_id = GetFileID(fd=fd)
237 if not VerifyFileID(disk_id, file_id):
238 raise errors.LockError("Cannot overwrite file %s, it has been modified"
239 " since last written" % file_name)
240 return WriteFile(file_name, **kwargs)
245 def ReadOneLineFile(file_name, strict=False):
246 """Return the first non-empty line from a file.
248 @type strict: boolean
249 @param strict: if True, abort if the file has more than one
253 file_lines = ReadFile(file_name).splitlines()
254 full_lines = filter(bool, file_lines)
255 if not file_lines or not full_lines:
256 raise errors.GenericError("No data in one-liner file %s" % file_name)
257 elif strict and len(full_lines) > 1:
258 raise errors.GenericError("Too many lines in one-liner file %s" %
263 def RemoveFile(filename):
264 """Remove a file ignoring some errors.
266 Remove a file, ignoring non-existing ones or directories. Other
270 @param filename: the file to be removed
276 if err.errno not in (errno.ENOENT, errno.EISDIR):
280 def RemoveDir(dirname):
281 """Remove an empty directory.
283 Remove a directory, ignoring non-existing ones.
284 Other errors are passed. This includes the case,
285 where the directory is not empty, so it can't be removed.
288 @param dirname: the empty directory to be removed
294 if err.errno != errno.ENOENT:
298 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
302 @param old: Original path
306 @param mkdir: Whether to create target directory if it doesn't exist
307 @type mkdir_mode: int
308 @param mkdir_mode: Mode for newly created directories
312 return os.rename(old, new)
314 # In at least one use case of this function, the job queue, directory
315 # creation is very rare. Checking for the directory before renaming is not
317 if mkdir and err.errno == errno.ENOENT:
318 # Create directory and try again
319 Makedirs(os.path.dirname(new), mode=mkdir_mode)
321 return os.rename(old, new)
326 def Makedirs(path, mode=0750):
327 """Super-mkdir; create a leaf directory and all intermediate ones.
329 This is a wrapper around C{os.makedirs} adding error handling not implemented
334 os.makedirs(path, mode)
336 # Ignore EEXIST. This is only handled in os.makedirs as included in
337 # Python 2.5 and above.
338 if err.errno != errno.EEXIST or not os.path.exists(path):
342 def TimestampForFilename():
343 """Returns the current time formatted for filenames.
345 The format doesn't contain colons as some shells and applications treat them
346 as separators. Uses the local timezone.
349 return time.strftime("%Y-%m-%d_%H_%M_%S")
352 def CreateBackup(file_name):
353 """Creates a backup of a file.
356 @param file_name: file to be backed up
358 @return: the path to the newly created backup
359 @raise errors.ProgrammerError: for invalid file names
362 if not os.path.isfile(file_name):
363 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
366 prefix = ("%s.backup-%s." %
367 (os.path.basename(file_name), TimestampForFilename()))
368 dir_name = os.path.dirname(file_name)
370 fsrc = open(file_name, "rb")
372 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
373 fdst = os.fdopen(fd, "wb")
375 logging.debug("Backing up %s at %s", file_name, backup_name)
376 shutil.copyfileobj(fsrc, fdst)
385 def ListVisibleFiles(path):
386 """Returns a list of visible files in a directory.
389 @param path: the directory to enumerate
391 @return: the list of all files not starting with a dot
392 @raise ProgrammerError: if L{path} is not an absolue and normalized path
395 if not IsNormAbsPath(path):
396 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
397 " absolute/normalized: '%s'" % path)
398 files = [i for i in os.listdir(path) if not i.startswith(".")]
402 def EnsureDirs(dirs):
403 """Make required directories, if they don't exist.
405 @param dirs: list of tuples (dir_name, dir_mode)
406 @type dirs: list of (string, integer)
409 for dir_name, dir_mode in dirs:
411 os.mkdir(dir_name, dir_mode)
412 except EnvironmentError, err:
413 if err.errno != errno.EEXIST:
414 raise errors.GenericError("Cannot create needed directory"
415 " '%s': %s" % (dir_name, err))
417 os.chmod(dir_name, dir_mode)
418 except EnvironmentError, err:
419 raise errors.GenericError("Cannot change directory permissions on"
420 " '%s': %s" % (dir_name, err))
421 if not os.path.isdir(dir_name):
422 raise errors.GenericError("%s is not a directory" % dir_name)
425 def FindFile(name, search_path, test=os.path.exists):
426 """Look for a filesystem object in a given path.
428 This is an abstract method to search for filesystem object (files,
429 dirs) under a given search path.
432 @param name: the name to look for
433 @type search_path: str
434 @param search_path: location to start at
436 @param test: a function taking one argument that should return True
437 if the a given object is valid; the default value is
438 os.path.exists, causing only existing files to be returned
440 @return: full path to the object if found, None otherwise
443 # validate the filename mask
444 if constants.EXT_PLUGIN_MASK.match(name) is None:
445 logging.critical("Invalid value passed for external script name: '%s'",
449 for dir_name in search_path:
450 # FIXME: investigate switch to PathJoin
451 item_name = os.path.sep.join([dir_name, name])
452 # check the user test and that we're indeed resolving to the given
454 if test(item_name) and os.path.basename(item_name) == name:
459 def IsNormAbsPath(path):
460 """Check whether a path is absolute and also normalized
462 This avoids things like /dir/../../other/path to be valid.
465 return os.path.normpath(path) == path and os.path.isabs(path)
469 """Safe-join a list of path components.
472 - the first argument must be an absolute path
473 - no component in the path must have backtracking (e.g. /../),
474 since we check for normalization at the end
476 @param args: the path components to be joined
477 @raise ValueError: for invalid paths
480 # ensure we're having at least one path passed in
482 # ensure the first component is an absolute and normalized path name
484 if not IsNormAbsPath(root):
485 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
486 result = os.path.join(*args)
487 # ensure that the whole path is normalized
488 if not IsNormAbsPath(result):
489 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
490 # check that we're still under the original prefix
491 prefix = os.path.commonprefix([root, result])
493 raise ValueError("Error: path joining resulted in different prefix"
494 " (%s != %s)" % (prefix, root))
498 def TailFile(fname, lines=20):
499 """Return the last lines from a file.
501 @note: this function will only read and parse the last 4KB of
502 the file; if the lines are very long, it could be that less
503 than the requested number of lines are returned
505 @param fname: the file name
507 @param lines: the (maximum) number of lines to return
510 fd = open(fname, "r")
514 pos = max(0, pos - 4096)
520 rows = raw_data.splitlines()
524 def BytesToMebibyte(value):
525 """Converts bytes to mebibytes.
528 @param value: Value in bytes
530 @return: Value in mebibytes
533 return int(round(value / (1024.0 * 1024.0), 0))
536 def CalculateDirectorySize(path):
537 """Calculates the size of a directory recursively.
540 @param path: Path to directory
542 @return: Size in mebibytes
547 for (curpath, _, files) in os.walk(path):
548 for filename in files:
549 st = os.lstat(PathJoin(curpath, filename))
552 return BytesToMebibyte(size)
555 def GetFilesystemStats(path):
556 """Returns the total and free space on a filesystem.
559 @param path: Path on filesystem to be examined
561 @return: tuple of (Total space, Free space) in mebibytes
564 st = os.statvfs(path)
566 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
567 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
568 return (tsize, fsize)
571 def ReadPidFile(pidfile):
572 """Read a pid from a file.
574 @type pidfile: string
575 @param pidfile: path to the file containing the pid
577 @return: The process id, if the file exists and contains a valid PID,
582 raw_data = ReadOneLineFile(pidfile)
583 except EnvironmentError, err:
584 if err.errno != errno.ENOENT:
585 logging.exception("Can't read pid file")
590 except (TypeError, ValueError), err:
591 logging.info("Can't parse pid file contents", exc_info=True)
597 def ReadLockedPidFile(path):
598 """Reads a locked PID file.
600 This can be used together with L{utils.process.StartDaemon}.
603 @param path: Path to PID file
604 @return: PID as integer or, if file was unlocked or couldn't be opened, None
608 fd = os.open(path, os.O_RDONLY)
609 except EnvironmentError, err:
610 if err.errno == errno.ENOENT:
611 # PID file doesn't exist
617 # Try to acquire lock
618 filelock.LockFile(fd)
619 except errors.LockError:
620 # Couldn't lock, daemon is running
621 return int(os.read(fd, 100))
628 def AddAuthorizedKey(file_obj, key):
629 """Adds an SSH public key to an authorized_keys file.
631 @type file_obj: str or file handle
632 @param file_obj: path to authorized_keys file
634 @param key: string containing key
637 key_fields = key.split()
639 if isinstance(file_obj, basestring):
640 f = open(file_obj, "a+")
647 # Ignore whitespace changes
648 if line.split() == key_fields:
650 nl = line.endswith("\n")
654 f.write(key.rstrip("\r\n"))
661 def RemoveAuthorizedKey(file_name, key):
662 """Removes an SSH public key from an authorized_keys file.
665 @param file_name: path to authorized_keys file
667 @param key: string containing key
670 key_fields = key.split()
672 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
674 out = os.fdopen(fd, "w")
676 f = open(file_name, "r")
679 # Ignore whitespace changes while comparing lines
680 if line.split() != key_fields:
684 os.rename(tmpname, file_name)
694 def DaemonPidFileName(name):
695 """Compute a ganeti pid file absolute path
698 @param name: the daemon name
700 @return: the full path to the pidfile corresponding to the given
704 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
707 def WritePidFile(pidfile):
708 """Write the current process pidfile.
710 @type pidfile: string
711 @param pidfile: the path to the file to be written
712 @raise errors.LockError: if the pid file already exists and
713 points to a live process
715 @return: the file descriptor of the lock file; do not close this unless
716 you want to unlock the pid file
719 # We don't rename nor truncate the file to not drop locks under
721 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
723 # Lock the PID file (and fail if not possible to do so). Any code
724 # wanting to send a signal to the daemon should try to lock the PID
725 # file before reading it. If acquiring the lock succeeds, the daemon is
726 # no longer running and the signal should not be sent.
727 filelock.LockFile(fd_pidfile)
729 os.write(fd_pidfile, "%d\n" % os.getpid())
734 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
735 """Reads the watcher pause file.
737 @type filename: string
738 @param filename: Path to watcher pause file
739 @type now: None, float or int
740 @param now: Current time as Unix timestamp
741 @type remove_after: int
742 @param remove_after: Remove watcher pause file after specified amount of
743 seconds past the pause end time
750 value = ReadFile(filename)
752 if err.errno != errno.ENOENT:
756 if value is not None:
760 logging.warning(("Watcher pause file (%s) contains invalid value,"
761 " removing it"), filename)
765 if value is not None:
766 # Remove file if it's outdated
767 if now > (value + remove_after):
778 """Returns a random UUID.
780 @note: This is a Linux-specific method as it uses the /proc
785 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")