4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for I/O.
32 from ganeti import errors
33 from ganeti import constants
34 from ganeti.utils import filelock
37 #: Path generating random UUID
38 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
41 def ReadFile(file_name, size=-1):
45 @param size: Read at most size bytes (if negative, entire file)
47 @return: the (possibly partial) content of the file
50 f = open(file_name, "r")
57 def WriteFile(file_name, fn=None, data=None,
58 mode=None, uid=-1, gid=-1,
59 atime=None, mtime=None, close=True,
60 dry_run=False, backup=False,
61 prewrite=None, postwrite=None):
62 """(Over)write a file atomically.
64 The file_name and either fn (a function taking one argument, the
65 file descriptor, and which should write the data to it) or data (the
66 contents of the file) must be passed. The other arguments are
67 optional and allow setting the file mode, owner and group, and the
68 mtime/atime of the file.
70 If the function doesn't raise an exception, it has succeeded and the
71 target file has the new contents. If the function has raised an
72 exception, an existing target file should be unmodified and the
73 temporary file should be removed.
76 @param file_name: the target filename
78 @param fn: content writing function, called with
79 file descriptor as parameter
81 @param data: contents of the file
83 @param mode: file mode
85 @param uid: the owner of the file
87 @param gid: the group of the file
89 @param atime: a custom access time to be set on the file
91 @param mtime: a custom modification time to be set on the file
93 @param close: whether to close file after writing it
94 @type prewrite: callable
95 @param prewrite: function to be called before writing content
96 @type postwrite: callable
97 @param postwrite: function to be called after writing content
100 @return: None if the 'close' parameter evaluates to True,
101 otherwise the file descriptor
103 @raise errors.ProgrammerError: if any of the arguments are not valid
106 if not os.path.isabs(file_name):
107 raise errors.ProgrammerError("Path passed to WriteFile is not"
108 " absolute: '%s'" % file_name)
110 if [fn, data].count(None) != 1:
111 raise errors.ProgrammerError("fn or data required")
113 if [atime, mtime].count(None) == 1:
114 raise errors.ProgrammerError("Both atime and mtime must be either"
117 if backup and not dry_run and os.path.isfile(file_name):
118 CreateBackup(file_name)
120 # Whether temporary file needs to be removed (e.g. if any error occurs)
126 (dir_name, base_name) = os.path.split(file_name)
127 (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
131 if uid != -1 or gid != -1:
132 os.chown(new_name, uid, gid)
134 os.chmod(new_name, mode)
135 if callable(prewrite):
138 if isinstance(data, unicode):
140 assert isinstance(data, str)
143 while offset < to_write:
144 written = os.write(fd, buffer(data, offset))
146 assert written <= to_write - offset
148 assert offset == to_write
151 if callable(postwrite):
154 if atime is not None and mtime is not None:
155 os.utime(new_name, (atime, mtime))
157 # Close file unless the file descriptor should be returned
163 # Rename file to destination name
165 os.rename(new_name, file_name)
166 # Successful, no need to remove anymore
175 def GetFileID(path=None, fd=None):
176 """Returns the file 'id', i.e. the dev/inode and mtime information.
178 Either the path to the file or the fd must be given.
180 @param path: the file path
181 @param fd: a file descriptor
182 @return: a tuple of (device number, inode number, mtime)
185 if [path, fd].count(None) != 1:
186 raise errors.ProgrammerError("One and only one of fd/path must be given")
193 return (st.st_dev, st.st_ino, st.st_mtime)
196 def VerifyFileID(fi_disk, fi_ours):
197 """Verifies that two file IDs are matching.
199 Differences in the inode/device are not accepted, but and older
200 timestamp for fi_disk is accepted.
202 @param fi_disk: tuple (dev, inode, mtime) representing the actual
204 @param fi_ours: tuple (dev, inode, mtime) representing the last
209 (d1, i1, m1) = fi_disk
210 (d2, i2, m2) = fi_ours
212 return (d1, i1) == (d2, i2) and m1 <= m2
215 def SafeWriteFile(file_name, file_id, **kwargs):
216 """Wraper over L{WriteFile} that locks the target file.
218 By keeping the target file locked during WriteFile, we ensure that
219 cooperating writers will safely serialise access to the file.
222 @param file_name: the target filename
224 @param file_id: a result from L{GetFileID}
227 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
229 filelock.LockFile(fd)
230 if file_id is not None:
231 disk_id = GetFileID(fd=fd)
232 if not VerifyFileID(disk_id, file_id):
233 raise errors.LockError("Cannot overwrite file %s, it has been modified"
234 " since last written" % file_name)
235 return WriteFile(file_name, **kwargs)
240 def ReadOneLineFile(file_name, strict=False):
241 """Return the first non-empty line from a file.
243 @type strict: boolean
244 @param strict: if True, abort if the file has more than one
248 file_lines = ReadFile(file_name).splitlines()
249 full_lines = filter(bool, file_lines)
250 if not file_lines or not full_lines:
251 raise errors.GenericError("No data in one-liner file %s" % file_name)
252 elif strict and len(full_lines) > 1:
253 raise errors.GenericError("Too many lines in one-liner file %s" %
258 def RemoveFile(filename):
259 """Remove a file ignoring some errors.
261 Remove a file, ignoring non-existing ones or directories. Other
265 @param filename: the file to be removed
271 if err.errno not in (errno.ENOENT, errno.EISDIR):
275 def RemoveDir(dirname):
276 """Remove an empty directory.
278 Remove a directory, ignoring non-existing ones.
279 Other errors are passed. This includes the case,
280 where the directory is not empty, so it can't be removed.
283 @param dirname: the empty directory to be removed
289 if err.errno != errno.ENOENT:
293 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
297 @param old: Original path
301 @param mkdir: Whether to create target directory if it doesn't exist
302 @type mkdir_mode: int
303 @param mkdir_mode: Mode for newly created directories
307 return os.rename(old, new)
309 # In at least one use case of this function, the job queue, directory
310 # creation is very rare. Checking for the directory before renaming is not
312 if mkdir and err.errno == errno.ENOENT:
313 # Create directory and try again
314 Makedirs(os.path.dirname(new), mode=mkdir_mode)
316 return os.rename(old, new)
321 def Makedirs(path, mode=0750):
322 """Super-mkdir; create a leaf directory and all intermediate ones.
324 This is a wrapper around C{os.makedirs} adding error handling not implemented
329 os.makedirs(path, mode)
331 # Ignore EEXIST. This is only handled in os.makedirs as included in
332 # Python 2.5 and above.
333 if err.errno != errno.EEXIST or not os.path.exists(path):
337 def TimestampForFilename():
338 """Returns the current time formatted for filenames.
340 The format doesn't contain colons as some shells and applications treat them
341 as separators. Uses the local timezone.
344 return time.strftime("%Y-%m-%d_%H_%M_%S")
347 def CreateBackup(file_name):
348 """Creates a backup of a file.
351 @param file_name: file to be backed up
353 @return: the path to the newly created backup
354 @raise errors.ProgrammerError: for invalid file names
357 if not os.path.isfile(file_name):
358 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
361 prefix = ("%s.backup-%s." %
362 (os.path.basename(file_name), TimestampForFilename()))
363 dir_name = os.path.dirname(file_name)
365 fsrc = open(file_name, 'rb')
367 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
368 fdst = os.fdopen(fd, 'wb')
370 logging.debug("Backing up %s at %s", file_name, backup_name)
371 shutil.copyfileobj(fsrc, fdst)
380 def ListVisibleFiles(path):
381 """Returns a list of visible files in a directory.
384 @param path: the directory to enumerate
386 @return: the list of all files not starting with a dot
387 @raise ProgrammerError: if L{path} is not an absolue and normalized path
390 if not IsNormAbsPath(path):
391 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
392 " absolute/normalized: '%s'" % path)
393 files = [i for i in os.listdir(path) if not i.startswith(".")]
397 def EnsureDirs(dirs):
398 """Make required directories, if they don't exist.
400 @param dirs: list of tuples (dir_name, dir_mode)
401 @type dirs: list of (string, integer)
404 for dir_name, dir_mode in dirs:
406 os.mkdir(dir_name, dir_mode)
407 except EnvironmentError, err:
408 if err.errno != errno.EEXIST:
409 raise errors.GenericError("Cannot create needed directory"
410 " '%s': %s" % (dir_name, err))
412 os.chmod(dir_name, dir_mode)
413 except EnvironmentError, err:
414 raise errors.GenericError("Cannot change directory permissions on"
415 " '%s': %s" % (dir_name, err))
416 if not os.path.isdir(dir_name):
417 raise errors.GenericError("%s is not a directory" % dir_name)
420 def FindFile(name, search_path, test=os.path.exists):
421 """Look for a filesystem object in a given path.
423 This is an abstract method to search for filesystem object (files,
424 dirs) under a given search path.
427 @param name: the name to look for
428 @type search_path: str
429 @param search_path: location to start at
431 @param test: a function taking one argument that should return True
432 if the a given object is valid; the default value is
433 os.path.exists, causing only existing files to be returned
435 @return: full path to the object if found, None otherwise
438 # validate the filename mask
439 if constants.EXT_PLUGIN_MASK.match(name) is None:
440 logging.critical("Invalid value passed for external script name: '%s'",
444 for dir_name in search_path:
445 # FIXME: investigate switch to PathJoin
446 item_name = os.path.sep.join([dir_name, name])
447 # check the user test and that we're indeed resolving to the given
449 if test(item_name) and os.path.basename(item_name) == name:
454 def IsNormAbsPath(path):
455 """Check whether a path is absolute and also normalized
457 This avoids things like /dir/../../other/path to be valid.
460 return os.path.normpath(path) == path and os.path.isabs(path)
464 """Safe-join a list of path components.
467 - the first argument must be an absolute path
468 - no component in the path must have backtracking (e.g. /../),
469 since we check for normalization at the end
471 @param args: the path components to be joined
472 @raise ValueError: for invalid paths
475 # ensure we're having at least one path passed in
477 # ensure the first component is an absolute and normalized path name
479 if not IsNormAbsPath(root):
480 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
481 result = os.path.join(*args)
482 # ensure that the whole path is normalized
483 if not IsNormAbsPath(result):
484 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
485 # check that we're still under the original prefix
486 prefix = os.path.commonprefix([root, result])
488 raise ValueError("Error: path joining resulted in different prefix"
489 " (%s != %s)" % (prefix, root))
493 def TailFile(fname, lines=20):
494 """Return the last lines from a file.
496 @note: this function will only read and parse the last 4KB of
497 the file; if the lines are very long, it could be that less
498 than the requested number of lines are returned
500 @param fname: the file name
502 @param lines: the (maximum) number of lines to return
505 fd = open(fname, "r")
509 pos = max(0, pos-4096)
515 rows = raw_data.splitlines()
519 def BytesToMebibyte(value):
520 """Converts bytes to mebibytes.
523 @param value: Value in bytes
525 @return: Value in mebibytes
528 return int(round(value / (1024.0 * 1024.0), 0))
531 def CalculateDirectorySize(path):
532 """Calculates the size of a directory recursively.
535 @param path: Path to directory
537 @return: Size in mebibytes
542 for (curpath, _, files) in os.walk(path):
543 for filename in files:
544 st = os.lstat(PathJoin(curpath, filename))
547 return BytesToMebibyte(size)
550 def GetFilesystemStats(path):
551 """Returns the total and free space on a filesystem.
554 @param path: Path on filesystem to be examined
556 @return: tuple of (Total space, Free space) in mebibytes
559 st = os.statvfs(path)
561 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
562 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
563 return (tsize, fsize)
566 def ReadPidFile(pidfile):
567 """Read a pid from a file.
569 @type pidfile: string
570 @param pidfile: path to the file containing the pid
572 @return: The process id, if the file exists and contains a valid PID,
577 raw_data = ReadOneLineFile(pidfile)
578 except EnvironmentError, err:
579 if err.errno != errno.ENOENT:
580 logging.exception("Can't read pid file")
585 except (TypeError, ValueError), err:
586 logging.info("Can't parse pid file contents", exc_info=True)
592 def ReadLockedPidFile(path):
593 """Reads a locked PID file.
595 This can be used together with L{utils.process.StartDaemon}.
598 @param path: Path to PID file
599 @return: PID as integer or, if file was unlocked or couldn't be opened, None
603 fd = os.open(path, os.O_RDONLY)
604 except EnvironmentError, err:
605 if err.errno == errno.ENOENT:
606 # PID file doesn't exist
612 # Try to acquire lock
613 filelock.LockFile(fd)
614 except errors.LockError:
615 # Couldn't lock, daemon is running
616 return int(os.read(fd, 100))
623 def AddAuthorizedKey(file_obj, key):
624 """Adds an SSH public key to an authorized_keys file.
626 @type file_obj: str or file handle
627 @param file_obj: path to authorized_keys file
629 @param key: string containing key
632 key_fields = key.split()
634 if isinstance(file_obj, basestring):
635 f = open(file_obj, 'a+')
642 # Ignore whitespace changes
643 if line.split() == key_fields:
645 nl = line.endswith('\n')
649 f.write(key.rstrip('\r\n'))
656 def RemoveAuthorizedKey(file_name, key):
657 """Removes an SSH public key from an authorized_keys file.
660 @param file_name: path to authorized_keys file
662 @param key: string containing key
665 key_fields = key.split()
667 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
669 out = os.fdopen(fd, 'w')
671 f = open(file_name, 'r')
674 # Ignore whitespace changes while comparing lines
675 if line.split() != key_fields:
679 os.rename(tmpname, file_name)
689 def DaemonPidFileName(name):
690 """Compute a ganeti pid file absolute path
693 @param name: the daemon name
695 @return: the full path to the pidfile corresponding to the given
699 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
702 def WritePidFile(pidfile):
703 """Write the current process pidfile.
705 @type pidfile: string
706 @param pidfile: the path to the file to be written
707 @raise errors.LockError: if the pid file already exists and
708 points to a live process
710 @return: the file descriptor of the lock file; do not close this unless
711 you want to unlock the pid file
714 # We don't rename nor truncate the file to not drop locks under
716 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
718 # Lock the PID file (and fail if not possible to do so). Any code
719 # wanting to send a signal to the daemon should try to lock the PID
720 # file before reading it. If acquiring the lock succeeds, the daemon is
721 # no longer running and the signal should not be sent.
722 filelock.LockFile(fd_pidfile)
724 os.write(fd_pidfile, "%d\n" % os.getpid())
729 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
730 """Reads the watcher pause file.
732 @type filename: string
733 @param filename: Path to watcher pause file
734 @type now: None, float or int
735 @param now: Current time as Unix timestamp
736 @type remove_after: int
737 @param remove_after: Remove watcher pause file after specified amount of
738 seconds past the pause end time
745 value = ReadFile(filename)
747 if err.errno != errno.ENOENT:
751 if value is not None:
755 logging.warning(("Watcher pause file (%s) contains invalid value,"
756 " removing it"), filename)
760 if value is not None:
761 # Remove file if it's outdated
762 if now > (value + remove_after):
773 """Returns a random UUID.
775 @note: This is a Linux-specific method as it uses the /proc
780 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")