4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for I/O.
32 from ganeti import errors
33 from ganeti import constants
34 from ganeti.utils import filelock
37 def ReadFile(file_name, size=-1):
41 @param size: Read at most size bytes (if negative, entire file)
43 @return: the (possibly partial) content of the file
46 f = open(file_name, "r")
53 def WriteFile(file_name, fn=None, data=None,
54 mode=None, uid=-1, gid=-1,
55 atime=None, mtime=None, close=True,
56 dry_run=False, backup=False,
57 prewrite=None, postwrite=None):
58 """(Over)write a file atomically.
60 The file_name and either fn (a function taking one argument, the
61 file descriptor, and which should write the data to it) or data (the
62 contents of the file) must be passed. The other arguments are
63 optional and allow setting the file mode, owner and group, and the
64 mtime/atime of the file.
66 If the function doesn't raise an exception, it has succeeded and the
67 target file has the new contents. If the function has raised an
68 exception, an existing target file should be unmodified and the
69 temporary file should be removed.
72 @param file_name: the target filename
74 @param fn: content writing function, called with
75 file descriptor as parameter
77 @param data: contents of the file
79 @param mode: file mode
81 @param uid: the owner of the file
83 @param gid: the group of the file
85 @param atime: a custom access time to be set on the file
87 @param mtime: a custom modification time to be set on the file
89 @param close: whether to close file after writing it
90 @type prewrite: callable
91 @param prewrite: function to be called before writing content
92 @type postwrite: callable
93 @param postwrite: function to be called after writing content
96 @return: None if the 'close' parameter evaluates to True,
97 otherwise the file descriptor
99 @raise errors.ProgrammerError: if any of the arguments are not valid
102 if not os.path.isabs(file_name):
103 raise errors.ProgrammerError("Path passed to WriteFile is not"
104 " absolute: '%s'" % file_name)
106 if [fn, data].count(None) != 1:
107 raise errors.ProgrammerError("fn or data required")
109 if [atime, mtime].count(None) == 1:
110 raise errors.ProgrammerError("Both atime and mtime must be either"
113 if backup and not dry_run and os.path.isfile(file_name):
114 CreateBackup(file_name)
116 dir_name, base_name = os.path.split(file_name)
117 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
119 # here we need to make sure we remove the temp file, if any error
122 if uid != -1 or gid != -1:
123 os.chown(new_name, uid, gid)
125 os.chmod(new_name, mode)
126 if callable(prewrite):
132 if callable(postwrite):
135 if atime is not None and mtime is not None:
136 os.utime(new_name, (atime, mtime))
138 os.rename(new_name, file_name)
152 def GetFileID(path=None, fd=None):
153 """Returns the file 'id', i.e. the dev/inode and mtime information.
155 Either the path to the file or the fd must be given.
157 @param path: the file path
158 @param fd: a file descriptor
159 @return: a tuple of (device number, inode number, mtime)
162 if [path, fd].count(None) != 1:
163 raise errors.ProgrammerError("One and only one of fd/path must be given")
170 return (st.st_dev, st.st_ino, st.st_mtime)
173 def VerifyFileID(fi_disk, fi_ours):
174 """Verifies that two file IDs are matching.
176 Differences in the inode/device are not accepted, but and older
177 timestamp for fi_disk is accepted.
179 @param fi_disk: tuple (dev, inode, mtime) representing the actual
181 @param fi_ours: tuple (dev, inode, mtime) representing the last
186 (d1, i1, m1) = fi_disk
187 (d2, i2, m2) = fi_ours
189 return (d1, i1) == (d2, i2) and m1 <= m2
192 def SafeWriteFile(file_name, file_id, **kwargs):
193 """Wraper over L{WriteFile} that locks the target file.
195 By keeping the target file locked during WriteFile, we ensure that
196 cooperating writers will safely serialise access to the file.
199 @param file_name: the target filename
201 @param file_id: a result from L{GetFileID}
204 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
206 filelock.LockFile(fd)
207 if file_id is not None:
208 disk_id = GetFileID(fd=fd)
209 if not VerifyFileID(disk_id, file_id):
210 raise errors.LockError("Cannot overwrite file %s, it has been modified"
211 " since last written" % file_name)
212 return WriteFile(file_name, **kwargs)
217 def ReadOneLineFile(file_name, strict=False):
218 """Return the first non-empty line from a file.
220 @type strict: boolean
221 @param strict: if True, abort if the file has more than one
225 file_lines = ReadFile(file_name).splitlines()
226 full_lines = filter(bool, file_lines)
227 if not file_lines or not full_lines:
228 raise errors.GenericError("No data in one-liner file %s" % file_name)
229 elif strict and len(full_lines) > 1:
230 raise errors.GenericError("Too many lines in one-liner file %s" %
235 def RemoveFile(filename):
236 """Remove a file ignoring some errors.
238 Remove a file, ignoring non-existing ones or directories. Other
242 @param filename: the file to be removed
248 if err.errno not in (errno.ENOENT, errno.EISDIR):
252 def RemoveDir(dirname):
253 """Remove an empty directory.
255 Remove a directory, ignoring non-existing ones.
256 Other errors are passed. This includes the case,
257 where the directory is not empty, so it can't be removed.
260 @param dirname: the empty directory to be removed
266 if err.errno != errno.ENOENT:
270 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
274 @param old: Original path
278 @param mkdir: Whether to create target directory if it doesn't exist
279 @type mkdir_mode: int
280 @param mkdir_mode: Mode for newly created directories
284 return os.rename(old, new)
286 # In at least one use case of this function, the job queue, directory
287 # creation is very rare. Checking for the directory before renaming is not
289 if mkdir and err.errno == errno.ENOENT:
290 # Create directory and try again
291 Makedirs(os.path.dirname(new), mode=mkdir_mode)
293 return os.rename(old, new)
298 def Makedirs(path, mode=0750):
299 """Super-mkdir; create a leaf directory and all intermediate ones.
301 This is a wrapper around C{os.makedirs} adding error handling not implemented
306 os.makedirs(path, mode)
308 # Ignore EEXIST. This is only handled in os.makedirs as included in
309 # Python 2.5 and above.
310 if err.errno != errno.EEXIST or not os.path.exists(path):
314 def TimestampForFilename():
315 """Returns the current time formatted for filenames.
317 The format doesn't contain colons as some shells and applications treat them
318 as separators. Uses the local timezone.
321 return time.strftime("%Y-%m-%d_%H_%M_%S")
324 def CreateBackup(file_name):
325 """Creates a backup of a file.
328 @param file_name: file to be backed up
330 @return: the path to the newly created backup
331 @raise errors.ProgrammerError: for invalid file names
334 if not os.path.isfile(file_name):
335 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
338 prefix = ("%s.backup-%s." %
339 (os.path.basename(file_name), TimestampForFilename()))
340 dir_name = os.path.dirname(file_name)
342 fsrc = open(file_name, 'rb')
344 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
345 fdst = os.fdopen(fd, 'wb')
347 logging.debug("Backing up %s at %s", file_name, backup_name)
348 shutil.copyfileobj(fsrc, fdst)
357 def ListVisibleFiles(path):
358 """Returns a list of visible files in a directory.
361 @param path: the directory to enumerate
363 @return: the list of all files not starting with a dot
364 @raise ProgrammerError: if L{path} is not an absolue and normalized path
367 if not IsNormAbsPath(path):
368 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
369 " absolute/normalized: '%s'" % path)
370 files = [i for i in os.listdir(path) if not i.startswith(".")]
374 def EnsureDirs(dirs):
375 """Make required directories, if they don't exist.
377 @param dirs: list of tuples (dir_name, dir_mode)
378 @type dirs: list of (string, integer)
381 for dir_name, dir_mode in dirs:
383 os.mkdir(dir_name, dir_mode)
384 except EnvironmentError, err:
385 if err.errno != errno.EEXIST:
386 raise errors.GenericError("Cannot create needed directory"
387 " '%s': %s" % (dir_name, err))
389 os.chmod(dir_name, dir_mode)
390 except EnvironmentError, err:
391 raise errors.GenericError("Cannot change directory permissions on"
392 " '%s': %s" % (dir_name, err))
393 if not os.path.isdir(dir_name):
394 raise errors.GenericError("%s is not a directory" % dir_name)
397 def FindFile(name, search_path, test=os.path.exists):
398 """Look for a filesystem object in a given path.
400 This is an abstract method to search for filesystem object (files,
401 dirs) under a given search path.
404 @param name: the name to look for
405 @type search_path: str
406 @param search_path: location to start at
408 @param test: a function taking one argument that should return True
409 if the a given object is valid; the default value is
410 os.path.exists, causing only existing files to be returned
412 @return: full path to the object if found, None otherwise
415 # validate the filename mask
416 if constants.EXT_PLUGIN_MASK.match(name) is None:
417 logging.critical("Invalid value passed for external script name: '%s'",
421 for dir_name in search_path:
422 # FIXME: investigate switch to PathJoin
423 item_name = os.path.sep.join([dir_name, name])
424 # check the user test and that we're indeed resolving to the given
426 if test(item_name) and os.path.basename(item_name) == name:
431 def IsNormAbsPath(path):
432 """Check whether a path is absolute and also normalized
434 This avoids things like /dir/../../other/path to be valid.
437 return os.path.normpath(path) == path and os.path.isabs(path)
441 """Safe-join a list of path components.
444 - the first argument must be an absolute path
445 - no component in the path must have backtracking (e.g. /../),
446 since we check for normalization at the end
448 @param args: the path components to be joined
449 @raise ValueError: for invalid paths
452 # ensure we're having at least one path passed in
454 # ensure the first component is an absolute and normalized path name
456 if not IsNormAbsPath(root):
457 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
458 result = os.path.join(*args)
459 # ensure that the whole path is normalized
460 if not IsNormAbsPath(result):
461 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
462 # check that we're still under the original prefix
463 prefix = os.path.commonprefix([root, result])
465 raise ValueError("Error: path joining resulted in different prefix"
466 " (%s != %s)" % (prefix, root))
470 def TailFile(fname, lines=20):
471 """Return the last lines from a file.
473 @note: this function will only read and parse the last 4KB of
474 the file; if the lines are very long, it could be that less
475 than the requested number of lines are returned
477 @param fname: the file name
479 @param lines: the (maximum) number of lines to return
482 fd = open(fname, "r")
486 pos = max(0, pos-4096)
492 rows = raw_data.splitlines()
496 def BytesToMebibyte(value):
497 """Converts bytes to mebibytes.
500 @param value: Value in bytes
502 @return: Value in mebibytes
505 return int(round(value / (1024.0 * 1024.0), 0))
508 def CalculateDirectorySize(path):
509 """Calculates the size of a directory recursively.
512 @param path: Path to directory
514 @return: Size in mebibytes
519 for (curpath, _, files) in os.walk(path):
520 for filename in files:
521 st = os.lstat(PathJoin(curpath, filename))
524 return BytesToMebibyte(size)
527 def GetFilesystemStats(path):
528 """Returns the total and free space on a filesystem.
531 @param path: Path on filesystem to be examined
533 @return: tuple of (Total space, Free space) in mebibytes
536 st = os.statvfs(path)
538 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
539 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
540 return (tsize, fsize)
543 def ReadPidFile(pidfile):
544 """Read a pid from a file.
546 @type pidfile: string
547 @param pidfile: path to the file containing the pid
549 @return: The process id, if the file exists and contains a valid PID,
554 raw_data = ReadOneLineFile(pidfile)
555 except EnvironmentError, err:
556 if err.errno != errno.ENOENT:
557 logging.exception("Can't read pid file")
562 except (TypeError, ValueError), err:
563 logging.info("Can't parse pid file contents", exc_info=True)
569 def ReadLockedPidFile(path):
570 """Reads a locked PID file.
572 This can be used together with L{utils.process.StartDaemon}.
575 @param path: Path to PID file
576 @return: PID as integer or, if file was unlocked or couldn't be opened, None
580 fd = os.open(path, os.O_RDONLY)
581 except EnvironmentError, err:
582 if err.errno == errno.ENOENT:
583 # PID file doesn't exist
589 # Try to acquire lock
590 filelock.LockFile(fd)
591 except errors.LockError:
592 # Couldn't lock, daemon is running
593 return int(os.read(fd, 100))
600 def AddAuthorizedKey(file_obj, key):
601 """Adds an SSH public key to an authorized_keys file.
603 @type file_obj: str or file handle
604 @param file_obj: path to authorized_keys file
606 @param key: string containing key
609 key_fields = key.split()
611 if isinstance(file_obj, basestring):
612 f = open(file_obj, 'a+')
619 # Ignore whitespace changes
620 if line.split() == key_fields:
622 nl = line.endswith('\n')
626 f.write(key.rstrip('\r\n'))
633 def RemoveAuthorizedKey(file_name, key):
634 """Removes an SSH public key from an authorized_keys file.
637 @param file_name: path to authorized_keys file
639 @param key: string containing key
642 key_fields = key.split()
644 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
646 out = os.fdopen(fd, 'w')
648 f = open(file_name, 'r')
651 # Ignore whitespace changes while comparing lines
652 if line.split() != key_fields:
656 os.rename(tmpname, file_name)
666 def DaemonPidFileName(name):
667 """Compute a ganeti pid file absolute path
670 @param name: the daemon name
672 @return: the full path to the pidfile corresponding to the given
676 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
679 def WritePidFile(pidfile):
680 """Write the current process pidfile.
682 @type pidfile: string
683 @param pidfile: the path to the file to be written
684 @raise errors.LockError: if the pid file already exists and
685 points to a live process
687 @return: the file descriptor of the lock file; do not close this unless
688 you want to unlock the pid file
691 # We don't rename nor truncate the file to not drop locks under
693 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
695 # Lock the PID file (and fail if not possible to do so). Any code
696 # wanting to send a signal to the daemon should try to lock the PID
697 # file before reading it. If acquiring the lock succeeds, the daemon is
698 # no longer running and the signal should not be sent.
699 filelock.LockFile(fd_pidfile)
701 os.write(fd_pidfile, "%d\n" % os.getpid())
706 def RemovePidFile(pidfile):
707 """Remove the current process pidfile.
709 Any errors are ignored.
711 @type pidfile: string
712 @param pidfile: Path to the file to be removed
715 # TODO: we could check here that the file contains our pid
718 except Exception: # pylint: disable-msg=W0703
722 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
723 """Reads the watcher pause file.
725 @type filename: string
726 @param filename: Path to watcher pause file
727 @type now: None, float or int
728 @param now: Current time as Unix timestamp
729 @type remove_after: int
730 @param remove_after: Remove watcher pause file after specified amount of
731 seconds past the pause end time
738 value = ReadFile(filename)
740 if err.errno != errno.ENOENT:
744 if value is not None:
748 logging.warning(("Watcher pause file (%s) contains invalid value,"
749 " removing it"), filename)
753 if value is not None:
754 # Remove file if it's outdated
755 if now > (value + remove_after):