4 # Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 """Utility functions for I/O.
32 from ganeti import errors
33 from ganeti import constants
34 from ganeti.utils import filelock
37 #: Path generating random UUID
38 _RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
41 def ReadFile(file_name, size=-1):
45 @param size: Read at most size bytes (if negative, entire file)
47 @return: the (possibly partial) content of the file
50 f = open(file_name, "r")
57 def WriteFile(file_name, fn=None, data=None,
58 mode=None, uid=-1, gid=-1,
59 atime=None, mtime=None, close=True,
60 dry_run=False, backup=False,
61 prewrite=None, postwrite=None):
62 """(Over)write a file atomically.
64 The file_name and either fn (a function taking one argument, the
65 file descriptor, and which should write the data to it) or data (the
66 contents of the file) must be passed. The other arguments are
67 optional and allow setting the file mode, owner and group, and the
68 mtime/atime of the file.
70 If the function doesn't raise an exception, it has succeeded and the
71 target file has the new contents. If the function has raised an
72 exception, an existing target file should be unmodified and the
73 temporary file should be removed.
76 @param file_name: the target filename
78 @param fn: content writing function, called with
79 file descriptor as parameter
81 @param data: contents of the file
83 @param mode: file mode
85 @param uid: the owner of the file
87 @param gid: the group of the file
89 @param atime: a custom access time to be set on the file
91 @param mtime: a custom modification time to be set on the file
93 @param close: whether to close file after writing it
94 @type prewrite: callable
95 @param prewrite: function to be called before writing content
96 @type postwrite: callable
97 @param postwrite: function to be called after writing content
100 @return: None if the 'close' parameter evaluates to True,
101 otherwise the file descriptor
103 @raise errors.ProgrammerError: if any of the arguments are not valid
106 if not os.path.isabs(file_name):
107 raise errors.ProgrammerError("Path passed to WriteFile is not"
108 " absolute: '%s'" % file_name)
110 if [fn, data].count(None) != 1:
111 raise errors.ProgrammerError("fn or data required")
113 if [atime, mtime].count(None) == 1:
114 raise errors.ProgrammerError("Both atime and mtime must be either"
117 if backup and not dry_run and os.path.isfile(file_name):
118 CreateBackup(file_name)
120 dir_name, base_name = os.path.split(file_name)
121 fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
123 # here we need to make sure we remove the temp file, if any error
126 if uid != -1 or gid != -1:
127 os.chown(new_name, uid, gid)
129 os.chmod(new_name, mode)
130 if callable(prewrite):
136 if callable(postwrite):
139 if atime is not None and mtime is not None:
140 os.utime(new_name, (atime, mtime))
142 os.rename(new_name, file_name)
156 def GetFileID(path=None, fd=None):
157 """Returns the file 'id', i.e. the dev/inode and mtime information.
159 Either the path to the file or the fd must be given.
161 @param path: the file path
162 @param fd: a file descriptor
163 @return: a tuple of (device number, inode number, mtime)
166 if [path, fd].count(None) != 1:
167 raise errors.ProgrammerError("One and only one of fd/path must be given")
174 return (st.st_dev, st.st_ino, st.st_mtime)
177 def VerifyFileID(fi_disk, fi_ours):
178 """Verifies that two file IDs are matching.
180 Differences in the inode/device are not accepted, but and older
181 timestamp for fi_disk is accepted.
183 @param fi_disk: tuple (dev, inode, mtime) representing the actual
185 @param fi_ours: tuple (dev, inode, mtime) representing the last
190 (d1, i1, m1) = fi_disk
191 (d2, i2, m2) = fi_ours
193 return (d1, i1) == (d2, i2) and m1 <= m2
196 def SafeWriteFile(file_name, file_id, **kwargs):
197 """Wraper over L{WriteFile} that locks the target file.
199 By keeping the target file locked during WriteFile, we ensure that
200 cooperating writers will safely serialise access to the file.
203 @param file_name: the target filename
205 @param file_id: a result from L{GetFileID}
208 fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
210 filelock.LockFile(fd)
211 if file_id is not None:
212 disk_id = GetFileID(fd=fd)
213 if not VerifyFileID(disk_id, file_id):
214 raise errors.LockError("Cannot overwrite file %s, it has been modified"
215 " since last written" % file_name)
216 return WriteFile(file_name, **kwargs)
221 def ReadOneLineFile(file_name, strict=False):
222 """Return the first non-empty line from a file.
224 @type strict: boolean
225 @param strict: if True, abort if the file has more than one
229 file_lines = ReadFile(file_name).splitlines()
230 full_lines = filter(bool, file_lines)
231 if not file_lines or not full_lines:
232 raise errors.GenericError("No data in one-liner file %s" % file_name)
233 elif strict and len(full_lines) > 1:
234 raise errors.GenericError("Too many lines in one-liner file %s" %
239 def RemoveFile(filename):
240 """Remove a file ignoring some errors.
242 Remove a file, ignoring non-existing ones or directories. Other
246 @param filename: the file to be removed
252 if err.errno not in (errno.ENOENT, errno.EISDIR):
256 def RemoveDir(dirname):
257 """Remove an empty directory.
259 Remove a directory, ignoring non-existing ones.
260 Other errors are passed. This includes the case,
261 where the directory is not empty, so it can't be removed.
264 @param dirname: the empty directory to be removed
270 if err.errno != errno.ENOENT:
274 def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
278 @param old: Original path
282 @param mkdir: Whether to create target directory if it doesn't exist
283 @type mkdir_mode: int
284 @param mkdir_mode: Mode for newly created directories
288 return os.rename(old, new)
290 # In at least one use case of this function, the job queue, directory
291 # creation is very rare. Checking for the directory before renaming is not
293 if mkdir and err.errno == errno.ENOENT:
294 # Create directory and try again
295 Makedirs(os.path.dirname(new), mode=mkdir_mode)
297 return os.rename(old, new)
302 def Makedirs(path, mode=0750):
303 """Super-mkdir; create a leaf directory and all intermediate ones.
305 This is a wrapper around C{os.makedirs} adding error handling not implemented
310 os.makedirs(path, mode)
312 # Ignore EEXIST. This is only handled in os.makedirs as included in
313 # Python 2.5 and above.
314 if err.errno != errno.EEXIST or not os.path.exists(path):
318 def TimestampForFilename():
319 """Returns the current time formatted for filenames.
321 The format doesn't contain colons as some shells and applications treat them
322 as separators. Uses the local timezone.
325 return time.strftime("%Y-%m-%d_%H_%M_%S")
328 def CreateBackup(file_name):
329 """Creates a backup of a file.
332 @param file_name: file to be backed up
334 @return: the path to the newly created backup
335 @raise errors.ProgrammerError: for invalid file names
338 if not os.path.isfile(file_name):
339 raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
342 prefix = ("%s.backup-%s." %
343 (os.path.basename(file_name), TimestampForFilename()))
344 dir_name = os.path.dirname(file_name)
346 fsrc = open(file_name, 'rb')
348 (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
349 fdst = os.fdopen(fd, 'wb')
351 logging.debug("Backing up %s at %s", file_name, backup_name)
352 shutil.copyfileobj(fsrc, fdst)
361 def ListVisibleFiles(path):
362 """Returns a list of visible files in a directory.
365 @param path: the directory to enumerate
367 @return: the list of all files not starting with a dot
368 @raise ProgrammerError: if L{path} is not an absolue and normalized path
371 if not IsNormAbsPath(path):
372 raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
373 " absolute/normalized: '%s'" % path)
374 files = [i for i in os.listdir(path) if not i.startswith(".")]
378 def EnsureDirs(dirs):
379 """Make required directories, if they don't exist.
381 @param dirs: list of tuples (dir_name, dir_mode)
382 @type dirs: list of (string, integer)
385 for dir_name, dir_mode in dirs:
387 os.mkdir(dir_name, dir_mode)
388 except EnvironmentError, err:
389 if err.errno != errno.EEXIST:
390 raise errors.GenericError("Cannot create needed directory"
391 " '%s': %s" % (dir_name, err))
393 os.chmod(dir_name, dir_mode)
394 except EnvironmentError, err:
395 raise errors.GenericError("Cannot change directory permissions on"
396 " '%s': %s" % (dir_name, err))
397 if not os.path.isdir(dir_name):
398 raise errors.GenericError("%s is not a directory" % dir_name)
401 def FindFile(name, search_path, test=os.path.exists):
402 """Look for a filesystem object in a given path.
404 This is an abstract method to search for filesystem object (files,
405 dirs) under a given search path.
408 @param name: the name to look for
409 @type search_path: str
410 @param search_path: location to start at
412 @param test: a function taking one argument that should return True
413 if the a given object is valid; the default value is
414 os.path.exists, causing only existing files to be returned
416 @return: full path to the object if found, None otherwise
419 # validate the filename mask
420 if constants.EXT_PLUGIN_MASK.match(name) is None:
421 logging.critical("Invalid value passed for external script name: '%s'",
425 for dir_name in search_path:
426 # FIXME: investigate switch to PathJoin
427 item_name = os.path.sep.join([dir_name, name])
428 # check the user test and that we're indeed resolving to the given
430 if test(item_name) and os.path.basename(item_name) == name:
435 def IsNormAbsPath(path):
436 """Check whether a path is absolute and also normalized
438 This avoids things like /dir/../../other/path to be valid.
441 return os.path.normpath(path) == path and os.path.isabs(path)
445 """Safe-join a list of path components.
448 - the first argument must be an absolute path
449 - no component in the path must have backtracking (e.g. /../),
450 since we check for normalization at the end
452 @param args: the path components to be joined
453 @raise ValueError: for invalid paths
456 # ensure we're having at least one path passed in
458 # ensure the first component is an absolute and normalized path name
460 if not IsNormAbsPath(root):
461 raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
462 result = os.path.join(*args)
463 # ensure that the whole path is normalized
464 if not IsNormAbsPath(result):
465 raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
466 # check that we're still under the original prefix
467 prefix = os.path.commonprefix([root, result])
469 raise ValueError("Error: path joining resulted in different prefix"
470 " (%s != %s)" % (prefix, root))
474 def TailFile(fname, lines=20):
475 """Return the last lines from a file.
477 @note: this function will only read and parse the last 4KB of
478 the file; if the lines are very long, it could be that less
479 than the requested number of lines are returned
481 @param fname: the file name
483 @param lines: the (maximum) number of lines to return
486 fd = open(fname, "r")
490 pos = max(0, pos-4096)
496 rows = raw_data.splitlines()
500 def BytesToMebibyte(value):
501 """Converts bytes to mebibytes.
504 @param value: Value in bytes
506 @return: Value in mebibytes
509 return int(round(value / (1024.0 * 1024.0), 0))
512 def CalculateDirectorySize(path):
513 """Calculates the size of a directory recursively.
516 @param path: Path to directory
518 @return: Size in mebibytes
523 for (curpath, _, files) in os.walk(path):
524 for filename in files:
525 st = os.lstat(PathJoin(curpath, filename))
528 return BytesToMebibyte(size)
531 def GetFilesystemStats(path):
532 """Returns the total and free space on a filesystem.
535 @param path: Path on filesystem to be examined
537 @return: tuple of (Total space, Free space) in mebibytes
540 st = os.statvfs(path)
542 fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
543 tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
544 return (tsize, fsize)
547 def ReadPidFile(pidfile):
548 """Read a pid from a file.
550 @type pidfile: string
551 @param pidfile: path to the file containing the pid
553 @return: The process id, if the file exists and contains a valid PID,
558 raw_data = ReadOneLineFile(pidfile)
559 except EnvironmentError, err:
560 if err.errno != errno.ENOENT:
561 logging.exception("Can't read pid file")
566 except (TypeError, ValueError), err:
567 logging.info("Can't parse pid file contents", exc_info=True)
573 def ReadLockedPidFile(path):
574 """Reads a locked PID file.
576 This can be used together with L{utils.process.StartDaemon}.
579 @param path: Path to PID file
580 @return: PID as integer or, if file was unlocked or couldn't be opened, None
584 fd = os.open(path, os.O_RDONLY)
585 except EnvironmentError, err:
586 if err.errno == errno.ENOENT:
587 # PID file doesn't exist
593 # Try to acquire lock
594 filelock.LockFile(fd)
595 except errors.LockError:
596 # Couldn't lock, daemon is running
597 return int(os.read(fd, 100))
604 def AddAuthorizedKey(file_obj, key):
605 """Adds an SSH public key to an authorized_keys file.
607 @type file_obj: str or file handle
608 @param file_obj: path to authorized_keys file
610 @param key: string containing key
613 key_fields = key.split()
615 if isinstance(file_obj, basestring):
616 f = open(file_obj, 'a+')
623 # Ignore whitespace changes
624 if line.split() == key_fields:
626 nl = line.endswith('\n')
630 f.write(key.rstrip('\r\n'))
637 def RemoveAuthorizedKey(file_name, key):
638 """Removes an SSH public key from an authorized_keys file.
641 @param file_name: path to authorized_keys file
643 @param key: string containing key
646 key_fields = key.split()
648 fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
650 out = os.fdopen(fd, 'w')
652 f = open(file_name, 'r')
655 # Ignore whitespace changes while comparing lines
656 if line.split() != key_fields:
660 os.rename(tmpname, file_name)
670 def DaemonPidFileName(name):
671 """Compute a ganeti pid file absolute path
674 @param name: the daemon name
676 @return: the full path to the pidfile corresponding to the given
680 return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
683 def WritePidFile(pidfile):
684 """Write the current process pidfile.
686 @type pidfile: string
687 @param pidfile: the path to the file to be written
688 @raise errors.LockError: if the pid file already exists and
689 points to a live process
691 @return: the file descriptor of the lock file; do not close this unless
692 you want to unlock the pid file
695 # We don't rename nor truncate the file to not drop locks under
697 fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
699 # Lock the PID file (and fail if not possible to do so). Any code
700 # wanting to send a signal to the daemon should try to lock the PID
701 # file before reading it. If acquiring the lock succeeds, the daemon is
702 # no longer running and the signal should not be sent.
703 filelock.LockFile(fd_pidfile)
705 os.write(fd_pidfile, "%d\n" % os.getpid())
710 def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
711 """Reads the watcher pause file.
713 @type filename: string
714 @param filename: Path to watcher pause file
715 @type now: None, float or int
716 @param now: Current time as Unix timestamp
717 @type remove_after: int
718 @param remove_after: Remove watcher pause file after specified amount of
719 seconds past the pause end time
726 value = ReadFile(filename)
728 if err.errno != errno.ENOENT:
732 if value is not None:
736 logging.warning(("Watcher pause file (%s) contains invalid value,"
737 " removing it"), filename)
741 if value is not None:
742 # Remove file if it's outdated
743 if now > (value + remove_after):
754 """Returns a random UUID.
756 @note: This is a Linux-specific method as it uses the /proc
761 return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")