Statistics
| Branch: | Tag: | Revision:

root / lib / utils / io.py @ 90e234a6

History | View | Annotate | Download (20.2 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21
"""Utility functions for I/O.
22

23
"""
24

    
25
import os
26
import logging
27
import shutil
28
import tempfile
29
import errno
30
import time
31

    
32
from ganeti import errors
33
from ganeti import constants
34
from ganeti.utils import filelock
35

    
36

    
37
#: Path generating random UUID
38
_RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
39

    
40

    
41
def ReadFile(file_name, size=-1):
42
  """Reads a file.
43

44
  @type size: int
45
  @param size: Read at most size bytes (if negative, entire file)
46
  @rtype: str
47
  @return: the (possibly partial) content of the file
48

49
  """
50
  f = open(file_name, "r")
51
  try:
52
    return f.read(size)
53
  finally:
54
    f.close()
55

    
56

    
57
def WriteFile(file_name, fn=None, data=None,
58
              mode=None, uid=-1, gid=-1,
59
              atime=None, mtime=None, close=True,
60
              dry_run=False, backup=False,
61
              prewrite=None, postwrite=None):
62
  """(Over)write a file atomically.
63

64
  The file_name and either fn (a function taking one argument, the
65
  file descriptor, and which should write the data to it) or data (the
66
  contents of the file) must be passed. The other arguments are
67
  optional and allow setting the file mode, owner and group, and the
68
  mtime/atime of the file.
69

70
  If the function doesn't raise an exception, it has succeeded and the
71
  target file has the new contents. If the function has raised an
72
  exception, an existing target file should be unmodified and the
73
  temporary file should be removed.
74

75
  @type file_name: str
76
  @param file_name: the target filename
77
  @type fn: callable
78
  @param fn: content writing function, called with
79
      file descriptor as parameter
80
  @type data: str
81
  @param data: contents of the file
82
  @type mode: int
83
  @param mode: file mode
84
  @type uid: int
85
  @param uid: the owner of the file
86
  @type gid: int
87
  @param gid: the group of the file
88
  @type atime: int
89
  @param atime: a custom access time to be set on the file
90
  @type mtime: int
91
  @param mtime: a custom modification time to be set on the file
92
  @type close: boolean
93
  @param close: whether to close file after writing it
94
  @type prewrite: callable
95
  @param prewrite: function to be called before writing content
96
  @type postwrite: callable
97
  @param postwrite: function to be called after writing content
98

99
  @rtype: None or int
100
  @return: None if the 'close' parameter evaluates to True,
101
      otherwise the file descriptor
102

103
  @raise errors.ProgrammerError: if any of the arguments are not valid
104

105
  """
106
  if not os.path.isabs(file_name):
107
    raise errors.ProgrammerError("Path passed to WriteFile is not"
108
                                 " absolute: '%s'" % file_name)
109

    
110
  if [fn, data].count(None) != 1:
111
    raise errors.ProgrammerError("fn or data required")
112

    
113
  if [atime, mtime].count(None) == 1:
114
    raise errors.ProgrammerError("Both atime and mtime must be either"
115
                                 " set or None")
116

    
117
  if backup and not dry_run and os.path.isfile(file_name):
118
    CreateBackup(file_name)
119

    
120
  dir_name, base_name = os.path.split(file_name)
121
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
122
  do_remove = True
123
  # here we need to make sure we remove the temp file, if any error
124
  # leaves it in place
125
  try:
126
    if uid != -1 or gid != -1:
127
      os.chown(new_name, uid, gid)
128
    if mode:
129
      os.chmod(new_name, mode)
130
    if callable(prewrite):
131
      prewrite(fd)
132
    if data is not None:
133
      os.write(fd, data)
134
    else:
135
      fn(fd)
136
    if callable(postwrite):
137
      postwrite(fd)
138
    os.fsync(fd)
139
    if atime is not None and mtime is not None:
140
      os.utime(new_name, (atime, mtime))
141
    if not dry_run:
142
      os.rename(new_name, file_name)
143
      do_remove = False
144
  finally:
145
    if close:
146
      os.close(fd)
147
      result = None
148
    else:
149
      result = fd
150
    if do_remove:
151
      RemoveFile(new_name)
152

    
153
  return result
154

    
155

    
156
def GetFileID(path=None, fd=None):
157
  """Returns the file 'id', i.e. the dev/inode and mtime information.
158

159
  Either the path to the file or the fd must be given.
160

161
  @param path: the file path
162
  @param fd: a file descriptor
163
  @return: a tuple of (device number, inode number, mtime)
164

165
  """
166
  if [path, fd].count(None) != 1:
167
    raise errors.ProgrammerError("One and only one of fd/path must be given")
168

    
169
  if fd is None:
170
    st = os.stat(path)
171
  else:
172
    st = os.fstat(fd)
173

    
174
  return (st.st_dev, st.st_ino, st.st_mtime)
175

    
176

    
177
def VerifyFileID(fi_disk, fi_ours):
178
  """Verifies that two file IDs are matching.
179

180
  Differences in the inode/device are not accepted, but and older
181
  timestamp for fi_disk is accepted.
182

183
  @param fi_disk: tuple (dev, inode, mtime) representing the actual
184
      file data
185
  @param fi_ours: tuple (dev, inode, mtime) representing the last
186
      written file data
187
  @rtype: boolean
188

189
  """
190
  (d1, i1, m1) = fi_disk
191
  (d2, i2, m2) = fi_ours
192

    
193
  return (d1, i1) == (d2, i2) and m1 <= m2
194

    
195

    
196
def SafeWriteFile(file_name, file_id, **kwargs):
197
  """Wraper over L{WriteFile} that locks the target file.
198

199
  By keeping the target file locked during WriteFile, we ensure that
200
  cooperating writers will safely serialise access to the file.
201

202
  @type file_name: str
203
  @param file_name: the target filename
204
  @type file_id: tuple
205
  @param file_id: a result from L{GetFileID}
206

207
  """
208
  fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
209
  try:
210
    filelock.LockFile(fd)
211
    if file_id is not None:
212
      disk_id = GetFileID(fd=fd)
213
      if not VerifyFileID(disk_id, file_id):
214
        raise errors.LockError("Cannot overwrite file %s, it has been modified"
215
                               " since last written" % file_name)
216
    return WriteFile(file_name, **kwargs)
217
  finally:
218
    os.close(fd)
219

    
220

    
221
def ReadOneLineFile(file_name, strict=False):
222
  """Return the first non-empty line from a file.
223

224
  @type strict: boolean
225
  @param strict: if True, abort if the file has more than one
226
      non-empty line
227

228
  """
229
  file_lines = ReadFile(file_name).splitlines()
230
  full_lines = filter(bool, file_lines)
231
  if not file_lines or not full_lines:
232
    raise errors.GenericError("No data in one-liner file %s" % file_name)
233
  elif strict and len(full_lines) > 1:
234
    raise errors.GenericError("Too many lines in one-liner file %s" %
235
                              file_name)
236
  return full_lines[0]
237

    
238

    
239
def RemoveFile(filename):
240
  """Remove a file ignoring some errors.
241

242
  Remove a file, ignoring non-existing ones or directories. Other
243
  errors are passed.
244

245
  @type filename: str
246
  @param filename: the file to be removed
247

248
  """
249
  try:
250
    os.unlink(filename)
251
  except OSError, err:
252
    if err.errno not in (errno.ENOENT, errno.EISDIR):
253
      raise
254

    
255

    
256
def RemoveDir(dirname):
257
  """Remove an empty directory.
258

259
  Remove a directory, ignoring non-existing ones.
260
  Other errors are passed. This includes the case,
261
  where the directory is not empty, so it can't be removed.
262

263
  @type dirname: str
264
  @param dirname: the empty directory to be removed
265

266
  """
267
  try:
268
    os.rmdir(dirname)
269
  except OSError, err:
270
    if err.errno != errno.ENOENT:
271
      raise
272

    
273

    
274
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
275
  """Renames a file.
276

277
  @type old: string
278
  @param old: Original path
279
  @type new: string
280
  @param new: New path
281
  @type mkdir: bool
282
  @param mkdir: Whether to create target directory if it doesn't exist
283
  @type mkdir_mode: int
284
  @param mkdir_mode: Mode for newly created directories
285

286
  """
287
  try:
288
    return os.rename(old, new)
289
  except OSError, err:
290
    # In at least one use case of this function, the job queue, directory
291
    # creation is very rare. Checking for the directory before renaming is not
292
    # as efficient.
293
    if mkdir and err.errno == errno.ENOENT:
294
      # Create directory and try again
295
      Makedirs(os.path.dirname(new), mode=mkdir_mode)
296

    
297
      return os.rename(old, new)
298

    
299
    raise
300

    
301

    
302
def Makedirs(path, mode=0750):
303
  """Super-mkdir; create a leaf directory and all intermediate ones.
304

305
  This is a wrapper around C{os.makedirs} adding error handling not implemented
306
  before Python 2.5.
307

308
  """
309
  try:
310
    os.makedirs(path, mode)
311
  except OSError, err:
312
    # Ignore EEXIST. This is only handled in os.makedirs as included in
313
    # Python 2.5 and above.
314
    if err.errno != errno.EEXIST or not os.path.exists(path):
315
      raise
316

    
317

    
318
def TimestampForFilename():
319
  """Returns the current time formatted for filenames.
320

321
  The format doesn't contain colons as some shells and applications treat them
322
  as separators. Uses the local timezone.
323

324
  """
325
  return time.strftime("%Y-%m-%d_%H_%M_%S")
326

    
327

    
328
def CreateBackup(file_name):
329
  """Creates a backup of a file.
330

331
  @type file_name: str
332
  @param file_name: file to be backed up
333
  @rtype: str
334
  @return: the path to the newly created backup
335
  @raise errors.ProgrammerError: for invalid file names
336

337
  """
338
  if not os.path.isfile(file_name):
339
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
340
                                file_name)
341

    
342
  prefix = ("%s.backup-%s." %
343
            (os.path.basename(file_name), TimestampForFilename()))
344
  dir_name = os.path.dirname(file_name)
345

    
346
  fsrc = open(file_name, 'rb')
347
  try:
348
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
349
    fdst = os.fdopen(fd, 'wb')
350
    try:
351
      logging.debug("Backing up %s at %s", file_name, backup_name)
352
      shutil.copyfileobj(fsrc, fdst)
353
    finally:
354
      fdst.close()
355
  finally:
356
    fsrc.close()
357

    
358
  return backup_name
359

    
360

    
361
def ListVisibleFiles(path):
362
  """Returns a list of visible files in a directory.
363

364
  @type path: str
365
  @param path: the directory to enumerate
366
  @rtype: list
367
  @return: the list of all files not starting with a dot
368
  @raise ProgrammerError: if L{path} is not an absolue and normalized path
369

370
  """
371
  if not IsNormAbsPath(path):
372
    raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
373
                                 " absolute/normalized: '%s'" % path)
374
  files = [i for i in os.listdir(path) if not i.startswith(".")]
375
  return files
376

    
377

    
378
def EnsureDirs(dirs):
379
  """Make required directories, if they don't exist.
380

381
  @param dirs: list of tuples (dir_name, dir_mode)
382
  @type dirs: list of (string, integer)
383

384
  """
385
  for dir_name, dir_mode in dirs:
386
    try:
387
      os.mkdir(dir_name, dir_mode)
388
    except EnvironmentError, err:
389
      if err.errno != errno.EEXIST:
390
        raise errors.GenericError("Cannot create needed directory"
391
                                  " '%s': %s" % (dir_name, err))
392
    try:
393
      os.chmod(dir_name, dir_mode)
394
    except EnvironmentError, err:
395
      raise errors.GenericError("Cannot change directory permissions on"
396
                                " '%s': %s" % (dir_name, err))
397
    if not os.path.isdir(dir_name):
398
      raise errors.GenericError("%s is not a directory" % dir_name)
399

    
400

    
401
def FindFile(name, search_path, test=os.path.exists):
402
  """Look for a filesystem object in a given path.
403

404
  This is an abstract method to search for filesystem object (files,
405
  dirs) under a given search path.
406

407
  @type name: str
408
  @param name: the name to look for
409
  @type search_path: str
410
  @param search_path: location to start at
411
  @type test: callable
412
  @param test: a function taking one argument that should return True
413
      if the a given object is valid; the default value is
414
      os.path.exists, causing only existing files to be returned
415
  @rtype: str or None
416
  @return: full path to the object if found, None otherwise
417

418
  """
419
  # validate the filename mask
420
  if constants.EXT_PLUGIN_MASK.match(name) is None:
421
    logging.critical("Invalid value passed for external script name: '%s'",
422
                     name)
423
    return None
424

    
425
  for dir_name in search_path:
426
    # FIXME: investigate switch to PathJoin
427
    item_name = os.path.sep.join([dir_name, name])
428
    # check the user test and that we're indeed resolving to the given
429
    # basename
430
    if test(item_name) and os.path.basename(item_name) == name:
431
      return item_name
432
  return None
433

    
434

    
435
def IsNormAbsPath(path):
436
  """Check whether a path is absolute and also normalized
437

438
  This avoids things like /dir/../../other/path to be valid.
439

440
  """
441
  return os.path.normpath(path) == path and os.path.isabs(path)
442

    
443

    
444
def PathJoin(*args):
445
  """Safe-join a list of path components.
446

447
  Requirements:
448
      - the first argument must be an absolute path
449
      - no component in the path must have backtracking (e.g. /../),
450
        since we check for normalization at the end
451

452
  @param args: the path components to be joined
453
  @raise ValueError: for invalid paths
454

455
  """
456
  # ensure we're having at least one path passed in
457
  assert args
458
  # ensure the first component is an absolute and normalized path name
459
  root = args[0]
460
  if not IsNormAbsPath(root):
461
    raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
462
  result = os.path.join(*args)
463
  # ensure that the whole path is normalized
464
  if not IsNormAbsPath(result):
465
    raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
466
  # check that we're still under the original prefix
467
  prefix = os.path.commonprefix([root, result])
468
  if prefix != root:
469
    raise ValueError("Error: path joining resulted in different prefix"
470
                     " (%s != %s)" % (prefix, root))
471
  return result
472

    
473

    
474
def TailFile(fname, lines=20):
475
  """Return the last lines from a file.
476

477
  @note: this function will only read and parse the last 4KB of
478
      the file; if the lines are very long, it could be that less
479
      than the requested number of lines are returned
480

481
  @param fname: the file name
482
  @type lines: int
483
  @param lines: the (maximum) number of lines to return
484

485
  """
486
  fd = open(fname, "r")
487
  try:
488
    fd.seek(0, 2)
489
    pos = fd.tell()
490
    pos = max(0, pos-4096)
491
    fd.seek(pos, 0)
492
    raw_data = fd.read()
493
  finally:
494
    fd.close()
495

    
496
  rows = raw_data.splitlines()
497
  return rows[-lines:]
498

    
499

    
500
def BytesToMebibyte(value):
501
  """Converts bytes to mebibytes.
502

503
  @type value: int
504
  @param value: Value in bytes
505
  @rtype: int
506
  @return: Value in mebibytes
507

508
  """
509
  return int(round(value / (1024.0 * 1024.0), 0))
510

    
511

    
512
def CalculateDirectorySize(path):
513
  """Calculates the size of a directory recursively.
514

515
  @type path: string
516
  @param path: Path to directory
517
  @rtype: int
518
  @return: Size in mebibytes
519

520
  """
521
  size = 0
522

    
523
  for (curpath, _, files) in os.walk(path):
524
    for filename in files:
525
      st = os.lstat(PathJoin(curpath, filename))
526
      size += st.st_size
527

    
528
  return BytesToMebibyte(size)
529

    
530

    
531
def GetFilesystemStats(path):
532
  """Returns the total and free space on a filesystem.
533

534
  @type path: string
535
  @param path: Path on filesystem to be examined
536
  @rtype: int
537
  @return: tuple of (Total space, Free space) in mebibytes
538

539
  """
540
  st = os.statvfs(path)
541

    
542
  fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
543
  tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
544
  return (tsize, fsize)
545

    
546

    
547
def ReadPidFile(pidfile):
548
  """Read a pid from a file.
549

550
  @type  pidfile: string
551
  @param pidfile: path to the file containing the pid
552
  @rtype: int
553
  @return: The process id, if the file exists and contains a valid PID,
554
           otherwise 0
555

556
  """
557
  try:
558
    raw_data = ReadOneLineFile(pidfile)
559
  except EnvironmentError, err:
560
    if err.errno != errno.ENOENT:
561
      logging.exception("Can't read pid file")
562
    return 0
563

    
564
  try:
565
    pid = int(raw_data)
566
  except (TypeError, ValueError), err:
567
    logging.info("Can't parse pid file contents", exc_info=True)
568
    return 0
569

    
570
  return pid
571

    
572

    
573
def ReadLockedPidFile(path):
574
  """Reads a locked PID file.
575

576
  This can be used together with L{utils.process.StartDaemon}.
577

578
  @type path: string
579
  @param path: Path to PID file
580
  @return: PID as integer or, if file was unlocked or couldn't be opened, None
581

582
  """
583
  try:
584
    fd = os.open(path, os.O_RDONLY)
585
  except EnvironmentError, err:
586
    if err.errno == errno.ENOENT:
587
      # PID file doesn't exist
588
      return None
589
    raise
590

    
591
  try:
592
    try:
593
      # Try to acquire lock
594
      filelock.LockFile(fd)
595
    except errors.LockError:
596
      # Couldn't lock, daemon is running
597
      return int(os.read(fd, 100))
598
  finally:
599
    os.close(fd)
600

    
601
  return None
602

    
603

    
604
def AddAuthorizedKey(file_obj, key):
605
  """Adds an SSH public key to an authorized_keys file.
606

607
  @type file_obj: str or file handle
608
  @param file_obj: path to authorized_keys file
609
  @type key: str
610
  @param key: string containing key
611

612
  """
613
  key_fields = key.split()
614

    
615
  if isinstance(file_obj, basestring):
616
    f = open(file_obj, 'a+')
617
  else:
618
    f = file_obj
619

    
620
  try:
621
    nl = True
622
    for line in f:
623
      # Ignore whitespace changes
624
      if line.split() == key_fields:
625
        break
626
      nl = line.endswith('\n')
627
    else:
628
      if not nl:
629
        f.write("\n")
630
      f.write(key.rstrip('\r\n'))
631
      f.write("\n")
632
      f.flush()
633
  finally:
634
    f.close()
635

    
636

    
637
def RemoveAuthorizedKey(file_name, key):
638
  """Removes an SSH public key from an authorized_keys file.
639

640
  @type file_name: str
641
  @param file_name: path to authorized_keys file
642
  @type key: str
643
  @param key: string containing key
644

645
  """
646
  key_fields = key.split()
647

    
648
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
649
  try:
650
    out = os.fdopen(fd, 'w')
651
    try:
652
      f = open(file_name, 'r')
653
      try:
654
        for line in f:
655
          # Ignore whitespace changes while comparing lines
656
          if line.split() != key_fields:
657
            out.write(line)
658

    
659
        out.flush()
660
        os.rename(tmpname, file_name)
661
      finally:
662
        f.close()
663
    finally:
664
      out.close()
665
  except:
666
    RemoveFile(tmpname)
667
    raise
668

    
669

    
670
def DaemonPidFileName(name):
671
  """Compute a ganeti pid file absolute path
672

673
  @type name: str
674
  @param name: the daemon name
675
  @rtype: str
676
  @return: the full path to the pidfile corresponding to the given
677
      daemon name
678

679
  """
680
  return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
681

    
682

    
683
def WritePidFile(pidfile):
684
  """Write the current process pidfile.
685

686
  @type pidfile: string
687
  @param pidfile: the path to the file to be written
688
  @raise errors.LockError: if the pid file already exists and
689
      points to a live process
690
  @rtype: int
691
  @return: the file descriptor of the lock file; do not close this unless
692
      you want to unlock the pid file
693

694
  """
695
  # We don't rename nor truncate the file to not drop locks under
696
  # existing processes
697
  fd_pidfile = os.open(pidfile, os.O_WRONLY | os.O_CREAT, 0600)
698

    
699
  # Lock the PID file (and fail if not possible to do so). Any code
700
  # wanting to send a signal to the daemon should try to lock the PID
701
  # file before reading it. If acquiring the lock succeeds, the daemon is
702
  # no longer running and the signal should not be sent.
703
  filelock.LockFile(fd_pidfile)
704

    
705
  os.write(fd_pidfile, "%d\n" % os.getpid())
706

    
707
  return fd_pidfile
708

    
709

    
710
def RemovePidFile(pidfile):
711
  """Remove the current process pidfile.
712

713
  Any errors are ignored.
714

715
  @type pidfile: string
716
  @param pidfile: Path to the file to be removed
717

718
  """
719
  # TODO: we could check here that the file contains our pid
720
  try:
721
    RemoveFile(pidfile)
722
  except Exception: # pylint: disable-msg=W0703
723
    pass
724

    
725

    
726
def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
727
  """Reads the watcher pause file.
728

729
  @type filename: string
730
  @param filename: Path to watcher pause file
731
  @type now: None, float or int
732
  @param now: Current time as Unix timestamp
733
  @type remove_after: int
734
  @param remove_after: Remove watcher pause file after specified amount of
735
    seconds past the pause end time
736

737
  """
738
  if now is None:
739
    now = time.time()
740

    
741
  try:
742
    value = ReadFile(filename)
743
  except IOError, err:
744
    if err.errno != errno.ENOENT:
745
      raise
746
    value = None
747

    
748
  if value is not None:
749
    try:
750
      value = int(value)
751
    except ValueError:
752
      logging.warning(("Watcher pause file (%s) contains invalid value,"
753
                       " removing it"), filename)
754
      RemoveFile(filename)
755
      value = None
756

    
757
    if value is not None:
758
      # Remove file if it's outdated
759
      if now > (value + remove_after):
760
        RemoveFile(filename)
761
        value = None
762

    
763
      elif now > value:
764
        value = None
765

    
766
  return value
767

    
768

    
769
def NewUUID():
770
  """Returns a random UUID.
771

772
  @note: This is a Linux-specific method as it uses the /proc
773
      filesystem.
774
  @rtype: str
775

776
  """
777
  return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")