Statistics
| Branch: | Tag: | Revision:

root / lib / utils / io.py @ 1cc324f0

History | View | Annotate | Download (29.9 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2010, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21
"""Utility functions for I/O.
22

23
"""
24

    
25
import os
26
import logging
27
import shutil
28
import tempfile
29
import errno
30
import time
31
import stat
32
import grp
33
import pwd
34

    
35
from ganeti import errors
36
from ganeti import constants
37
from ganeti import pathutils
38
from ganeti.utils import filelock
39

    
40
#: Directory used by fsck(8) to store recovered data, usually at a file
41
#: system's root directory
42
_LOST_AND_FOUND = "lost+found"
43

    
44
# Possible values for keep_perms in WriteFile()
45
KP_NEVER = 0
46
KP_ALWAYS = 1
47
KP_IF_EXISTS = 2
48

    
49
KEEP_PERMS_VALUES = [
50
  KP_NEVER,
51
  KP_ALWAYS,
52
  KP_IF_EXISTS,
53
  ]
54

    
55

    
56
def ErrnoOrStr(err):
57
  """Format an EnvironmentError exception.
58

59
  If the L{err} argument has an errno attribute, it will be looked up
60
  and converted into a textual C{E...} description. Otherwise the
61
  string representation of the error will be returned.
62

63
  @type err: L{EnvironmentError}
64
  @param err: the exception to format
65

66
  """
67
  if hasattr(err, "errno"):
68
    detail = errno.errorcode[err.errno]
69
  else:
70
    detail = str(err)
71
  return detail
72

    
73

    
74
class FileStatHelper:
75
  """Helper to store file handle's C{fstat}.
76

77
  Useful in combination with L{ReadFile}'s C{preread} parameter.
78

79
  """
80
  def __init__(self):
81
    """Initializes this class.
82

83
    """
84
    self.st = None
85

    
86
  def __call__(self, fh):
87
    """Calls C{fstat} on file handle.
88

89
    """
90
    self.st = os.fstat(fh.fileno())
91

    
92

    
93
def ReadFile(file_name, size=-1, preread=None):
94
  """Reads a file.
95

96
  @type size: int
97
  @param size: Read at most size bytes (if negative, entire file)
98
  @type preread: callable receiving file handle as single parameter
99
  @param preread: Function called before file is read
100
  @rtype: str
101
  @return: the (possibly partial) content of the file
102

103
  """
104
  f = open(file_name, "r")
105
  try:
106
    if preread:
107
      preread(f)
108

    
109
    return f.read(size)
110
  finally:
111
    f.close()
112

    
113

    
114
def WriteFile(file_name, fn=None, data=None,
115
              mode=None, uid=-1, gid=-1,
116
              atime=None, mtime=None, close=True,
117
              dry_run=False, backup=False,
118
              prewrite=None, postwrite=None, keep_perms=KP_NEVER):
119
  """(Over)write a file atomically.
120

121
  The file_name and either fn (a function taking one argument, the
122
  file descriptor, and which should write the data to it) or data (the
123
  contents of the file) must be passed. The other arguments are
124
  optional and allow setting the file mode, owner and group, and the
125
  mtime/atime of the file.
126

127
  If the function doesn't raise an exception, it has succeeded and the
128
  target file has the new contents. If the function has raised an
129
  exception, an existing target file should be unmodified and the
130
  temporary file should be removed.
131

132
  @type file_name: str
133
  @param file_name: the target filename
134
  @type fn: callable
135
  @param fn: content writing function, called with
136
      file descriptor as parameter
137
  @type data: str
138
  @param data: contents of the file
139
  @type mode: int
140
  @param mode: file mode
141
  @type uid: int
142
  @param uid: the owner of the file
143
  @type gid: int
144
  @param gid: the group of the file
145
  @type atime: int
146
  @param atime: a custom access time to be set on the file
147
  @type mtime: int
148
  @param mtime: a custom modification time to be set on the file
149
  @type close: boolean
150
  @param close: whether to close file after writing it
151
  @type prewrite: callable
152
  @param prewrite: function to be called before writing content
153
  @type postwrite: callable
154
  @param postwrite: function to be called after writing content
155
  @type keep_perms: members of L{KEEP_PERMS_VALUES}
156
  @param keep_perms: if L{KP_NEVER} (default), owner, group, and mode are
157
      taken from the other parameters; if L{KP_ALWAYS}, owner, group, and
158
      mode are copied from the existing file; if L{KP_IF_EXISTS}, owner,
159
      group, and mode are taken from the file, and if the file doesn't
160
      exist, they are taken from the other parameters. It is an error to
161
      pass L{KP_ALWAYS} when the file doesn't exist or when C{uid}, C{gid},
162
      or C{mode} are set to non-default values.
163

164
  @rtype: None or int
165
  @return: None if the 'close' parameter evaluates to True,
166
      otherwise the file descriptor
167

168
  @raise errors.ProgrammerError: if any of the arguments are not valid
169

170
  """
171
  if not os.path.isabs(file_name):
172
    raise errors.ProgrammerError("Path passed to WriteFile is not"
173
                                 " absolute: '%s'" % file_name)
174

    
175
  if [fn, data].count(None) != 1:
176
    raise errors.ProgrammerError("fn or data required")
177

    
178
  if [atime, mtime].count(None) == 1:
179
    raise errors.ProgrammerError("Both atime and mtime must be either"
180
                                 " set or None")
181

    
182
  if not keep_perms in KEEP_PERMS_VALUES:
183
    raise errors.ProgrammerError("Invalid value for keep_perms: %s" %
184
                                 keep_perms)
185
  if keep_perms == KP_ALWAYS and (uid != -1 or gid != -1 or mode is not None):
186
    raise errors.ProgrammerError("When keep_perms==KP_ALWAYS, 'uid', 'gid',"
187
                                 " and 'mode' cannot be set")
188

    
189
  if backup and not dry_run and os.path.isfile(file_name):
190
    CreateBackup(file_name)
191

    
192
  if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS:
193
    # os.stat() raises an exception if the file doesn't exist
194
    try:
195
      file_stat = os.stat(file_name)
196
      mode = stat.S_IMODE(file_stat.st_mode)
197
      uid = file_stat.st_uid
198
      gid = file_stat.st_gid
199
    except OSError:
200
      if keep_perms == KP_ALWAYS:
201
        raise
202
      # else: if keeep_perms == KP_IF_EXISTS it's ok if the file doesn't exist
203

    
204
  # Whether temporary file needs to be removed (e.g. if any error occurs)
205
  do_remove = True
206

    
207
  # Function result
208
  result = None
209

    
210
  (dir_name, base_name) = os.path.split(file_name)
211
  (fd, new_name) = tempfile.mkstemp(suffix=".new", prefix=base_name,
212
                                    dir=dir_name)
213
  try:
214
    try:
215
      if uid != -1 or gid != -1:
216
        os.chown(new_name, uid, gid)
217
      if mode:
218
        os.chmod(new_name, mode)
219
      if callable(prewrite):
220
        prewrite(fd)
221
      if data is not None:
222
        if isinstance(data, unicode):
223
          data = data.encode()
224
        assert isinstance(data, str)
225
        to_write = len(data)
226
        offset = 0
227
        while offset < to_write:
228
          written = os.write(fd, buffer(data, offset))
229
          assert written >= 0
230
          assert written <= to_write - offset
231
          offset += written
232
        assert offset == to_write
233
      else:
234
        fn(fd)
235
      if callable(postwrite):
236
        postwrite(fd)
237
      os.fsync(fd)
238
      if atime is not None and mtime is not None:
239
        os.utime(new_name, (atime, mtime))
240
    finally:
241
      # Close file unless the file descriptor should be returned
242
      if close:
243
        os.close(fd)
244
      else:
245
        result = fd
246

    
247
    # Rename file to destination name
248
    if not dry_run:
249
      os.rename(new_name, file_name)
250
      # Successful, no need to remove anymore
251
      do_remove = False
252
  finally:
253
    if do_remove:
254
      RemoveFile(new_name)
255

    
256
  return result
257

    
258

    
259
def GetFileID(path=None, fd=None):
260
  """Returns the file 'id', i.e. the dev/inode and mtime information.
261

262
  Either the path to the file or the fd must be given.
263

264
  @param path: the file path
265
  @param fd: a file descriptor
266
  @return: a tuple of (device number, inode number, mtime)
267

268
  """
269
  if [path, fd].count(None) != 1:
270
    raise errors.ProgrammerError("One and only one of fd/path must be given")
271

    
272
  if fd is None:
273
    st = os.stat(path)
274
  else:
275
    st = os.fstat(fd)
276

    
277
  return (st.st_dev, st.st_ino, st.st_mtime)
278

    
279

    
280
def VerifyFileID(fi_disk, fi_ours):
281
  """Verifies that two file IDs are matching.
282

283
  Differences in the inode/device are not accepted, but and older
284
  timestamp for fi_disk is accepted.
285

286
  @param fi_disk: tuple (dev, inode, mtime) representing the actual
287
      file data
288
  @param fi_ours: tuple (dev, inode, mtime) representing the last
289
      written file data
290
  @rtype: boolean
291

292
  """
293
  (d1, i1, m1) = fi_disk
294
  (d2, i2, m2) = fi_ours
295

    
296
  return (d1, i1) == (d2, i2) and m1 <= m2
297

    
298

    
299
def SafeWriteFile(file_name, file_id, **kwargs):
300
  """Wraper over L{WriteFile} that locks the target file.
301

302
  By keeping the target file locked during WriteFile, we ensure that
303
  cooperating writers will safely serialise access to the file.
304

305
  @type file_name: str
306
  @param file_name: the target filename
307
  @type file_id: tuple
308
  @param file_id: a result from L{GetFileID}
309

310
  """
311
  fd = os.open(file_name, os.O_RDONLY | os.O_CREAT)
312
  try:
313
    filelock.LockFile(fd)
314
    if file_id is not None:
315
      disk_id = GetFileID(fd=fd)
316
      if not VerifyFileID(disk_id, file_id):
317
        raise errors.LockError("Cannot overwrite file %s, it has been modified"
318
                               " since last written" % file_name)
319
    return WriteFile(file_name, **kwargs)
320
  finally:
321
    os.close(fd)
322

    
323

    
324
def ReadOneLineFile(file_name, strict=False):
325
  """Return the first non-empty line from a file.
326

327
  @type strict: boolean
328
  @param strict: if True, abort if the file has more than one
329
      non-empty line
330

331
  """
332
  file_lines = ReadFile(file_name).splitlines()
333
  full_lines = filter(bool, file_lines)
334
  if not file_lines or not full_lines:
335
    raise errors.GenericError("No data in one-liner file %s" % file_name)
336
  elif strict and len(full_lines) > 1:
337
    raise errors.GenericError("Too many lines in one-liner file %s" %
338
                              file_name)
339
  return full_lines[0]
340

    
341

    
342
def RemoveFile(filename):
343
  """Remove a file ignoring some errors.
344

345
  Remove a file, ignoring non-existing ones or directories. Other
346
  errors are passed.
347

348
  @type filename: str
349
  @param filename: the file to be removed
350

351
  """
352
  try:
353
    os.unlink(filename)
354
  except OSError, err:
355
    if err.errno not in (errno.ENOENT, errno.EISDIR):
356
      raise
357

    
358

    
359
def RemoveDir(dirname):
360
  """Remove an empty directory.
361

362
  Remove a directory, ignoring non-existing ones.
363
  Other errors are passed. This includes the case,
364
  where the directory is not empty, so it can't be removed.
365

366
  @type dirname: str
367
  @param dirname: the empty directory to be removed
368

369
  """
370
  try:
371
    os.rmdir(dirname)
372
  except OSError, err:
373
    if err.errno != errno.ENOENT:
374
      raise
375

    
376

    
377
def RenameFile(old, new, mkdir=False, mkdir_mode=0750, dir_uid=None,
378
               dir_gid=None):
379
  """Renames a file.
380

381
  This just creates the very least directory if it does not exist and C{mkdir}
382
  is set to true.
383

384
  @type old: string
385
  @param old: Original path
386
  @type new: string
387
  @param new: New path
388
  @type mkdir: bool
389
  @param mkdir: Whether to create target directory if it doesn't exist
390
  @type mkdir_mode: int
391
  @param mkdir_mode: Mode for newly created directories
392
  @type dir_uid: int
393
  @param dir_uid: The uid for the (if fresh created) dir
394
  @type dir_gid: int
395
  @param dir_gid: The gid for the (if fresh created) dir
396

397
  """
398
  try:
399
    return os.rename(old, new)
400
  except OSError, err:
401
    # In at least one use case of this function, the job queue, directory
402
    # creation is very rare. Checking for the directory before renaming is not
403
    # as efficient.
404
    if mkdir and err.errno == errno.ENOENT:
405
      # Create directory and try again
406
      dir_path = os.path.dirname(new)
407
      MakeDirWithPerm(dir_path, mkdir_mode, dir_uid, dir_gid)
408

    
409
      return os.rename(old, new)
410

    
411
    raise
412

    
413

    
414
def EnforcePermission(path, mode, uid=None, gid=None, must_exist=True,
415
                      _chmod_fn=os.chmod, _chown_fn=os.chown, _stat_fn=os.stat):
416
  """Enforces that given path has given permissions.
417

418
  @param path: The path to the file
419
  @param mode: The mode of the file
420
  @param uid: The uid of the owner of this file
421
  @param gid: The gid of the owner of this file
422
  @param must_exist: Specifies if non-existance of path will be an error
423
  @param _chmod_fn: chmod function to use (unittest only)
424
  @param _chown_fn: chown function to use (unittest only)
425

426
  """
427
  logging.debug("Checking %s", path)
428

    
429
  # chown takes -1 if you want to keep one part of the ownership, however
430
  # None is Python standard for that. So we remap them here.
431
  if uid is None:
432
    uid = -1
433
  if gid is None:
434
    gid = -1
435

    
436
  try:
437
    st = _stat_fn(path)
438

    
439
    fmode = stat.S_IMODE(st[stat.ST_MODE])
440
    if fmode != mode:
441
      logging.debug("Changing mode of %s from %#o to %#o", path, fmode, mode)
442
      _chmod_fn(path, mode)
443

    
444
    if max(uid, gid) > -1:
445
      fuid = st[stat.ST_UID]
446
      fgid = st[stat.ST_GID]
447
      if fuid != uid or fgid != gid:
448
        logging.debug("Changing owner of %s from UID %s/GID %s to"
449
                      " UID %s/GID %s", path, fuid, fgid, uid, gid)
450
        _chown_fn(path, uid, gid)
451
  except EnvironmentError, err:
452
    if err.errno == errno.ENOENT:
453
      if must_exist:
454
        raise errors.GenericError("Path %s should exist, but does not" % path)
455
    else:
456
      raise errors.GenericError("Error while changing permissions on %s: %s" %
457
                                (path, err))
458

    
459

    
460
def MakeDirWithPerm(path, mode, uid, gid, _lstat_fn=os.lstat,
461
                    _mkdir_fn=os.mkdir, _perm_fn=EnforcePermission):
462
  """Enforces that given path is a dir and has given mode, uid and gid set.
463

464
  @param path: The path to the file
465
  @param mode: The mode of the file
466
  @param uid: The uid of the owner of this file
467
  @param gid: The gid of the owner of this file
468
  @param _lstat_fn: Stat function to use (unittest only)
469
  @param _mkdir_fn: mkdir function to use (unittest only)
470
  @param _perm_fn: permission setter function to use (unittest only)
471

472
  """
473
  logging.debug("Checking directory %s", path)
474
  try:
475
    # We don't want to follow symlinks
476
    st = _lstat_fn(path)
477
  except EnvironmentError, err:
478
    if err.errno != errno.ENOENT:
479
      raise errors.GenericError("stat(2) on %s failed: %s" % (path, err))
480
    _mkdir_fn(path)
481
  else:
482
    if not stat.S_ISDIR(st[stat.ST_MODE]):
483
      raise errors.GenericError(("Path %s is expected to be a directory, but "
484
                                 "isn't") % path)
485

    
486
  _perm_fn(path, mode, uid=uid, gid=gid)
487

    
488

    
489
def Makedirs(path, mode=0750):
490
  """Super-mkdir; create a leaf directory and all intermediate ones.
491

492
  This is a wrapper around C{os.makedirs} adding error handling not implemented
493
  before Python 2.5.
494

495
  """
496
  try:
497
    os.makedirs(path, mode)
498
  except OSError, err:
499
    # Ignore EEXIST. This is only handled in os.makedirs as included in
500
    # Python 2.5 and above.
501
    if err.errno != errno.EEXIST or not os.path.exists(path):
502
      raise
503

    
504

    
505
def TimestampForFilename():
506
  """Returns the current time formatted for filenames.
507

508
  The format doesn't contain colons as some shells and applications treat them
509
  as separators. Uses the local timezone.
510

511
  """
512
  return time.strftime("%Y-%m-%d_%H_%M_%S")
513

    
514

    
515
def CreateBackup(file_name):
516
  """Creates a backup of a file.
517

518
  @type file_name: str
519
  @param file_name: file to be backed up
520
  @rtype: str
521
  @return: the path to the newly created backup
522
  @raise errors.ProgrammerError: for invalid file names
523

524
  """
525
  if not os.path.isfile(file_name):
526
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
527
                                 file_name)
528

    
529
  prefix = ("%s.backup-%s." %
530
            (os.path.basename(file_name), TimestampForFilename()))
531
  dir_name = os.path.dirname(file_name)
532

    
533
  fsrc = open(file_name, "rb")
534
  try:
535
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
536
    fdst = os.fdopen(fd, "wb")
537
    try:
538
      logging.debug("Backing up %s at %s", file_name, backup_name)
539
      shutil.copyfileobj(fsrc, fdst)
540
    finally:
541
      fdst.close()
542
  finally:
543
    fsrc.close()
544

    
545
  return backup_name
546

    
547

    
548
def ListVisibleFiles(path, _is_mountpoint=os.path.ismount):
549
  """Returns a list of visible files in a directory.
550

551
  @type path: str
552
  @param path: the directory to enumerate
553
  @rtype: list
554
  @return: the list of all files not starting with a dot
555
  @raise ProgrammerError: if L{path} is not an absolue and normalized path
556

557
  """
558
  if not IsNormAbsPath(path):
559
    raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
560
                                 " absolute/normalized: '%s'" % path)
561

    
562
  mountpoint = _is_mountpoint(path)
563

    
564
  def fn(name):
565
    """File name filter.
566

567
    Ignores files starting with a dot (".") as by Unix convention they're
568
    considered hidden. The "lost+found" directory found at the root of some
569
    filesystems is also hidden.
570

571
    """
572
    return not (name.startswith(".") or
573
                (mountpoint and name == _LOST_AND_FOUND and
574
                 os.path.isdir(os.path.join(path, name))))
575

    
576
  return filter(fn, os.listdir(path))
577

    
578

    
579
def EnsureDirs(dirs):
580
  """Make required directories, if they don't exist.
581

582
  @param dirs: list of tuples (dir_name, dir_mode)
583
  @type dirs: list of (string, integer)
584

585
  """
586
  for dir_name, dir_mode in dirs:
587
    try:
588
      os.mkdir(dir_name, dir_mode)
589
    except EnvironmentError, err:
590
      if err.errno != errno.EEXIST:
591
        raise errors.GenericError("Cannot create needed directory"
592
                                  " '%s': %s" % (dir_name, err))
593
    try:
594
      os.chmod(dir_name, dir_mode)
595
    except EnvironmentError, err:
596
      raise errors.GenericError("Cannot change directory permissions on"
597
                                " '%s': %s" % (dir_name, err))
598
    if not os.path.isdir(dir_name):
599
      raise errors.GenericError("%s is not a directory" % dir_name)
600

    
601

    
602
def FindFile(name, search_path, test=os.path.exists):
603
  """Look for a filesystem object in a given path.
604

605
  This is an abstract method to search for filesystem object (files,
606
  dirs) under a given search path.
607

608
  @type name: str
609
  @param name: the name to look for
610
  @type search_path: str
611
  @param search_path: location to start at
612
  @type test: callable
613
  @param test: a function taking one argument that should return True
614
      if the a given object is valid; the default value is
615
      os.path.exists, causing only existing files to be returned
616
  @rtype: str or None
617
  @return: full path to the object if found, None otherwise
618

619
  """
620
  # validate the filename mask
621
  if constants.EXT_PLUGIN_MASK.match(name) is None:
622
    logging.critical("Invalid value passed for external script name: '%s'",
623
                     name)
624
    return None
625

    
626
  for dir_name in search_path:
627
    # FIXME: investigate switch to PathJoin
628
    item_name = os.path.sep.join([dir_name, name])
629
    # check the user test and that we're indeed resolving to the given
630
    # basename
631
    if test(item_name) and os.path.basename(item_name) == name:
632
      return item_name
633
  return None
634

    
635

    
636
def IsNormAbsPath(path):
637
  """Check whether a path is absolute and also normalized
638

639
  This avoids things like /dir/../../other/path to be valid.
640

641
  """
642
  return os.path.normpath(path) == path and os.path.isabs(path)
643

    
644

    
645
def IsBelowDir(root, other_path):
646
  """Check whether a path is below a root dir.
647

648
  This works around the nasty byte-byte comparison of commonprefix.
649

650
  """
651
  if not (os.path.isabs(root) and os.path.isabs(other_path)):
652
    raise ValueError("Provided paths '%s' and '%s' are not absolute" %
653
                     (root, other_path))
654

    
655
  norm_other = os.path.normpath(other_path)
656

    
657
  if norm_other == os.sep:
658
    # The root directory can never be below another path
659
    return False
660

    
661
  norm_root = os.path.normpath(root)
662

    
663
  if norm_root == os.sep:
664
    # This is the root directory, no need to add another slash
665
    prepared_root = norm_root
666
  else:
667
    prepared_root = "%s%s" % (norm_root, os.sep)
668

    
669
  return os.path.commonprefix([prepared_root, norm_other]) == prepared_root
670

    
671

    
672
def PathJoin(*args):
673
  """Safe-join a list of path components.
674

675
  Requirements:
676
      - the first argument must be an absolute path
677
      - no component in the path must have backtracking (e.g. /../),
678
        since we check for normalization at the end
679

680
  @param args: the path components to be joined
681
  @raise ValueError: for invalid paths
682

683
  """
684
  # ensure we're having at least one path passed in
685
  assert args
686
  # ensure the first component is an absolute and normalized path name
687
  root = args[0]
688
  if not IsNormAbsPath(root):
689
    raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
690
  result = os.path.join(*args)
691
  # ensure that the whole path is normalized
692
  if not IsNormAbsPath(result):
693
    raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
694
  # check that we're still under the original prefix
695
  if not IsBelowDir(root, result):
696
    raise ValueError("Error: path joining resulted in different prefix"
697
                     " (%s != %s)" % (result, root))
698
  return result
699

    
700

    
701
def TailFile(fname, lines=20):
702
  """Return the last lines from a file.
703

704
  @note: this function will only read and parse the last 4KB of
705
      the file; if the lines are very long, it could be that less
706
      than the requested number of lines are returned
707

708
  @param fname: the file name
709
  @type lines: int
710
  @param lines: the (maximum) number of lines to return
711

712
  """
713
  fd = open(fname, "r")
714
  try:
715
    fd.seek(0, 2)
716
    pos = fd.tell()
717
    pos = max(0, pos - 4096)
718
    fd.seek(pos, 0)
719
    raw_data = fd.read()
720
  finally:
721
    fd.close()
722

    
723
  rows = raw_data.splitlines()
724
  return rows[-lines:]
725

    
726

    
727
def BytesToMebibyte(value):
728
  """Converts bytes to mebibytes.
729

730
  @type value: int
731
  @param value: Value in bytes
732
  @rtype: int
733
  @return: Value in mebibytes
734

735
  """
736
  return int(round(value / (1024.0 * 1024.0), 0))
737

    
738

    
739
def CalculateDirectorySize(path):
740
  """Calculates the size of a directory recursively.
741

742
  @type path: string
743
  @param path: Path to directory
744
  @rtype: int
745
  @return: Size in mebibytes
746

747
  """
748
  size = 0
749

    
750
  for (curpath, _, files) in os.walk(path):
751
    for filename in files:
752
      st = os.lstat(PathJoin(curpath, filename))
753
      size += st.st_size
754

    
755
  return BytesToMebibyte(size)
756

    
757

    
758
def GetFilesystemStats(path):
759
  """Returns the total and free space on a filesystem.
760

761
  @type path: string
762
  @param path: Path on filesystem to be examined
763
  @rtype: int
764
  @return: tuple of (Total space, Free space) in mebibytes
765

766
  """
767
  st = os.statvfs(path)
768

    
769
  fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
770
  tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
771
  return (tsize, fsize)
772

    
773

    
774
def ReadPidFile(pidfile):
775
  """Read a pid from a file.
776

777
  @type  pidfile: string
778
  @param pidfile: path to the file containing the pid
779
  @rtype: int
780
  @return: The process id, if the file exists and contains a valid PID,
781
           otherwise 0
782

783
  """
784
  try:
785
    raw_data = ReadOneLineFile(pidfile)
786
  except EnvironmentError, err:
787
    if err.errno != errno.ENOENT:
788
      logging.exception("Can't read pid file")
789
    return 0
790

    
791
  return _ParsePidFileContents(raw_data)
792

    
793

    
794
def _ParsePidFileContents(data):
795
  """Tries to extract a process ID from a PID file's content.
796

797
  @type data: string
798
  @rtype: int
799
  @return: Zero if nothing could be read, PID otherwise
800

801
  """
802
  try:
803
    pid = int(data)
804
  except (TypeError, ValueError):
805
    logging.info("Can't parse pid file contents", exc_info=True)
806
    return 0
807
  else:
808
    return pid
809

    
810

    
811
def ReadLockedPidFile(path):
812
  """Reads a locked PID file.
813

814
  This can be used together with L{utils.process.StartDaemon}.
815

816
  @type path: string
817
  @param path: Path to PID file
818
  @return: PID as integer or, if file was unlocked or couldn't be opened, None
819

820
  """
821
  try:
822
    fd = os.open(path, os.O_RDONLY)
823
  except EnvironmentError, err:
824
    if err.errno == errno.ENOENT:
825
      # PID file doesn't exist
826
      return None
827
    raise
828

    
829
  try:
830
    try:
831
      # Try to acquire lock
832
      filelock.LockFile(fd)
833
    except errors.LockError:
834
      # Couldn't lock, daemon is running
835
      return int(os.read(fd, 100))
836
  finally:
837
    os.close(fd)
838

    
839
  return None
840

    
841

    
842
def _SplitSshKey(key):
843
  """Splits a line for SSH's C{authorized_keys} file.
844

845
  If the line has no options (e.g. no C{command="..."}), only the significant
846
  parts, the key type and its hash, are used. Otherwise the whole line is used
847
  (split at whitespace).
848

849
  @type key: string
850
  @param key: Key line
851
  @rtype: tuple
852

853
  """
854
  parts = key.split()
855

    
856
  if parts and parts[0] in constants.SSHAK_ALL:
857
    # If the key has no options in front of it, we only want the significant
858
    # fields
859
    return (False, parts[:2])
860
  else:
861
    # Can't properly split the line, so use everything
862
    return (True, parts)
863

    
864

    
865
def AddAuthorizedKey(file_obj, key):
866
  """Adds an SSH public key to an authorized_keys file.
867

868
  @type file_obj: str or file handle
869
  @param file_obj: path to authorized_keys file
870
  @type key: str
871
  @param key: string containing key
872

873
  """
874
  key_fields = _SplitSshKey(key)
875

    
876
  if isinstance(file_obj, basestring):
877
    f = open(file_obj, "a+")
878
  else:
879
    f = file_obj
880

    
881
  try:
882
    nl = True
883
    for line in f:
884
      # Ignore whitespace changes
885
      if _SplitSshKey(line) == key_fields:
886
        break
887
      nl = line.endswith("\n")
888
    else:
889
      if not nl:
890
        f.write("\n")
891
      f.write(key.rstrip("\r\n"))
892
      f.write("\n")
893
      f.flush()
894
  finally:
895
    f.close()
896

    
897

    
898
def RemoveAuthorizedKey(file_name, key):
899
  """Removes an SSH public key from an authorized_keys file.
900

901
  @type file_name: str
902
  @param file_name: path to authorized_keys file
903
  @type key: str
904
  @param key: string containing key
905

906
  """
907
  key_fields = _SplitSshKey(key)
908

    
909
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
910
  try:
911
    out = os.fdopen(fd, "w")
912
    try:
913
      f = open(file_name, "r")
914
      try:
915
        for line in f:
916
          # Ignore whitespace changes while comparing lines
917
          if _SplitSshKey(line) != key_fields:
918
            out.write(line)
919

    
920
        out.flush()
921
        os.rename(tmpname, file_name)
922
      finally:
923
        f.close()
924
    finally:
925
      out.close()
926
  except:
927
    RemoveFile(tmpname)
928
    raise
929

    
930

    
931
def DaemonPidFileName(name):
932
  """Compute a ganeti pid file absolute path
933

934
  @type name: str
935
  @param name: the daemon name
936
  @rtype: str
937
  @return: the full path to the pidfile corresponding to the given
938
      daemon name
939

940
  """
941
  return PathJoin(pathutils.RUN_DIR, "%s.pid" % name)
942

    
943

    
944
def WritePidFile(pidfile):
945
  """Write the current process pidfile.
946

947
  @type pidfile: string
948
  @param pidfile: the path to the file to be written
949
  @raise errors.LockError: if the pid file already exists and
950
      points to a live process
951
  @rtype: int
952
  @return: the file descriptor of the lock file; do not close this unless
953
      you want to unlock the pid file
954

955
  """
956
  # We don't rename nor truncate the file to not drop locks under
957
  # existing processes
958
  fd_pidfile = os.open(pidfile, os.O_RDWR | os.O_CREAT, 0600)
959

    
960
  # Lock the PID file (and fail if not possible to do so). Any code
961
  # wanting to send a signal to the daemon should try to lock the PID
962
  # file before reading it. If acquiring the lock succeeds, the daemon is
963
  # no longer running and the signal should not be sent.
964
  try:
965
    filelock.LockFile(fd_pidfile)
966
  except errors.LockError:
967
    msg = ["PID file '%s' is already locked by another process" % pidfile]
968
    # Try to read PID file
969
    pid = _ParsePidFileContents(os.read(fd_pidfile, 100))
970
    if pid > 0:
971
      msg.append(", PID read from file is %s" % pid)
972
    raise errors.PidFileLockError("".join(msg))
973

    
974
  os.write(fd_pidfile, "%d\n" % os.getpid())
975

    
976
  return fd_pidfile
977

    
978

    
979
def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
980
  """Reads the watcher pause file.
981

982
  @type filename: string
983
  @param filename: Path to watcher pause file
984
  @type now: None, float or int
985
  @param now: Current time as Unix timestamp
986
  @type remove_after: int
987
  @param remove_after: Remove watcher pause file after specified amount of
988
    seconds past the pause end time
989

990
  """
991
  if now is None:
992
    now = time.time()
993

    
994
  try:
995
    value = ReadFile(filename)
996
  except IOError, err:
997
    if err.errno != errno.ENOENT:
998
      raise
999
    value = None
1000

    
1001
  if value is not None:
1002
    try:
1003
      value = int(value)
1004
    except ValueError:
1005
      logging.warning(("Watcher pause file (%s) contains invalid value,"
1006
                       " removing it"), filename)
1007
      RemoveFile(filename)
1008
      value = None
1009

    
1010
    if value is not None:
1011
      # Remove file if it's outdated
1012
      if now > (value + remove_after):
1013
        RemoveFile(filename)
1014
        value = None
1015

    
1016
      elif now > value:
1017
        value = None
1018

    
1019
  return value
1020

    
1021

    
1022
def NewUUID():
1023
  """Returns a random UUID.
1024

1025
  @note: This is a Linux-specific method as it uses the /proc
1026
      filesystem.
1027
  @rtype: str
1028

1029
  """
1030
  return ReadFile(constants.RANDOM_UUID_FILE, size=128).rstrip("\n")
1031

    
1032

    
1033
class TemporaryFileManager(object):
1034
  """Stores the list of files to be deleted and removes them on demand.
1035

1036
  """
1037

    
1038
  def __init__(self):
1039
    self._files = []
1040

    
1041
  def __del__(self):
1042
    self.Cleanup()
1043

    
1044
  def Add(self, filename):
1045
    """Add file to list of files to be deleted.
1046

1047
    @type filename: string
1048
    @param filename: path to filename to be added
1049

1050
    """
1051
    self._files.append(filename)
1052

    
1053
  def Remove(self, filename):
1054
    """Remove file from list of files to be deleted.
1055

1056
    @type filename: string
1057
    @param filename: path to filename to be deleted
1058

1059
    """
1060
    self._files.remove(filename)
1061

    
1062
  def Cleanup(self):
1063
    """Delete all files marked for deletion
1064

1065
    """
1066
    while self._files:
1067
      RemoveFile(self._files.pop())
1068

    
1069

    
1070
def IsUserInGroup(uid, gid):
1071
  """Returns True if the user belongs to the group.
1072

1073
  @type uid: int
1074
  @param uid: the user id
1075
  @type gid: int
1076
  @param gid: the group id
1077
  @rtype: bool
1078

1079
  """
1080
  user = pwd.getpwuid(uid)
1081
  group = grp.getgrgid(gid)
1082
  return user.pw_gid == gid or user.pw_name in group.gr_mem
1083

    
1084

    
1085
def CanRead(username, filename):
1086
  """Returns True if the user can access (read) the file.
1087

1088
  @type username: string
1089
  @param username: the name of the user
1090
  @type filename: string
1091
  @param filename: the name of the file
1092
  @rtype: bool
1093

1094
  """
1095
  filestats = os.stat(filename)
1096
  user = pwd.getpwnam(username)
1097
  uid = user.pw_uid
1098
  user_readable = filestats.st_mode & stat.S_IRUSR != 0
1099
  group_readable = filestats.st_mode & stat.S_IRGRP != 0
1100
  return ((filestats.st_uid == uid and user_readable)
1101
          or (filestats.st_uid != uid and
1102
              IsUserInGroup(uid, filestats.st_gid) and group_readable))