Statistics
| Branch: | Tag: | Revision:

root / lib / utils.py @ 14aeab22

History | View | Annotate | Download (84.6 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Ganeti utility module.
23

24
This module holds functions that can be used in both daemons (all) and
25
the command line scripts.
26

27
"""
28

    
29

    
30
import os
31
import time
32
import subprocess
33
import re
34
import socket
35
import tempfile
36
import shutil
37
import errno
38
import pwd
39
import itertools
40
import select
41
import fcntl
42
import resource
43
import logging
44
import logging.handlers
45
import signal
46
import datetime
47
import calendar
48
import collections
49
import struct
50
import IN
51

    
52
from cStringIO import StringIO
53

    
54
try:
55
  from hashlib import sha1
56
except ImportError:
57
  import sha
58
  sha1 = sha.new
59

    
60
try:
61
  import ctypes
62
except ImportError:
63
  ctypes = None
64

    
65
from ganeti import errors
66
from ganeti import constants
67

    
68

    
69
_locksheld = []
70
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
71

    
72
debug_locks = False
73

    
74
#: when set to True, L{RunCmd} is disabled
75
no_fork = False
76

    
77
_RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
78

    
79
# Structure definition for getsockopt(SOL_SOCKET, SO_PEERCRED, ...):
80
# struct ucred { pid_t pid; uid_t uid; gid_t gid; };
81
#
82
# The GNU C Library defines gid_t and uid_t to be "unsigned int" and
83
# pid_t to "int".
84
#
85
# IEEE Std 1003.1-2008:
86
# "nlink_t, uid_t, gid_t, and id_t shall be integer types"
87
# "blksize_t, pid_t, and ssize_t shall be signed integer types"
88
_STRUCT_UCRED = "iII"
89
_STRUCT_UCRED_SIZE = struct.calcsize(_STRUCT_UCRED)
90

    
91
# Flags for mlockall() (from bits/mman.h)
92
_MCL_CURRENT = 1
93
_MCL_FUTURE = 2
94

    
95

    
96
class RunResult(object):
97
  """Holds the result of running external programs.
98

99
  @type exit_code: int
100
  @ivar exit_code: the exit code of the program, or None (if the program
101
      didn't exit())
102
  @type signal: int or None
103
  @ivar signal: the signal that caused the program to finish, or None
104
      (if the program wasn't terminated by a signal)
105
  @type stdout: str
106
  @ivar stdout: the standard output of the program
107
  @type stderr: str
108
  @ivar stderr: the standard error of the program
109
  @type failed: boolean
110
  @ivar failed: True in case the program was
111
      terminated by a signal or exited with a non-zero exit code
112
  @ivar fail_reason: a string detailing the termination reason
113

114
  """
115
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
116
               "failed", "fail_reason", "cmd"]
117

    
118

    
119
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
120
    self.cmd = cmd
121
    self.exit_code = exit_code
122
    self.signal = signal_
123
    self.stdout = stdout
124
    self.stderr = stderr
125
    self.failed = (signal_ is not None or exit_code != 0)
126

    
127
    if self.signal is not None:
128
      self.fail_reason = "terminated by signal %s" % self.signal
129
    elif self.exit_code is not None:
130
      self.fail_reason = "exited with exit code %s" % self.exit_code
131
    else:
132
      self.fail_reason = "unable to determine termination reason"
133

    
134
    if self.failed:
135
      logging.debug("Command '%s' failed (%s); output: %s",
136
                    self.cmd, self.fail_reason, self.output)
137

    
138
  def _GetOutput(self):
139
    """Returns the combined stdout and stderr for easier usage.
140

141
    """
142
    return self.stdout + self.stderr
143

    
144
  output = property(_GetOutput, None, None, "Return full output")
145

    
146

    
147
def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
148
  """Execute a (shell) command.
149

150
  The command should not read from its standard input, as it will be
151
  closed.
152

153
  @type cmd: string or list
154
  @param cmd: Command to run
155
  @type env: dict
156
  @param env: Additional environment
157
  @type output: str
158
  @param output: if desired, the output of the command can be
159
      saved in a file instead of the RunResult instance; this
160
      parameter denotes the file name (if not None)
161
  @type cwd: string
162
  @param cwd: if specified, will be used as the working
163
      directory for the command; the default will be /
164
  @type reset_env: boolean
165
  @param reset_env: whether to reset or keep the default os environment
166
  @rtype: L{RunResult}
167
  @return: RunResult instance
168
  @raise errors.ProgrammerError: if we call this when forks are disabled
169

170
  """
171
  if no_fork:
172
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
173

    
174
  if isinstance(cmd, list):
175
    cmd = [str(val) for val in cmd]
176
    strcmd = " ".join(cmd)
177
    shell = False
178
  else:
179
    strcmd = cmd
180
    shell = True
181
  logging.debug("RunCmd '%s'", strcmd)
182

    
183
  if not reset_env:
184
    cmd_env = os.environ.copy()
185
    cmd_env["LC_ALL"] = "C"
186
  else:
187
    cmd_env = {}
188

    
189
  if env is not None:
190
    cmd_env.update(env)
191

    
192
  try:
193
    if output is None:
194
      out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
195
    else:
196
      status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
197
      out = err = ""
198
  except OSError, err:
199
    if err.errno == errno.ENOENT:
200
      raise errors.OpExecError("Can't execute '%s': not found (%s)" %
201
                               (strcmd, err))
202
    else:
203
      raise
204

    
205
  if status >= 0:
206
    exitcode = status
207
    signal_ = None
208
  else:
209
    exitcode = None
210
    signal_ = -status
211

    
212
  return RunResult(exitcode, signal_, out, err, strcmd)
213

    
214

    
215
def _RunCmdPipe(cmd, env, via_shell, cwd):
216
  """Run a command and return its output.
217

218
  @type  cmd: string or list
219
  @param cmd: Command to run
220
  @type env: dict
221
  @param env: The environment to use
222
  @type via_shell: bool
223
  @param via_shell: if we should run via the shell
224
  @type cwd: string
225
  @param cwd: the working directory for the program
226
  @rtype: tuple
227
  @return: (out, err, status)
228

229
  """
230
  poller = select.poll()
231
  child = subprocess.Popen(cmd, shell=via_shell,
232
                           stderr=subprocess.PIPE,
233
                           stdout=subprocess.PIPE,
234
                           stdin=subprocess.PIPE,
235
                           close_fds=True, env=env,
236
                           cwd=cwd)
237

    
238
  child.stdin.close()
239
  poller.register(child.stdout, select.POLLIN)
240
  poller.register(child.stderr, select.POLLIN)
241
  out = StringIO()
242
  err = StringIO()
243
  fdmap = {
244
    child.stdout.fileno(): (out, child.stdout),
245
    child.stderr.fileno(): (err, child.stderr),
246
    }
247
  for fd in fdmap:
248
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
249
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
250

    
251
  while fdmap:
252
    try:
253
      pollresult = poller.poll()
254
    except EnvironmentError, eerr:
255
      if eerr.errno == errno.EINTR:
256
        continue
257
      raise
258
    except select.error, serr:
259
      if serr[0] == errno.EINTR:
260
        continue
261
      raise
262

    
263
    for fd, event in pollresult:
264
      if event & select.POLLIN or event & select.POLLPRI:
265
        data = fdmap[fd][1].read()
266
        # no data from read signifies EOF (the same as POLLHUP)
267
        if not data:
268
          poller.unregister(fd)
269
          del fdmap[fd]
270
          continue
271
        fdmap[fd][0].write(data)
272
      if (event & select.POLLNVAL or event & select.POLLHUP or
273
          event & select.POLLERR):
274
        poller.unregister(fd)
275
        del fdmap[fd]
276

    
277
  out = out.getvalue()
278
  err = err.getvalue()
279

    
280
  status = child.wait()
281
  return out, err, status
282

    
283

    
284
def _RunCmdFile(cmd, env, via_shell, output, cwd):
285
  """Run a command and save its output to a file.
286

287
  @type  cmd: string or list
288
  @param cmd: Command to run
289
  @type env: dict
290
  @param env: The environment to use
291
  @type via_shell: bool
292
  @param via_shell: if we should run via the shell
293
  @type output: str
294
  @param output: the filename in which to save the output
295
  @type cwd: string
296
  @param cwd: the working directory for the program
297
  @rtype: int
298
  @return: the exit status
299

300
  """
301
  fh = open(output, "a")
302
  try:
303
    child = subprocess.Popen(cmd, shell=via_shell,
304
                             stderr=subprocess.STDOUT,
305
                             stdout=fh,
306
                             stdin=subprocess.PIPE,
307
                             close_fds=True, env=env,
308
                             cwd=cwd)
309

    
310
    child.stdin.close()
311
    status = child.wait()
312
  finally:
313
    fh.close()
314
  return status
315

    
316

    
317
def RunParts(dir_name, env=None, reset_env=False):
318
  """Run Scripts or programs in a directory
319

320
  @type dir_name: string
321
  @param dir_name: absolute path to a directory
322
  @type env: dict
323
  @param env: The environment to use
324
  @type reset_env: boolean
325
  @param reset_env: whether to reset or keep the default os environment
326
  @rtype: list of tuples
327
  @return: list of (name, (one of RUNDIR_STATUS), RunResult)
328

329
  """
330
  rr = []
331

    
332
  try:
333
    dir_contents = ListVisibleFiles(dir_name)
334
  except OSError, err:
335
    logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
336
    return rr
337

    
338
  for relname in sorted(dir_contents):
339
    fname = PathJoin(dir_name, relname)
340
    if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
341
            constants.EXT_PLUGIN_MASK.match(relname) is not None):
342
      rr.append((relname, constants.RUNPARTS_SKIP, None))
343
    else:
344
      try:
345
        result = RunCmd([fname], env=env, reset_env=reset_env)
346
      except Exception, err: # pylint: disable-msg=W0703
347
        rr.append((relname, constants.RUNPARTS_ERR, str(err)))
348
      else:
349
        rr.append((relname, constants.RUNPARTS_RUN, result))
350

    
351
  return rr
352

    
353

    
354
def GetSocketCredentials(sock):
355
  """Returns the credentials of the foreign process connected to a socket.
356

357
  @param sock: Unix socket
358
  @rtype: tuple; (number, number, number)
359
  @return: The PID, UID and GID of the connected foreign process.
360

361
  """
362
  peercred = sock.getsockopt(socket.SOL_SOCKET, IN.SO_PEERCRED,
363
                             _STRUCT_UCRED_SIZE)
364
  return struct.unpack(_STRUCT_UCRED, peercred)
365

    
366

    
367
def RemoveFile(filename):
368
  """Remove a file ignoring some errors.
369

370
  Remove a file, ignoring non-existing ones or directories. Other
371
  errors are passed.
372

373
  @type filename: str
374
  @param filename: the file to be removed
375

376
  """
377
  try:
378
    os.unlink(filename)
379
  except OSError, err:
380
    if err.errno not in (errno.ENOENT, errno.EISDIR):
381
      raise
382

    
383

    
384
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
385
  """Renames a file.
386

387
  @type old: string
388
  @param old: Original path
389
  @type new: string
390
  @param new: New path
391
  @type mkdir: bool
392
  @param mkdir: Whether to create target directory if it doesn't exist
393
  @type mkdir_mode: int
394
  @param mkdir_mode: Mode for newly created directories
395

396
  """
397
  try:
398
    return os.rename(old, new)
399
  except OSError, err:
400
    # In at least one use case of this function, the job queue, directory
401
    # creation is very rare. Checking for the directory before renaming is not
402
    # as efficient.
403
    if mkdir and err.errno == errno.ENOENT:
404
      # Create directory and try again
405
      Makedirs(os.path.dirname(new), mode=mkdir_mode)
406

    
407
      return os.rename(old, new)
408

    
409
    raise
410

    
411

    
412
def Makedirs(path, mode=0750):
413
  """Super-mkdir; create a leaf directory and all intermediate ones.
414

415
  This is a wrapper around C{os.makedirs} adding error handling not implemented
416
  before Python 2.5.
417

418
  """
419
  try:
420
    os.makedirs(path, mode)
421
  except OSError, err:
422
    # Ignore EEXIST. This is only handled in os.makedirs as included in
423
    # Python 2.5 and above.
424
    if err.errno != errno.EEXIST or not os.path.exists(path):
425
      raise
426

    
427

    
428
def ResetTempfileModule():
429
  """Resets the random name generator of the tempfile module.
430

431
  This function should be called after C{os.fork} in the child process to
432
  ensure it creates a newly seeded random generator. Otherwise it would
433
  generate the same random parts as the parent process. If several processes
434
  race for the creation of a temporary file, this could lead to one not getting
435
  a temporary name.
436

437
  """
438
  # pylint: disable-msg=W0212
439
  if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
440
    tempfile._once_lock.acquire()
441
    try:
442
      # Reset random name generator
443
      tempfile._name_sequence = None
444
    finally:
445
      tempfile._once_lock.release()
446
  else:
447
    logging.critical("The tempfile module misses at least one of the"
448
                     " '_once_lock' and '_name_sequence' attributes")
449

    
450

    
451
def _FingerprintFile(filename):
452
  """Compute the fingerprint of a file.
453

454
  If the file does not exist, a None will be returned
455
  instead.
456

457
  @type filename: str
458
  @param filename: the filename to checksum
459
  @rtype: str
460
  @return: the hex digest of the sha checksum of the contents
461
      of the file
462

463
  """
464
  if not (os.path.exists(filename) and os.path.isfile(filename)):
465
    return None
466

    
467
  f = open(filename)
468

    
469
  fp = sha1()
470
  while True:
471
    data = f.read(4096)
472
    if not data:
473
      break
474

    
475
    fp.update(data)
476

    
477
  return fp.hexdigest()
478

    
479

    
480
def FingerprintFiles(files):
481
  """Compute fingerprints for a list of files.
482

483
  @type files: list
484
  @param files: the list of filename to fingerprint
485
  @rtype: dict
486
  @return: a dictionary filename: fingerprint, holding only
487
      existing files
488

489
  """
490
  ret = {}
491

    
492
  for filename in files:
493
    cksum = _FingerprintFile(filename)
494
    if cksum:
495
      ret[filename] = cksum
496

    
497
  return ret
498

    
499

    
500
def ForceDictType(target, key_types, allowed_values=None):
501
  """Force the values of a dict to have certain types.
502

503
  @type target: dict
504
  @param target: the dict to update
505
  @type key_types: dict
506
  @param key_types: dict mapping target dict keys to types
507
                    in constants.ENFORCEABLE_TYPES
508
  @type allowed_values: list
509
  @keyword allowed_values: list of specially allowed values
510

511
  """
512
  if allowed_values is None:
513
    allowed_values = []
514

    
515
  if not isinstance(target, dict):
516
    msg = "Expected dictionary, got '%s'" % target
517
    raise errors.TypeEnforcementError(msg)
518

    
519
  for key in target:
520
    if key not in key_types:
521
      msg = "Unknown key '%s'" % key
522
      raise errors.TypeEnforcementError(msg)
523

    
524
    if target[key] in allowed_values:
525
      continue
526

    
527
    ktype = key_types[key]
528
    if ktype not in constants.ENFORCEABLE_TYPES:
529
      msg = "'%s' has non-enforceable type %s" % (key, ktype)
530
      raise errors.ProgrammerError(msg)
531

    
532
    if ktype == constants.VTYPE_STRING:
533
      if not isinstance(target[key], basestring):
534
        if isinstance(target[key], bool) and not target[key]:
535
          target[key] = ''
536
        else:
537
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
538
          raise errors.TypeEnforcementError(msg)
539
    elif ktype == constants.VTYPE_BOOL:
540
      if isinstance(target[key], basestring) and target[key]:
541
        if target[key].lower() == constants.VALUE_FALSE:
542
          target[key] = False
543
        elif target[key].lower() == constants.VALUE_TRUE:
544
          target[key] = True
545
        else:
546
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
547
          raise errors.TypeEnforcementError(msg)
548
      elif target[key]:
549
        target[key] = True
550
      else:
551
        target[key] = False
552
    elif ktype == constants.VTYPE_SIZE:
553
      try:
554
        target[key] = ParseUnit(target[key])
555
      except errors.UnitParseError, err:
556
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
557
              (key, target[key], err)
558
        raise errors.TypeEnforcementError(msg)
559
    elif ktype == constants.VTYPE_INT:
560
      try:
561
        target[key] = int(target[key])
562
      except (ValueError, TypeError):
563
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
564
        raise errors.TypeEnforcementError(msg)
565

    
566

    
567
def IsProcessAlive(pid):
568
  """Check if a given pid exists on the system.
569

570
  @note: zombie status is not handled, so zombie processes
571
      will be returned as alive
572
  @type pid: int
573
  @param pid: the process ID to check
574
  @rtype: boolean
575
  @return: True if the process exists
576

577
  """
578
  def _TryStat(name):
579
    try:
580
      os.stat(name)
581
      return True
582
    except EnvironmentError, err:
583
      if err.errno in (errno.ENOENT, errno.ENOTDIR):
584
        return False
585
      elif err.errno == errno.EINVAL:
586
        raise RetryAgain(err)
587
      raise
588

    
589
  assert isinstance(pid, int), "pid must be an integer"
590
  if pid <= 0:
591
    return False
592

    
593
  proc_entry = "/proc/%d/status" % pid
594
  # /proc in a multiprocessor environment can have strange behaviors.
595
  # Retry the os.stat a few times until we get a good result.
596
  try:
597
    return Retry(_TryStat, (0.01, 1.5, 0.1), 0.5, args=[proc_entry])
598
  except RetryTimeout, err:
599
    err.RaiseInner()
600

    
601

    
602
def ReadPidFile(pidfile):
603
  """Read a pid from a file.
604

605
  @type  pidfile: string
606
  @param pidfile: path to the file containing the pid
607
  @rtype: int
608
  @return: The process id, if the file exists and contains a valid PID,
609
           otherwise 0
610

611
  """
612
  try:
613
    raw_data = ReadOneLineFile(pidfile)
614
  except EnvironmentError, err:
615
    if err.errno != errno.ENOENT:
616
      logging.exception("Can't read pid file")
617
    return 0
618

    
619
  try:
620
    pid = int(raw_data)
621
  except (TypeError, ValueError), err:
622
    logging.info("Can't parse pid file contents", exc_info=True)
623
    return 0
624

    
625
  return pid
626

    
627

    
628
def MatchNameComponent(key, name_list, case_sensitive=True):
629
  """Try to match a name against a list.
630

631
  This function will try to match a name like test1 against a list
632
  like C{['test1.example.com', 'test2.example.com', ...]}. Against
633
  this list, I{'test1'} as well as I{'test1.example'} will match, but
634
  not I{'test1.ex'}. A multiple match will be considered as no match
635
  at all (e.g. I{'test1'} against C{['test1.example.com',
636
  'test1.example.org']}), except when the key fully matches an entry
637
  (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
638

639
  @type key: str
640
  @param key: the name to be searched
641
  @type name_list: list
642
  @param name_list: the list of strings against which to search the key
643
  @type case_sensitive: boolean
644
  @param case_sensitive: whether to provide a case-sensitive match
645

646
  @rtype: None or str
647
  @return: None if there is no match I{or} if there are multiple matches,
648
      otherwise the element from the list which matches
649

650
  """
651
  if key in name_list:
652
    return key
653

    
654
  re_flags = 0
655
  if not case_sensitive:
656
    re_flags |= re.IGNORECASE
657
    key = key.upper()
658
  mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
659
  names_filtered = []
660
  string_matches = []
661
  for name in name_list:
662
    if mo.match(name) is not None:
663
      names_filtered.append(name)
664
      if not case_sensitive and key == name.upper():
665
        string_matches.append(name)
666

    
667
  if len(string_matches) == 1:
668
    return string_matches[0]
669
  if len(names_filtered) == 1:
670
    return names_filtered[0]
671
  return None
672

    
673

    
674
class HostInfo:
675
  """Class implementing resolver and hostname functionality
676

677
  """
678
  _VALID_NAME_RE = re.compile("^[a-z0-9._-]{1,255}$")
679

    
680
  def __init__(self, name=None):
681
    """Initialize the host name object.
682

683
    If the name argument is not passed, it will use this system's
684
    name.
685

686
    """
687
    if name is None:
688
      name = self.SysName()
689

    
690
    self.query = name
691
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
692
    self.ip = self.ipaddrs[0]
693

    
694
  def ShortName(self):
695
    """Returns the hostname without domain.
696

697
    """
698
    return self.name.split('.')[0]
699

    
700
  @staticmethod
701
  def SysName():
702
    """Return the current system's name.
703

704
    This is simply a wrapper over C{socket.gethostname()}.
705

706
    """
707
    return socket.gethostname()
708

    
709
  @staticmethod
710
  def LookupHostname(hostname):
711
    """Look up hostname
712

713
    @type hostname: str
714
    @param hostname: hostname to look up
715

716
    @rtype: tuple
717
    @return: a tuple (name, aliases, ipaddrs) as returned by
718
        C{socket.gethostbyname_ex}
719
    @raise errors.ResolverError: in case of errors in resolving
720

721
    """
722
    try:
723
      result = socket.gethostbyname_ex(hostname)
724
    except socket.gaierror, err:
725
      # hostname not found in DNS
726
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
727

    
728
    return result
729

    
730
  @classmethod
731
  def NormalizeName(cls, hostname):
732
    """Validate and normalize the given hostname.
733

734
    @attention: the validation is a bit more relaxed than the standards
735
        require; most importantly, we allow underscores in names
736
    @raise errors.OpPrereqError: when the name is not valid
737

738
    """
739
    hostname = hostname.lower()
740
    if (not cls._VALID_NAME_RE.match(hostname) or
741
        # double-dots, meaning empty label
742
        ".." in hostname or
743
        # empty initial label
744
        hostname.startswith(".")):
745
      raise errors.OpPrereqError("Invalid hostname '%s'" % hostname,
746
                                 errors.ECODE_INVAL)
747
    if hostname.endswith("."):
748
      hostname = hostname.rstrip(".")
749
    return hostname
750

    
751

    
752
def GetHostInfo(name=None):
753
  """Lookup host name and raise an OpPrereqError for failures"""
754

    
755
  try:
756
    return HostInfo(name)
757
  except errors.ResolverError, err:
758
    raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
759
                               (err[0], err[2]), errors.ECODE_RESOLVER)
760

    
761

    
762
def ListVolumeGroups():
763
  """List volume groups and their size
764

765
  @rtype: dict
766
  @return:
767
       Dictionary with keys volume name and values
768
       the size of the volume
769

770
  """
771
  command = "vgs --noheadings --units m --nosuffix -o name,size"
772
  result = RunCmd(command)
773
  retval = {}
774
  if result.failed:
775
    return retval
776

    
777
  for line in result.stdout.splitlines():
778
    try:
779
      name, size = line.split()
780
      size = int(float(size))
781
    except (IndexError, ValueError), err:
782
      logging.error("Invalid output from vgs (%s): %s", err, line)
783
      continue
784

    
785
    retval[name] = size
786

    
787
  return retval
788

    
789

    
790
def BridgeExists(bridge):
791
  """Check whether the given bridge exists in the system
792

793
  @type bridge: str
794
  @param bridge: the bridge name to check
795
  @rtype: boolean
796
  @return: True if it does
797

798
  """
799
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
800

    
801

    
802
def NiceSort(name_list):
803
  """Sort a list of strings based on digit and non-digit groupings.
804

805
  Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
806
  will sort the list in the logical order C{['a1', 'a2', 'a10',
807
  'a11']}.
808

809
  The sort algorithm breaks each name in groups of either only-digits
810
  or no-digits. Only the first eight such groups are considered, and
811
  after that we just use what's left of the string.
812

813
  @type name_list: list
814
  @param name_list: the names to be sorted
815
  @rtype: list
816
  @return: a copy of the name list sorted with our algorithm
817

818
  """
819
  _SORTER_BASE = "(\D+|\d+)"
820
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
821
                                                  _SORTER_BASE, _SORTER_BASE,
822
                                                  _SORTER_BASE, _SORTER_BASE,
823
                                                  _SORTER_BASE, _SORTER_BASE)
824
  _SORTER_RE = re.compile(_SORTER_FULL)
825
  _SORTER_NODIGIT = re.compile("^\D*$")
826
  def _TryInt(val):
827
    """Attempts to convert a variable to integer."""
828
    if val is None or _SORTER_NODIGIT.match(val):
829
      return val
830
    rval = int(val)
831
    return rval
832

    
833
  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
834
             for name in name_list]
835
  to_sort.sort()
836
  return [tup[1] for tup in to_sort]
837

    
838

    
839
def TryConvert(fn, val):
840
  """Try to convert a value ignoring errors.
841

842
  This function tries to apply function I{fn} to I{val}. If no
843
  C{ValueError} or C{TypeError} exceptions are raised, it will return
844
  the result, else it will return the original value. Any other
845
  exceptions are propagated to the caller.
846

847
  @type fn: callable
848
  @param fn: function to apply to the value
849
  @param val: the value to be converted
850
  @return: The converted value if the conversion was successful,
851
      otherwise the original value.
852

853
  """
854
  try:
855
    nv = fn(val)
856
  except (ValueError, TypeError):
857
    nv = val
858
  return nv
859

    
860

    
861
def IsValidIP(ip):
862
  """Verifies the syntax of an IPv4 address.
863

864
  This function checks if the IPv4 address passes is valid or not based
865
  on syntax (not IP range, class calculations, etc.).
866

867
  @type ip: str
868
  @param ip: the address to be checked
869
  @rtype: a regular expression match object
870
  @return: a regular expression match object, or None if the
871
      address is not valid
872

873
  """
874
  unit = "(0|[1-9]\d{0,2})"
875
  #TODO: convert and return only boolean
876
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
877

    
878

    
879
def IsValidShellParam(word):
880
  """Verifies is the given word is safe from the shell's p.o.v.
881

882
  This means that we can pass this to a command via the shell and be
883
  sure that it doesn't alter the command line and is passed as such to
884
  the actual command.
885

886
  Note that we are overly restrictive here, in order to be on the safe
887
  side.
888

889
  @type word: str
890
  @param word: the word to check
891
  @rtype: boolean
892
  @return: True if the word is 'safe'
893

894
  """
895
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
896

    
897

    
898
def BuildShellCmd(template, *args):
899
  """Build a safe shell command line from the given arguments.
900

901
  This function will check all arguments in the args list so that they
902
  are valid shell parameters (i.e. they don't contain shell
903
  metacharacters). If everything is ok, it will return the result of
904
  template % args.
905

906
  @type template: str
907
  @param template: the string holding the template for the
908
      string formatting
909
  @rtype: str
910
  @return: the expanded command line
911

912
  """
913
  for word in args:
914
    if not IsValidShellParam(word):
915
      raise errors.ProgrammerError("Shell argument '%s' contains"
916
                                   " invalid characters" % word)
917
  return template % args
918

    
919

    
920
def FormatUnit(value, units):
921
  """Formats an incoming number of MiB with the appropriate unit.
922

923
  @type value: int
924
  @param value: integer representing the value in MiB (1048576)
925
  @type units: char
926
  @param units: the type of formatting we should do:
927
      - 'h' for automatic scaling
928
      - 'm' for MiBs
929
      - 'g' for GiBs
930
      - 't' for TiBs
931
  @rtype: str
932
  @return: the formatted value (with suffix)
933

934
  """
935
  if units not in ('m', 'g', 't', 'h'):
936
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
937

    
938
  suffix = ''
939

    
940
  if units == 'm' or (units == 'h' and value < 1024):
941
    if units == 'h':
942
      suffix = 'M'
943
    return "%d%s" % (round(value, 0), suffix)
944

    
945
  elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
946
    if units == 'h':
947
      suffix = 'G'
948
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
949

    
950
  else:
951
    if units == 'h':
952
      suffix = 'T'
953
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
954

    
955

    
956
def ParseUnit(input_string):
957
  """Tries to extract number and scale from the given string.
958

959
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
960
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
961
  is always an int in MiB.
962

963
  """
964
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
965
  if not m:
966
    raise errors.UnitParseError("Invalid format")
967

    
968
  value = float(m.groups()[0])
969

    
970
  unit = m.groups()[1]
971
  if unit:
972
    lcunit = unit.lower()
973
  else:
974
    lcunit = 'm'
975

    
976
  if lcunit in ('m', 'mb', 'mib'):
977
    # Value already in MiB
978
    pass
979

    
980
  elif lcunit in ('g', 'gb', 'gib'):
981
    value *= 1024
982

    
983
  elif lcunit in ('t', 'tb', 'tib'):
984
    value *= 1024 * 1024
985

    
986
  else:
987
    raise errors.UnitParseError("Unknown unit: %s" % unit)
988

    
989
  # Make sure we round up
990
  if int(value) < value:
991
    value += 1
992

    
993
  # Round up to the next multiple of 4
994
  value = int(value)
995
  if value % 4:
996
    value += 4 - value % 4
997

    
998
  return value
999

    
1000

    
1001
def AddAuthorizedKey(file_name, key):
1002
  """Adds an SSH public key to an authorized_keys file.
1003

1004
  @type file_name: str
1005
  @param file_name: path to authorized_keys file
1006
  @type key: str
1007
  @param key: string containing key
1008

1009
  """
1010
  key_fields = key.split()
1011

    
1012
  f = open(file_name, 'a+')
1013
  try:
1014
    nl = True
1015
    for line in f:
1016
      # Ignore whitespace changes
1017
      if line.split() == key_fields:
1018
        break
1019
      nl = line.endswith('\n')
1020
    else:
1021
      if not nl:
1022
        f.write("\n")
1023
      f.write(key.rstrip('\r\n'))
1024
      f.write("\n")
1025
      f.flush()
1026
  finally:
1027
    f.close()
1028

    
1029

    
1030
def RemoveAuthorizedKey(file_name, key):
1031
  """Removes an SSH public key from an authorized_keys file.
1032

1033
  @type file_name: str
1034
  @param file_name: path to authorized_keys file
1035
  @type key: str
1036
  @param key: string containing key
1037

1038
  """
1039
  key_fields = key.split()
1040

    
1041
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1042
  try:
1043
    out = os.fdopen(fd, 'w')
1044
    try:
1045
      f = open(file_name, 'r')
1046
      try:
1047
        for line in f:
1048
          # Ignore whitespace changes while comparing lines
1049
          if line.split() != key_fields:
1050
            out.write(line)
1051

    
1052
        out.flush()
1053
        os.rename(tmpname, file_name)
1054
      finally:
1055
        f.close()
1056
    finally:
1057
      out.close()
1058
  except:
1059
    RemoveFile(tmpname)
1060
    raise
1061

    
1062

    
1063
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1064
  """Sets the name of an IP address and hostname in /etc/hosts.
1065

1066
  @type file_name: str
1067
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1068
  @type ip: str
1069
  @param ip: the IP address
1070
  @type hostname: str
1071
  @param hostname: the hostname to be added
1072
  @type aliases: list
1073
  @param aliases: the list of aliases to add for the hostname
1074

1075
  """
1076
  # FIXME: use WriteFile + fn rather than duplicating its efforts
1077
  # Ensure aliases are unique
1078
  aliases = UniqueSequence([hostname] + aliases)[1:]
1079

    
1080
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1081
  try:
1082
    out = os.fdopen(fd, 'w')
1083
    try:
1084
      f = open(file_name, 'r')
1085
      try:
1086
        for line in f:
1087
          fields = line.split()
1088
          if fields and not fields[0].startswith('#') and ip == fields[0]:
1089
            continue
1090
          out.write(line)
1091

    
1092
        out.write("%s\t%s" % (ip, hostname))
1093
        if aliases:
1094
          out.write(" %s" % ' '.join(aliases))
1095
        out.write('\n')
1096

    
1097
        out.flush()
1098
        os.fsync(out)
1099
        os.chmod(tmpname, 0644)
1100
        os.rename(tmpname, file_name)
1101
      finally:
1102
        f.close()
1103
    finally:
1104
      out.close()
1105
  except:
1106
    RemoveFile(tmpname)
1107
    raise
1108

    
1109

    
1110
def AddHostToEtcHosts(hostname):
1111
  """Wrapper around SetEtcHostsEntry.
1112

1113
  @type hostname: str
1114
  @param hostname: a hostname that will be resolved and added to
1115
      L{constants.ETC_HOSTS}
1116

1117
  """
1118
  hi = HostInfo(name=hostname)
1119
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1120

    
1121

    
1122
def RemoveEtcHostsEntry(file_name, hostname):
1123
  """Removes a hostname from /etc/hosts.
1124

1125
  IP addresses without names are removed from the file.
1126

1127
  @type file_name: str
1128
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1129
  @type hostname: str
1130
  @param hostname: the hostname to be removed
1131

1132
  """
1133
  # FIXME: use WriteFile + fn rather than duplicating its efforts
1134
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1135
  try:
1136
    out = os.fdopen(fd, 'w')
1137
    try:
1138
      f = open(file_name, 'r')
1139
      try:
1140
        for line in f:
1141
          fields = line.split()
1142
          if len(fields) > 1 and not fields[0].startswith('#'):
1143
            names = fields[1:]
1144
            if hostname in names:
1145
              while hostname in names:
1146
                names.remove(hostname)
1147
              if names:
1148
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
1149
              continue
1150

    
1151
          out.write(line)
1152

    
1153
        out.flush()
1154
        os.fsync(out)
1155
        os.chmod(tmpname, 0644)
1156
        os.rename(tmpname, file_name)
1157
      finally:
1158
        f.close()
1159
    finally:
1160
      out.close()
1161
  except:
1162
    RemoveFile(tmpname)
1163
    raise
1164

    
1165

    
1166
def RemoveHostFromEtcHosts(hostname):
1167
  """Wrapper around RemoveEtcHostsEntry.
1168

1169
  @type hostname: str
1170
  @param hostname: hostname that will be resolved and its
1171
      full and shot name will be removed from
1172
      L{constants.ETC_HOSTS}
1173

1174
  """
1175
  hi = HostInfo(name=hostname)
1176
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1177
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1178

    
1179

    
1180
def TimestampForFilename():
1181
  """Returns the current time formatted for filenames.
1182

1183
  The format doesn't contain colons as some shells and applications them as
1184
  separators.
1185

1186
  """
1187
  return time.strftime("%Y-%m-%d_%H_%M_%S")
1188

    
1189

    
1190
def CreateBackup(file_name):
1191
  """Creates a backup of a file.
1192

1193
  @type file_name: str
1194
  @param file_name: file to be backed up
1195
  @rtype: str
1196
  @return: the path to the newly created backup
1197
  @raise errors.ProgrammerError: for invalid file names
1198

1199
  """
1200
  if not os.path.isfile(file_name):
1201
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1202
                                file_name)
1203

    
1204
  prefix = ("%s.backup-%s." %
1205
            (os.path.basename(file_name), TimestampForFilename()))
1206
  dir_name = os.path.dirname(file_name)
1207

    
1208
  fsrc = open(file_name, 'rb')
1209
  try:
1210
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1211
    fdst = os.fdopen(fd, 'wb')
1212
    try:
1213
      logging.debug("Backing up %s at %s", file_name, backup_name)
1214
      shutil.copyfileobj(fsrc, fdst)
1215
    finally:
1216
      fdst.close()
1217
  finally:
1218
    fsrc.close()
1219

    
1220
  return backup_name
1221

    
1222

    
1223
def ShellQuote(value):
1224
  """Quotes shell argument according to POSIX.
1225

1226
  @type value: str
1227
  @param value: the argument to be quoted
1228
  @rtype: str
1229
  @return: the quoted value
1230

1231
  """
1232
  if _re_shell_unquoted.match(value):
1233
    return value
1234
  else:
1235
    return "'%s'" % value.replace("'", "'\\''")
1236

    
1237

    
1238
def ShellQuoteArgs(args):
1239
  """Quotes a list of shell arguments.
1240

1241
  @type args: list
1242
  @param args: list of arguments to be quoted
1243
  @rtype: str
1244
  @return: the quoted arguments concatenated with spaces
1245

1246
  """
1247
  return ' '.join([ShellQuote(i) for i in args])
1248

    
1249

    
1250
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1251
  """Simple ping implementation using TCP connect(2).
1252

1253
  Check if the given IP is reachable by doing attempting a TCP connect
1254
  to it.
1255

1256
  @type target: str
1257
  @param target: the IP or hostname to ping
1258
  @type port: int
1259
  @param port: the port to connect to
1260
  @type timeout: int
1261
  @param timeout: the timeout on the connection attempt
1262
  @type live_port_needed: boolean
1263
  @param live_port_needed: whether a closed port will cause the
1264
      function to return failure, as if there was a timeout
1265
  @type source: str or None
1266
  @param source: if specified, will cause the connect to be made
1267
      from this specific source address; failures to bind other
1268
      than C{EADDRNOTAVAIL} will be ignored
1269

1270
  """
1271
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1272

    
1273
  success = False
1274

    
1275
  if source is not None:
1276
    try:
1277
      sock.bind((source, 0))
1278
    except socket.error, (errcode, _):
1279
      if errcode == errno.EADDRNOTAVAIL:
1280
        success = False
1281

    
1282
  sock.settimeout(timeout)
1283

    
1284
  try:
1285
    sock.connect((target, port))
1286
    sock.close()
1287
    success = True
1288
  except socket.timeout:
1289
    success = False
1290
  except socket.error, (errcode, _):
1291
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1292

    
1293
  return success
1294

    
1295

    
1296
def OwnIpAddress(address):
1297
  """Check if the current host has the the given IP address.
1298

1299
  Currently this is done by TCP-pinging the address from the loopback
1300
  address.
1301

1302
  @type address: string
1303
  @param address: the address to check
1304
  @rtype: bool
1305
  @return: True if we own the address
1306

1307
  """
1308
  return TcpPing(address, constants.DEFAULT_NODED_PORT,
1309
                 source=constants.LOCALHOST_IP_ADDRESS)
1310

    
1311

    
1312
def ListVisibleFiles(path):
1313
  """Returns a list of visible files in a directory.
1314

1315
  @type path: str
1316
  @param path: the directory to enumerate
1317
  @rtype: list
1318
  @return: the list of all files not starting with a dot
1319
  @raise ProgrammerError: if L{path} is not an absolue and normalized path
1320

1321
  """
1322
  if not IsNormAbsPath(path):
1323
    raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
1324
                                 " absolute/normalized: '%s'" % path)
1325
  files = [i for i in os.listdir(path) if not i.startswith(".")]
1326
  files.sort()
1327
  return files
1328

    
1329

    
1330
def GetHomeDir(user, default=None):
1331
  """Try to get the homedir of the given user.
1332

1333
  The user can be passed either as a string (denoting the name) or as
1334
  an integer (denoting the user id). If the user is not found, the
1335
  'default' argument is returned, which defaults to None.
1336

1337
  """
1338
  try:
1339
    if isinstance(user, basestring):
1340
      result = pwd.getpwnam(user)
1341
    elif isinstance(user, (int, long)):
1342
      result = pwd.getpwuid(user)
1343
    else:
1344
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1345
                                   type(user))
1346
  except KeyError:
1347
    return default
1348
  return result.pw_dir
1349

    
1350

    
1351
def NewUUID():
1352
  """Returns a random UUID.
1353

1354
  @note: This is a Linux-specific method as it uses the /proc
1355
      filesystem.
1356
  @rtype: str
1357

1358
  """
1359
  return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1360

    
1361

    
1362
def GenerateSecret(numbytes=20):
1363
  """Generates a random secret.
1364

1365
  This will generate a pseudo-random secret returning an hex string
1366
  (so that it can be used where an ASCII string is needed).
1367

1368
  @param numbytes: the number of bytes which will be represented by the returned
1369
      string (defaulting to 20, the length of a SHA1 hash)
1370
  @rtype: str
1371
  @return: an hex representation of the pseudo-random sequence
1372

1373
  """
1374
  return os.urandom(numbytes).encode('hex')
1375

    
1376

    
1377
def EnsureDirs(dirs):
1378
  """Make required directories, if they don't exist.
1379

1380
  @param dirs: list of tuples (dir_name, dir_mode)
1381
  @type dirs: list of (string, integer)
1382

1383
  """
1384
  for dir_name, dir_mode in dirs:
1385
    try:
1386
      os.mkdir(dir_name, dir_mode)
1387
    except EnvironmentError, err:
1388
      if err.errno != errno.EEXIST:
1389
        raise errors.GenericError("Cannot create needed directory"
1390
                                  " '%s': %s" % (dir_name, err))
1391
    try:
1392
      os.chmod(dir_name, dir_mode)
1393
    except EnvironmentError, err:
1394
      raise errors.GenericError("Cannot change directory permissions on"
1395
                                " '%s': %s" % (dir_name, err))
1396
    if not os.path.isdir(dir_name):
1397
      raise errors.GenericError("%s is not a directory" % dir_name)
1398

    
1399

    
1400
def ReadFile(file_name, size=-1):
1401
  """Reads a file.
1402

1403
  @type size: int
1404
  @param size: Read at most size bytes (if negative, entire file)
1405
  @rtype: str
1406
  @return: the (possibly partial) content of the file
1407

1408
  """
1409
  f = open(file_name, "r")
1410
  try:
1411
    return f.read(size)
1412
  finally:
1413
    f.close()
1414

    
1415

    
1416
def WriteFile(file_name, fn=None, data=None,
1417
              mode=None, uid=-1, gid=-1,
1418
              atime=None, mtime=None, close=True,
1419
              dry_run=False, backup=False,
1420
              prewrite=None, postwrite=None):
1421
  """(Over)write a file atomically.
1422

1423
  The file_name and either fn (a function taking one argument, the
1424
  file descriptor, and which should write the data to it) or data (the
1425
  contents of the file) must be passed. The other arguments are
1426
  optional and allow setting the file mode, owner and group, and the
1427
  mtime/atime of the file.
1428

1429
  If the function doesn't raise an exception, it has succeeded and the
1430
  target file has the new contents. If the function has raised an
1431
  exception, an existing target file should be unmodified and the
1432
  temporary file should be removed.
1433

1434
  @type file_name: str
1435
  @param file_name: the target filename
1436
  @type fn: callable
1437
  @param fn: content writing function, called with
1438
      file descriptor as parameter
1439
  @type data: str
1440
  @param data: contents of the file
1441
  @type mode: int
1442
  @param mode: file mode
1443
  @type uid: int
1444
  @param uid: the owner of the file
1445
  @type gid: int
1446
  @param gid: the group of the file
1447
  @type atime: int
1448
  @param atime: a custom access time to be set on the file
1449
  @type mtime: int
1450
  @param mtime: a custom modification time to be set on the file
1451
  @type close: boolean
1452
  @param close: whether to close file after writing it
1453
  @type prewrite: callable
1454
  @param prewrite: function to be called before writing content
1455
  @type postwrite: callable
1456
  @param postwrite: function to be called after writing content
1457

1458
  @rtype: None or int
1459
  @return: None if the 'close' parameter evaluates to True,
1460
      otherwise the file descriptor
1461

1462
  @raise errors.ProgrammerError: if any of the arguments are not valid
1463

1464
  """
1465
  if not os.path.isabs(file_name):
1466
    raise errors.ProgrammerError("Path passed to WriteFile is not"
1467
                                 " absolute: '%s'" % file_name)
1468

    
1469
  if [fn, data].count(None) != 1:
1470
    raise errors.ProgrammerError("fn or data required")
1471

    
1472
  if [atime, mtime].count(None) == 1:
1473
    raise errors.ProgrammerError("Both atime and mtime must be either"
1474
                                 " set or None")
1475

    
1476
  if backup and not dry_run and os.path.isfile(file_name):
1477
    CreateBackup(file_name)
1478

    
1479
  dir_name, base_name = os.path.split(file_name)
1480
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1481
  do_remove = True
1482
  # here we need to make sure we remove the temp file, if any error
1483
  # leaves it in place
1484
  try:
1485
    if uid != -1 or gid != -1:
1486
      os.chown(new_name, uid, gid)
1487
    if mode:
1488
      os.chmod(new_name, mode)
1489
    if callable(prewrite):
1490
      prewrite(fd)
1491
    if data is not None:
1492
      os.write(fd, data)
1493
    else:
1494
      fn(fd)
1495
    if callable(postwrite):
1496
      postwrite(fd)
1497
    os.fsync(fd)
1498
    if atime is not None and mtime is not None:
1499
      os.utime(new_name, (atime, mtime))
1500
    if not dry_run:
1501
      os.rename(new_name, file_name)
1502
      do_remove = False
1503
  finally:
1504
    if close:
1505
      os.close(fd)
1506
      result = None
1507
    else:
1508
      result = fd
1509
    if do_remove:
1510
      RemoveFile(new_name)
1511

    
1512
  return result
1513

    
1514

    
1515
def ReadOneLineFile(file_name, strict=False):
1516
  """Return the first non-empty line from a file.
1517

1518
  @type strict: boolean
1519
  @param strict: if True, abort if the file has more than one
1520
      non-empty line
1521

1522
  """
1523
  file_lines = ReadFile(file_name).splitlines()
1524
  full_lines = filter(bool, file_lines)
1525
  if not file_lines or not full_lines:
1526
    raise errors.GenericError("No data in one-liner file %s" % file_name)
1527
  elif strict and len(full_lines) > 1:
1528
    raise errors.GenericError("Too many lines in one-liner file %s" %
1529
                              file_name)
1530
  return full_lines[0]
1531

    
1532

    
1533
def FirstFree(seq, base=0):
1534
  """Returns the first non-existing integer from seq.
1535

1536
  The seq argument should be a sorted list of positive integers. The
1537
  first time the index of an element is smaller than the element
1538
  value, the index will be returned.
1539

1540
  The base argument is used to start at a different offset,
1541
  i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1542

1543
  Example: C{[0, 1, 3]} will return I{2}.
1544

1545
  @type seq: sequence
1546
  @param seq: the sequence to be analyzed.
1547
  @type base: int
1548
  @param base: use this value as the base index of the sequence
1549
  @rtype: int
1550
  @return: the first non-used index in the sequence
1551

1552
  """
1553
  for idx, elem in enumerate(seq):
1554
    assert elem >= base, "Passed element is higher than base offset"
1555
    if elem > idx + base:
1556
      # idx is not used
1557
      return idx + base
1558
  return None
1559

    
1560

    
1561
def SingleWaitForFdCondition(fdobj, event, timeout):
1562
  """Waits for a condition to occur on the socket.
1563

1564
  Immediately returns at the first interruption.
1565

1566
  @type fdobj: integer or object supporting a fileno() method
1567
  @param fdobj: entity to wait for events on
1568
  @type event: integer
1569
  @param event: ORed condition (see select module)
1570
  @type timeout: float or None
1571
  @param timeout: Timeout in seconds
1572
  @rtype: int or None
1573
  @return: None for timeout, otherwise occured conditions
1574

1575
  """
1576
  check = (event | select.POLLPRI |
1577
           select.POLLNVAL | select.POLLHUP | select.POLLERR)
1578

    
1579
  if timeout is not None:
1580
    # Poller object expects milliseconds
1581
    timeout *= 1000
1582

    
1583
  poller = select.poll()
1584
  poller.register(fdobj, event)
1585
  try:
1586
    # TODO: If the main thread receives a signal and we have no timeout, we
1587
    # could wait forever. This should check a global "quit" flag or something
1588
    # every so often.
1589
    io_events = poller.poll(timeout)
1590
  except select.error, err:
1591
    if err[0] != errno.EINTR:
1592
      raise
1593
    io_events = []
1594
  if io_events and io_events[0][1] & check:
1595
    return io_events[0][1]
1596
  else:
1597
    return None
1598

    
1599

    
1600
class FdConditionWaiterHelper(object):
1601
  """Retry helper for WaitForFdCondition.
1602

1603
  This class contains the retried and wait functions that make sure
1604
  WaitForFdCondition can continue waiting until the timeout is actually
1605
  expired.
1606

1607
  """
1608

    
1609
  def __init__(self, timeout):
1610
    self.timeout = timeout
1611

    
1612
  def Poll(self, fdobj, event):
1613
    result = SingleWaitForFdCondition(fdobj, event, self.timeout)
1614
    if result is None:
1615
      raise RetryAgain()
1616
    else:
1617
      return result
1618

    
1619
  def UpdateTimeout(self, timeout):
1620
    self.timeout = timeout
1621

    
1622

    
1623
def WaitForFdCondition(fdobj, event, timeout):
1624
  """Waits for a condition to occur on the socket.
1625

1626
  Retries until the timeout is expired, even if interrupted.
1627

1628
  @type fdobj: integer or object supporting a fileno() method
1629
  @param fdobj: entity to wait for events on
1630
  @type event: integer
1631
  @param event: ORed condition (see select module)
1632
  @type timeout: float or None
1633
  @param timeout: Timeout in seconds
1634
  @rtype: int or None
1635
  @return: None for timeout, otherwise occured conditions
1636

1637
  """
1638
  if timeout is not None:
1639
    retrywaiter = FdConditionWaiterHelper(timeout)
1640
    try:
1641
      result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
1642
                     args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
1643
    except RetryTimeout:
1644
      result = None
1645
  else:
1646
    result = None
1647
    while result is None:
1648
      result = SingleWaitForFdCondition(fdobj, event, timeout)
1649
  return result
1650

    
1651

    
1652
def UniqueSequence(seq):
1653
  """Returns a list with unique elements.
1654

1655
  Element order is preserved.
1656

1657
  @type seq: sequence
1658
  @param seq: the sequence with the source elements
1659
  @rtype: list
1660
  @return: list of unique elements from seq
1661

1662
  """
1663
  seen = set()
1664
  return [i for i in seq if i not in seen and not seen.add(i)]
1665

    
1666

    
1667
def NormalizeAndValidateMac(mac):
1668
  """Normalizes and check if a MAC address is valid.
1669

1670
  Checks whether the supplied MAC address is formally correct, only
1671
  accepts colon separated format. Normalize it to all lower.
1672

1673
  @type mac: str
1674
  @param mac: the MAC to be validated
1675
  @rtype: str
1676
  @return: returns the normalized and validated MAC.
1677

1678
  @raise errors.OpPrereqError: If the MAC isn't valid
1679

1680
  """
1681
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1682
  if not mac_check.match(mac):
1683
    raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1684
                               mac, errors.ECODE_INVAL)
1685

    
1686
  return mac.lower()
1687

    
1688

    
1689
def TestDelay(duration):
1690
  """Sleep for a fixed amount of time.
1691

1692
  @type duration: float
1693
  @param duration: the sleep duration
1694
  @rtype: boolean
1695
  @return: False for negative value, True otherwise
1696

1697
  """
1698
  if duration < 0:
1699
    return False, "Invalid sleep duration"
1700
  time.sleep(duration)
1701
  return True, None
1702

    
1703

    
1704
def _CloseFDNoErr(fd, retries=5):
1705
  """Close a file descriptor ignoring errors.
1706

1707
  @type fd: int
1708
  @param fd: the file descriptor
1709
  @type retries: int
1710
  @param retries: how many retries to make, in case we get any
1711
      other error than EBADF
1712

1713
  """
1714
  try:
1715
    os.close(fd)
1716
  except OSError, err:
1717
    if err.errno != errno.EBADF:
1718
      if retries > 0:
1719
        _CloseFDNoErr(fd, retries - 1)
1720
    # else either it's closed already or we're out of retries, so we
1721
    # ignore this and go on
1722

    
1723

    
1724
def CloseFDs(noclose_fds=None):
1725
  """Close file descriptors.
1726

1727
  This closes all file descriptors above 2 (i.e. except
1728
  stdin/out/err).
1729

1730
  @type noclose_fds: list or None
1731
  @param noclose_fds: if given, it denotes a list of file descriptor
1732
      that should not be closed
1733

1734
  """
1735
  # Default maximum for the number of available file descriptors.
1736
  if 'SC_OPEN_MAX' in os.sysconf_names:
1737
    try:
1738
      MAXFD = os.sysconf('SC_OPEN_MAX')
1739
      if MAXFD < 0:
1740
        MAXFD = 1024
1741
    except OSError:
1742
      MAXFD = 1024
1743
  else:
1744
    MAXFD = 1024
1745
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1746
  if (maxfd == resource.RLIM_INFINITY):
1747
    maxfd = MAXFD
1748

    
1749
  # Iterate through and close all file descriptors (except the standard ones)
1750
  for fd in range(3, maxfd):
1751
    if noclose_fds and fd in noclose_fds:
1752
      continue
1753
    _CloseFDNoErr(fd)
1754

    
1755

    
1756
def Mlockall():
1757
  """Lock current process' virtual address space into RAM.
1758

1759
  This is equivalent to the C call mlockall(MCL_CURRENT|MCL_FUTURE),
1760
  see mlock(2) for more details. This function requires ctypes module.
1761

1762
  """
1763
  if ctypes is None:
1764
    logging.warning("Cannot set memory lock, ctypes module not found")
1765
    return
1766

    
1767
  libc = ctypes.cdll.LoadLibrary("libc.so.6")
1768
  if libc is None:
1769
    logging.error("Cannot set memory lock, ctypes cannot load libc")
1770
    return
1771

    
1772
  # Some older version of the ctypes module don't have built-in functionality
1773
  # to access the errno global variable, where function error codes are stored.
1774
  # By declaring this variable as a pointer to an integer we can then access
1775
  # its value correctly, should the mlockall call fail, in order to see what
1776
  # the actual error code was.
1777
  # pylint: disable-msg=W0212
1778
  libc.__errno_location.restype = ctypes.POINTER(ctypes.c_int)
1779

    
1780
  if libc.mlockall(_MCL_CURRENT | _MCL_FUTURE):
1781
    # pylint: disable-msg=W0212
1782
    logging.error("Cannot set memory lock: %s",
1783
                  os.strerror(libc.__errno_location().contents.value))
1784
    return
1785

    
1786
  logging.debug("Memory lock set")
1787

    
1788

    
1789
def Daemonize(logfile):
1790
  """Daemonize the current process.
1791

1792
  This detaches the current process from the controlling terminal and
1793
  runs it in the background as a daemon.
1794

1795
  @type logfile: str
1796
  @param logfile: the logfile to which we should redirect stdout/stderr
1797
  @rtype: int
1798
  @return: the value zero
1799

1800
  """
1801
  # pylint: disable-msg=W0212
1802
  # yes, we really want os._exit
1803
  UMASK = 077
1804
  WORKDIR = "/"
1805

    
1806
  # this might fail
1807
  pid = os.fork()
1808
  if (pid == 0):  # The first child.
1809
    os.setsid()
1810
    # this might fail
1811
    pid = os.fork() # Fork a second child.
1812
    if (pid == 0):  # The second child.
1813
      os.chdir(WORKDIR)
1814
      os.umask(UMASK)
1815
    else:
1816
      # exit() or _exit()?  See below.
1817
      os._exit(0) # Exit parent (the first child) of the second child.
1818
  else:
1819
    os._exit(0) # Exit parent of the first child.
1820

    
1821
  for fd in range(3):
1822
    _CloseFDNoErr(fd)
1823
  i = os.open("/dev/null", os.O_RDONLY) # stdin
1824
  assert i == 0, "Can't close/reopen stdin"
1825
  i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1826
  assert i == 1, "Can't close/reopen stdout"
1827
  # Duplicate standard output to standard error.
1828
  os.dup2(1, 2)
1829
  return 0
1830

    
1831

    
1832
def DaemonPidFileName(name):
1833
  """Compute a ganeti pid file absolute path
1834

1835
  @type name: str
1836
  @param name: the daemon name
1837
  @rtype: str
1838
  @return: the full path to the pidfile corresponding to the given
1839
      daemon name
1840

1841
  """
1842
  return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
1843

    
1844

    
1845
def EnsureDaemon(name):
1846
  """Check for and start daemon if not alive.
1847

1848
  """
1849
  result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
1850
  if result.failed:
1851
    logging.error("Can't start daemon '%s', failure %s, output: %s",
1852
                  name, result.fail_reason, result.output)
1853
    return False
1854

    
1855
  return True
1856

    
1857

    
1858
def WritePidFile(name):
1859
  """Write the current process pidfile.
1860

1861
  The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1862

1863
  @type name: str
1864
  @param name: the daemon name to use
1865
  @raise errors.GenericError: if the pid file already exists and
1866
      points to a live process
1867

1868
  """
1869
  pid = os.getpid()
1870
  pidfilename = DaemonPidFileName(name)
1871
  if IsProcessAlive(ReadPidFile(pidfilename)):
1872
    raise errors.GenericError("%s contains a live process" % pidfilename)
1873

    
1874
  WriteFile(pidfilename, data="%d\n" % pid)
1875

    
1876

    
1877
def RemovePidFile(name):
1878
  """Remove the current process pidfile.
1879

1880
  Any errors are ignored.
1881

1882
  @type name: str
1883
  @param name: the daemon name used to derive the pidfile name
1884

1885
  """
1886
  pidfilename = DaemonPidFileName(name)
1887
  # TODO: we could check here that the file contains our pid
1888
  try:
1889
    RemoveFile(pidfilename)
1890
  except: # pylint: disable-msg=W0702
1891
    pass
1892

    
1893

    
1894
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1895
                waitpid=False):
1896
  """Kill a process given by its pid.
1897

1898
  @type pid: int
1899
  @param pid: The PID to terminate.
1900
  @type signal_: int
1901
  @param signal_: The signal to send, by default SIGTERM
1902
  @type timeout: int
1903
  @param timeout: The timeout after which, if the process is still alive,
1904
                  a SIGKILL will be sent. If not positive, no such checking
1905
                  will be done
1906
  @type waitpid: boolean
1907
  @param waitpid: If true, we should waitpid on this process after
1908
      sending signals, since it's our own child and otherwise it
1909
      would remain as zombie
1910

1911
  """
1912
  def _helper(pid, signal_, wait):
1913
    """Simple helper to encapsulate the kill/waitpid sequence"""
1914
    os.kill(pid, signal_)
1915
    if wait:
1916
      try:
1917
        os.waitpid(pid, os.WNOHANG)
1918
      except OSError:
1919
        pass
1920

    
1921
  if pid <= 0:
1922
    # kill with pid=0 == suicide
1923
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1924

    
1925
  if not IsProcessAlive(pid):
1926
    return
1927

    
1928
  _helper(pid, signal_, waitpid)
1929

    
1930
  if timeout <= 0:
1931
    return
1932

    
1933
  def _CheckProcess():
1934
    if not IsProcessAlive(pid):
1935
      return
1936

    
1937
    try:
1938
      (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1939
    except OSError:
1940
      raise RetryAgain()
1941

    
1942
    if result_pid > 0:
1943
      return
1944

    
1945
    raise RetryAgain()
1946

    
1947
  try:
1948
    # Wait up to $timeout seconds
1949
    Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1950
  except RetryTimeout:
1951
    pass
1952

    
1953
  if IsProcessAlive(pid):
1954
    # Kill process if it's still alive
1955
    _helper(pid, signal.SIGKILL, waitpid)
1956

    
1957

    
1958
def FindFile(name, search_path, test=os.path.exists):
1959
  """Look for a filesystem object in a given path.
1960

1961
  This is an abstract method to search for filesystem object (files,
1962
  dirs) under a given search path.
1963

1964
  @type name: str
1965
  @param name: the name to look for
1966
  @type search_path: str
1967
  @param search_path: location to start at
1968
  @type test: callable
1969
  @param test: a function taking one argument that should return True
1970
      if the a given object is valid; the default value is
1971
      os.path.exists, causing only existing files to be returned
1972
  @rtype: str or None
1973
  @return: full path to the object if found, None otherwise
1974

1975
  """
1976
  # validate the filename mask
1977
  if constants.EXT_PLUGIN_MASK.match(name) is None:
1978
    logging.critical("Invalid value passed for external script name: '%s'",
1979
                     name)
1980
    return None
1981

    
1982
  for dir_name in search_path:
1983
    # FIXME: investigate switch to PathJoin
1984
    item_name = os.path.sep.join([dir_name, name])
1985
    # check the user test and that we're indeed resolving to the given
1986
    # basename
1987
    if test(item_name) and os.path.basename(item_name) == name:
1988
      return item_name
1989
  return None
1990

    
1991

    
1992
def CheckVolumeGroupSize(vglist, vgname, minsize):
1993
  """Checks if the volume group list is valid.
1994

1995
  The function will check if a given volume group is in the list of
1996
  volume groups and has a minimum size.
1997

1998
  @type vglist: dict
1999
  @param vglist: dictionary of volume group names and their size
2000
  @type vgname: str
2001
  @param vgname: the volume group we should check
2002
  @type minsize: int
2003
  @param minsize: the minimum size we accept
2004
  @rtype: None or str
2005
  @return: None for success, otherwise the error message
2006

2007
  """
2008
  vgsize = vglist.get(vgname, None)
2009
  if vgsize is None:
2010
    return "volume group '%s' missing" % vgname
2011
  elif vgsize < minsize:
2012
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
2013
            (vgname, minsize, vgsize))
2014
  return None
2015

    
2016

    
2017
def SplitTime(value):
2018
  """Splits time as floating point number into a tuple.
2019

2020
  @param value: Time in seconds
2021
  @type value: int or float
2022
  @return: Tuple containing (seconds, microseconds)
2023

2024
  """
2025
  (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
2026

    
2027
  assert 0 <= seconds, \
2028
    "Seconds must be larger than or equal to 0, but are %s" % seconds
2029
  assert 0 <= microseconds <= 999999, \
2030
    "Microseconds must be 0-999999, but are %s" % microseconds
2031

    
2032
  return (int(seconds), int(microseconds))
2033

    
2034

    
2035
def MergeTime(timetuple):
2036
  """Merges a tuple into time as a floating point number.
2037

2038
  @param timetuple: Time as tuple, (seconds, microseconds)
2039
  @type timetuple: tuple
2040
  @return: Time as a floating point number expressed in seconds
2041

2042
  """
2043
  (seconds, microseconds) = timetuple
2044

    
2045
  assert 0 <= seconds, \
2046
    "Seconds must be larger than or equal to 0, but are %s" % seconds
2047
  assert 0 <= microseconds <= 999999, \
2048
    "Microseconds must be 0-999999, but are %s" % microseconds
2049

    
2050
  return float(seconds) + (float(microseconds) * 0.000001)
2051

    
2052

    
2053
def GetDaemonPort(daemon_name):
2054
  """Get the daemon port for this cluster.
2055

2056
  Note that this routine does not read a ganeti-specific file, but
2057
  instead uses C{socket.getservbyname} to allow pre-customization of
2058
  this parameter outside of Ganeti.
2059

2060
  @type daemon_name: string
2061
  @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
2062
  @rtype: int
2063

2064
  """
2065
  if daemon_name not in constants.DAEMONS_PORTS:
2066
    raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
2067

    
2068
  (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
2069
  try:
2070
    port = socket.getservbyname(daemon_name, proto)
2071
  except socket.error:
2072
    port = default_port
2073

    
2074
  return port
2075

    
2076

    
2077
class LogFileHandler(logging.FileHandler):
2078
  """Log handler that doesn't fallback to stderr.
2079

2080
  When an error occurs while writing on the logfile, logging.FileHandler tries
2081
  to log on stderr. This doesn't work in ganeti since stderr is redirected to
2082
  the logfile. This class avoids failures reporting errors to /dev/console.
2083

2084
  """
2085
  def __init__(self, filename, mode="a", encoding=None):
2086
    """Open the specified file and use it as the stream for logging.
2087

2088
    Also open /dev/console to report errors while logging.
2089

2090
    """
2091
    logging.FileHandler.__init__(self, filename, mode, encoding)
2092
    self.console = open(constants.DEV_CONSOLE, "a")
2093

    
2094
  def handleError(self, record): # pylint: disable-msg=C0103
2095
    """Handle errors which occur during an emit() call.
2096

2097
    Try to handle errors with FileHandler method, if it fails write to
2098
    /dev/console.
2099

2100
    """
2101
    try:
2102
      logging.FileHandler.handleError(self, record)
2103
    except Exception: # pylint: disable-msg=W0703
2104
      try:
2105
        self.console.write("Cannot log message:\n%s\n" % self.format(record))
2106
      except Exception: # pylint: disable-msg=W0703
2107
        # Log handler tried everything it could, now just give up
2108
        pass
2109

    
2110

    
2111
def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
2112
                 multithreaded=False, syslog=constants.SYSLOG_USAGE,
2113
                 console_logging=False):
2114
  """Configures the logging module.
2115

2116
  @type logfile: str
2117
  @param logfile: the filename to which we should log
2118
  @type debug: integer
2119
  @param debug: if greater than zero, enable debug messages, otherwise
2120
      only those at C{INFO} and above level
2121
  @type stderr_logging: boolean
2122
  @param stderr_logging: whether we should also log to the standard error
2123
  @type program: str
2124
  @param program: the name under which we should log messages
2125
  @type multithreaded: boolean
2126
  @param multithreaded: if True, will add the thread name to the log file
2127
  @type syslog: string
2128
  @param syslog: one of 'no', 'yes', 'only':
2129
      - if no, syslog is not used
2130
      - if yes, syslog is used (in addition to file-logging)
2131
      - if only, only syslog is used
2132
  @type console_logging: boolean
2133
  @param console_logging: if True, will use a FileHandler which falls back to
2134
      the system console if logging fails
2135
  @raise EnvironmentError: if we can't open the log file and
2136
      syslog/stderr logging is disabled
2137

2138
  """
2139
  fmt = "%(asctime)s: " + program + " pid=%(process)d"
2140
  sft = program + "[%(process)d]:"
2141
  if multithreaded:
2142
    fmt += "/%(threadName)s"
2143
    sft += " (%(threadName)s)"
2144
  if debug:
2145
    fmt += " %(module)s:%(lineno)s"
2146
    # no debug info for syslog loggers
2147
  fmt += " %(levelname)s %(message)s"
2148
  # yes, we do want the textual level, as remote syslog will probably
2149
  # lose the error level, and it's easier to grep for it
2150
  sft += " %(levelname)s %(message)s"
2151
  formatter = logging.Formatter(fmt)
2152
  sys_fmt = logging.Formatter(sft)
2153

    
2154
  root_logger = logging.getLogger("")
2155
  root_logger.setLevel(logging.NOTSET)
2156

    
2157
  # Remove all previously setup handlers
2158
  for handler in root_logger.handlers:
2159
    handler.close()
2160
    root_logger.removeHandler(handler)
2161

    
2162
  if stderr_logging:
2163
    stderr_handler = logging.StreamHandler()
2164
    stderr_handler.setFormatter(formatter)
2165
    if debug:
2166
      stderr_handler.setLevel(logging.NOTSET)
2167
    else:
2168
      stderr_handler.setLevel(logging.CRITICAL)
2169
    root_logger.addHandler(stderr_handler)
2170

    
2171
  if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
2172
    facility = logging.handlers.SysLogHandler.LOG_DAEMON
2173
    syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
2174
                                                    facility)
2175
    syslog_handler.setFormatter(sys_fmt)
2176
    # Never enable debug over syslog
2177
    syslog_handler.setLevel(logging.INFO)
2178
    root_logger.addHandler(syslog_handler)
2179

    
2180
  if syslog != constants.SYSLOG_ONLY:
2181
    # this can fail, if the logging directories are not setup or we have
2182
    # a permisssion problem; in this case, it's best to log but ignore
2183
    # the error if stderr_logging is True, and if false we re-raise the
2184
    # exception since otherwise we could run but without any logs at all
2185
    try:
2186
      if console_logging:
2187
        logfile_handler = LogFileHandler(logfile)
2188
      else:
2189
        logfile_handler = logging.FileHandler(logfile)
2190
      logfile_handler.setFormatter(formatter)
2191
      if debug:
2192
        logfile_handler.setLevel(logging.DEBUG)
2193
      else:
2194
        logfile_handler.setLevel(logging.INFO)
2195
      root_logger.addHandler(logfile_handler)
2196
    except EnvironmentError:
2197
      if stderr_logging or syslog == constants.SYSLOG_YES:
2198
        logging.exception("Failed to enable logging to file '%s'", logfile)
2199
      else:
2200
        # we need to re-raise the exception
2201
        raise
2202

    
2203

    
2204
def IsNormAbsPath(path):
2205
  """Check whether a path is absolute and also normalized
2206

2207
  This avoids things like /dir/../../other/path to be valid.
2208

2209
  """
2210
  return os.path.normpath(path) == path and os.path.isabs(path)
2211

    
2212

    
2213
def PathJoin(*args):
2214
  """Safe-join a list of path components.
2215

2216
  Requirements:
2217
      - the first argument must be an absolute path
2218
      - no component in the path must have backtracking (e.g. /../),
2219
        since we check for normalization at the end
2220

2221
  @param args: the path components to be joined
2222
  @raise ValueError: for invalid paths
2223

2224
  """
2225
  # ensure we're having at least one path passed in
2226
  assert args
2227
  # ensure the first component is an absolute and normalized path name
2228
  root = args[0]
2229
  if not IsNormAbsPath(root):
2230
    raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
2231
  result = os.path.join(*args)
2232
  # ensure that the whole path is normalized
2233
  if not IsNormAbsPath(result):
2234
    raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
2235
  # check that we're still under the original prefix
2236
  prefix = os.path.commonprefix([root, result])
2237
  if prefix != root:
2238
    raise ValueError("Error: path joining resulted in different prefix"
2239
                     " (%s != %s)" % (prefix, root))
2240
  return result
2241

    
2242

    
2243
def TailFile(fname, lines=20):
2244
  """Return the last lines from a file.
2245

2246
  @note: this function will only read and parse the last 4KB of
2247
      the file; if the lines are very long, it could be that less
2248
      than the requested number of lines are returned
2249

2250
  @param fname: the file name
2251
  @type lines: int
2252
  @param lines: the (maximum) number of lines to return
2253

2254
  """
2255
  fd = open(fname, "r")
2256
  try:
2257
    fd.seek(0, 2)
2258
    pos = fd.tell()
2259
    pos = max(0, pos-4096)
2260
    fd.seek(pos, 0)
2261
    raw_data = fd.read()
2262
  finally:
2263
    fd.close()
2264

    
2265
  rows = raw_data.splitlines()
2266
  return rows[-lines:]
2267

    
2268

    
2269
def _ParseAsn1Generalizedtime(value):
2270
  """Parses an ASN1 GENERALIZEDTIME timestamp as used by pyOpenSSL.
2271

2272
  @type value: string
2273
  @param value: ASN1 GENERALIZEDTIME timestamp
2274

2275
  """
2276
  m = re.match(r"^(\d+)([-+]\d\d)(\d\d)$", value)
2277
  if m:
2278
    # We have an offset
2279
    asn1time = m.group(1)
2280
    hours = int(m.group(2))
2281
    minutes = int(m.group(3))
2282
    utcoffset = (60 * hours) + minutes
2283
  else:
2284
    if not value.endswith("Z"):
2285
      raise ValueError("Missing timezone")
2286
    asn1time = value[:-1]
2287
    utcoffset = 0
2288

    
2289
  parsed = time.strptime(asn1time, "%Y%m%d%H%M%S")
2290

    
2291
  tt = datetime.datetime(*(parsed[:7])) - datetime.timedelta(minutes=utcoffset)
2292

    
2293
  return calendar.timegm(tt.utctimetuple())
2294

    
2295

    
2296
def GetX509CertValidity(cert):
2297
  """Returns the validity period of the certificate.
2298

2299
  @type cert: OpenSSL.crypto.X509
2300
  @param cert: X509 certificate object
2301

2302
  """
2303
  # The get_notBefore and get_notAfter functions are only supported in
2304
  # pyOpenSSL 0.7 and above.
2305
  try:
2306
    get_notbefore_fn = cert.get_notBefore
2307
  except AttributeError:
2308
    not_before = None
2309
  else:
2310
    not_before_asn1 = get_notbefore_fn()
2311

    
2312
    if not_before_asn1 is None:
2313
      not_before = None
2314
    else:
2315
      not_before = _ParseAsn1Generalizedtime(not_before_asn1)
2316

    
2317
  try:
2318
    get_notafter_fn = cert.get_notAfter
2319
  except AttributeError:
2320
    not_after = None
2321
  else:
2322
    not_after_asn1 = get_notafter_fn()
2323

    
2324
    if not_after_asn1 is None:
2325
      not_after = None
2326
    else:
2327
      not_after = _ParseAsn1Generalizedtime(not_after_asn1)
2328

    
2329
  return (not_before, not_after)
2330

    
2331

    
2332
def SafeEncode(text):
2333
  """Return a 'safe' version of a source string.
2334

2335
  This function mangles the input string and returns a version that
2336
  should be safe to display/encode as ASCII. To this end, we first
2337
  convert it to ASCII using the 'backslashreplace' encoding which
2338
  should get rid of any non-ASCII chars, and then we process it
2339
  through a loop copied from the string repr sources in the python; we
2340
  don't use string_escape anymore since that escape single quotes and
2341
  backslashes too, and that is too much; and that escaping is not
2342
  stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2343

2344
  @type text: str or unicode
2345
  @param text: input data
2346
  @rtype: str
2347
  @return: a safe version of text
2348

2349
  """
2350
  if isinstance(text, unicode):
2351
    # only if unicode; if str already, we handle it below
2352
    text = text.encode('ascii', 'backslashreplace')
2353
  resu = ""
2354
  for char in text:
2355
    c = ord(char)
2356
    if char  == '\t':
2357
      resu += r'\t'
2358
    elif char == '\n':
2359
      resu += r'\n'
2360
    elif char == '\r':
2361
      resu += r'\'r'
2362
    elif c < 32 or c >= 127: # non-printable
2363
      resu += "\\x%02x" % (c & 0xff)
2364
    else:
2365
      resu += char
2366
  return resu
2367

    
2368

    
2369
def UnescapeAndSplit(text, sep=","):
2370
  """Split and unescape a string based on a given separator.
2371

2372
  This function splits a string based on a separator where the
2373
  separator itself can be escape in order to be an element of the
2374
  elements. The escaping rules are (assuming coma being the
2375
  separator):
2376
    - a plain , separates the elements
2377
    - a sequence \\\\, (double backslash plus comma) is handled as a
2378
      backslash plus a separator comma
2379
    - a sequence \, (backslash plus comma) is handled as a
2380
      non-separator comma
2381

2382
  @type text: string
2383
  @param text: the string to split
2384
  @type sep: string
2385
  @param text: the separator
2386
  @rtype: string
2387
  @return: a list of strings
2388

2389
  """
2390
  # we split the list by sep (with no escaping at this stage)
2391
  slist = text.split(sep)
2392
  # next, we revisit the elements and if any of them ended with an odd
2393
  # number of backslashes, then we join it with the next
2394
  rlist = []
2395
  while slist:
2396
    e1 = slist.pop(0)
2397
    if e1.endswith("\\"):
2398
      num_b = len(e1) - len(e1.rstrip("\\"))
2399
      if num_b % 2 == 1:
2400
        e2 = slist.pop(0)
2401
        # here the backslashes remain (all), and will be reduced in
2402
        # the next step
2403
        rlist.append(e1 + sep + e2)
2404
        continue
2405
    rlist.append(e1)
2406
  # finally, replace backslash-something with something
2407
  rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2408
  return rlist
2409

    
2410

    
2411
def CommaJoin(names):
2412
  """Nicely join a set of identifiers.
2413

2414
  @param names: set, list or tuple
2415
  @return: a string with the formatted results
2416

2417
  """
2418
  return ", ".join([str(val) for val in names])
2419

    
2420

    
2421
def BytesToMebibyte(value):
2422
  """Converts bytes to mebibytes.
2423

2424
  @type value: int
2425
  @param value: Value in bytes
2426
  @rtype: int
2427
  @return: Value in mebibytes
2428

2429
  """
2430
  return int(round(value / (1024.0 * 1024.0), 0))
2431

    
2432

    
2433
def CalculateDirectorySize(path):
2434
  """Calculates the size of a directory recursively.
2435

2436
  @type path: string
2437
  @param path: Path to directory
2438
  @rtype: int
2439
  @return: Size in mebibytes
2440

2441
  """
2442
  size = 0
2443

    
2444
  for (curpath, _, files) in os.walk(path):
2445
    for filename in files:
2446
      st = os.lstat(PathJoin(curpath, filename))
2447
      size += st.st_size
2448

    
2449
  return BytesToMebibyte(size)
2450

    
2451

    
2452
def GetFilesystemStats(path):
2453
  """Returns the total and free space on a filesystem.
2454

2455
  @type path: string
2456
  @param path: Path on filesystem to be examined
2457
  @rtype: int
2458
  @return: tuple of (Total space, Free space) in mebibytes
2459

2460
  """
2461
  st = os.statvfs(path)
2462

    
2463
  fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2464
  tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2465
  return (tsize, fsize)
2466

    
2467

    
2468
def RunInSeparateProcess(fn, *args):
2469
  """Runs a function in a separate process.
2470

2471
  Note: Only boolean return values are supported.
2472

2473
  @type fn: callable
2474
  @param fn: Function to be called
2475
  @rtype: bool
2476
  @return: Function's result
2477

2478
  """
2479
  pid = os.fork()
2480
  if pid == 0:
2481
    # Child process
2482
    try:
2483
      # In case the function uses temporary files
2484
      ResetTempfileModule()
2485

    
2486
      # Call function
2487
      result = int(bool(fn(*args)))
2488
      assert result in (0, 1)
2489
    except: # pylint: disable-msg=W0702
2490
      logging.exception("Error while calling function in separate process")
2491
      # 0 and 1 are reserved for the return value
2492
      result = 33
2493

    
2494
    os._exit(result) # pylint: disable-msg=W0212
2495

    
2496
  # Parent process
2497

    
2498
  # Avoid zombies and check exit code
2499
  (_, status) = os.waitpid(pid, 0)
2500

    
2501
  if os.WIFSIGNALED(status):
2502
    exitcode = None
2503
    signum = os.WTERMSIG(status)
2504
  else:
2505
    exitcode = os.WEXITSTATUS(status)
2506
    signum = None
2507

    
2508
  if not (exitcode in (0, 1) and signum is None):
2509
    raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2510
                              (exitcode, signum))
2511

    
2512
  return bool(exitcode)
2513

    
2514

    
2515
def IgnoreSignals(fn, *args, **kwargs):
2516
  """Tries to call a function ignoring failures due to EINTR.
2517

2518
  """
2519
  try:
2520
    return fn(*args, **kwargs)
2521
  except EnvironmentError, err:
2522
    if err.errno != errno.EINTR:
2523
      raise
2524
  except (select.error, socket.error), err:
2525
    # In python 2.6 and above select.error is an IOError, so it's handled
2526
    # above, in 2.5 and below it's not, and it's handled here.
2527
    if not (err.args and err.args[0] == errno.EINTR):
2528
      raise
2529

    
2530

    
2531
def LockedMethod(fn):
2532
  """Synchronized object access decorator.
2533

2534
  This decorator is intended to protect access to an object using the
2535
  object's own lock which is hardcoded to '_lock'.
2536

2537
  """
2538
  def _LockDebug(*args, **kwargs):
2539
    if debug_locks:
2540
      logging.debug(*args, **kwargs)
2541

    
2542
  def wrapper(self, *args, **kwargs):
2543
    # pylint: disable-msg=W0212
2544
    assert hasattr(self, '_lock')
2545
    lock = self._lock
2546
    _LockDebug("Waiting for %s", lock)
2547
    lock.acquire()
2548
    try:
2549
      _LockDebug("Acquired %s", lock)
2550
      result = fn(self, *args, **kwargs)
2551
    finally:
2552
      _LockDebug("Releasing %s", lock)
2553
      lock.release()
2554
      _LockDebug("Released %s", lock)
2555
    return result
2556
  return wrapper
2557

    
2558

    
2559
def LockFile(fd):
2560
  """Locks a file using POSIX locks.
2561

2562
  @type fd: int
2563
  @param fd: the file descriptor we need to lock
2564

2565
  """
2566
  try:
2567
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2568
  except IOError, err:
2569
    if err.errno == errno.EAGAIN:
2570
      raise errors.LockError("File already locked")
2571
    raise
2572

    
2573

    
2574
def FormatTime(val):
2575
  """Formats a time value.
2576

2577
  @type val: float or None
2578
  @param val: the timestamp as returned by time.time()
2579
  @return: a string value or N/A if we don't have a valid timestamp
2580

2581
  """
2582
  if val is None or not isinstance(val, (int, float)):
2583
    return "N/A"
2584
  # these two codes works on Linux, but they are not guaranteed on all
2585
  # platforms
2586
  return time.strftime("%F %T", time.localtime(val))
2587

    
2588

    
2589
def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2590
  """Reads the watcher pause file.
2591

2592
  @type filename: string
2593
  @param filename: Path to watcher pause file
2594
  @type now: None, float or int
2595
  @param now: Current time as Unix timestamp
2596
  @type remove_after: int
2597
  @param remove_after: Remove watcher pause file after specified amount of
2598
    seconds past the pause end time
2599

2600
  """
2601
  if now is None:
2602
    now = time.time()
2603

    
2604
  try:
2605
    value = ReadFile(filename)
2606
  except IOError, err:
2607
    if err.errno != errno.ENOENT:
2608
      raise
2609
    value = None
2610

    
2611
  if value is not None:
2612
    try:
2613
      value = int(value)
2614
    except ValueError:
2615
      logging.warning(("Watcher pause file (%s) contains invalid value,"
2616
                       " removing it"), filename)
2617
      RemoveFile(filename)
2618
      value = None
2619

    
2620
    if value is not None:
2621
      # Remove file if it's outdated
2622
      if now > (value + remove_after):
2623
        RemoveFile(filename)
2624
        value = None
2625

    
2626
      elif now > value:
2627
        value = None
2628

    
2629
  return value
2630

    
2631

    
2632
class RetryTimeout(Exception):
2633
  """Retry loop timed out.
2634

2635
  Any arguments which was passed by the retried function to RetryAgain will be
2636
  preserved in RetryTimeout, if it is raised. If such argument was an exception
2637
  the RaiseInner helper method will reraise it.
2638

2639
  """
2640
  def RaiseInner(self):
2641
    if self.args and isinstance(self.args[0], Exception):
2642
      raise self.args[0]
2643
    else:
2644
      raise RetryTimeout(*self.args)
2645

    
2646

    
2647
class RetryAgain(Exception):
2648
  """Retry again.
2649

2650
  Any arguments passed to RetryAgain will be preserved, if a timeout occurs, as
2651
  arguments to RetryTimeout. If an exception is passed, the RaiseInner() method
2652
  of the RetryTimeout() method can be used to reraise it.
2653

2654
  """
2655

    
2656

    
2657
class _RetryDelayCalculator(object):
2658
  """Calculator for increasing delays.
2659

2660
  """
2661
  __slots__ = [
2662
    "_factor",
2663
    "_limit",
2664
    "_next",
2665
    "_start",
2666
    ]
2667

    
2668
  def __init__(self, start, factor, limit):
2669
    """Initializes this class.
2670

2671
    @type start: float
2672
    @param start: Initial delay
2673
    @type factor: float
2674
    @param factor: Factor for delay increase
2675
    @type limit: float or None
2676
    @param limit: Upper limit for delay or None for no limit
2677

2678
    """
2679
    assert start > 0.0
2680
    assert factor >= 1.0
2681
    assert limit is None or limit >= 0.0
2682

    
2683
    self._start = start
2684
    self._factor = factor
2685
    self._limit = limit
2686

    
2687
    self._next = start
2688

    
2689
  def __call__(self):
2690
    """Returns current delay and calculates the next one.
2691

2692
    """
2693
    current = self._next
2694

    
2695
    # Update for next run
2696
    if self._limit is None or self._next < self._limit:
2697
      self._next = min(self._limit, self._next * self._factor)
2698

    
2699
    return current
2700

    
2701

    
2702
#: Special delay to specify whole remaining timeout
2703
RETRY_REMAINING_TIME = object()
2704

    
2705

    
2706
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2707
          _time_fn=time.time):
2708
  """Call a function repeatedly until it succeeds.
2709

2710
  The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2711
  anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2712
  total of C{timeout} seconds, this function throws L{RetryTimeout}.
2713

2714
  C{delay} can be one of the following:
2715
    - callable returning the delay length as a float
2716
    - Tuple of (start, factor, limit)
2717
    - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2718
      useful when overriding L{wait_fn} to wait for an external event)
2719
    - A static delay as a number (int or float)
2720

2721
  @type fn: callable
2722
  @param fn: Function to be called
2723
  @param delay: Either a callable (returning the delay), a tuple of (start,
2724
                factor, limit) (see L{_RetryDelayCalculator}),
2725
                L{RETRY_REMAINING_TIME} or a number (int or float)
2726
  @type timeout: float
2727
  @param timeout: Total timeout
2728
  @type wait_fn: callable
2729
  @param wait_fn: Waiting function
2730
  @return: Return value of function
2731

2732
  """
2733
  assert callable(fn)
2734
  assert callable(wait_fn)
2735
  assert callable(_time_fn)
2736

    
2737
  if args is None:
2738
    args = []
2739

    
2740
  end_time = _time_fn() + timeout
2741

    
2742
  if callable(delay):
2743
    # External function to calculate delay
2744
    calc_delay = delay
2745

    
2746
  elif isinstance(delay, (tuple, list)):
2747
    # Increasing delay with optional upper boundary
2748
    (start, factor, limit) = delay
2749
    calc_delay = _RetryDelayCalculator(start, factor, limit)
2750

    
2751
  elif delay is RETRY_REMAINING_TIME:
2752
    # Always use the remaining time
2753
    calc_delay = None
2754

    
2755
  else:
2756
    # Static delay
2757
    calc_delay = lambda: delay
2758

    
2759
  assert calc_delay is None or callable(calc_delay)
2760

    
2761
  while True:
2762
    retry_args = []
2763
    try:
2764
      # pylint: disable-msg=W0142
2765
      return fn(*args)
2766
    except RetryAgain, err:
2767
      retry_args = err.args
2768
    except RetryTimeout:
2769
      raise errors.ProgrammerError("Nested retry loop detected that didn't"
2770
                                   " handle RetryTimeout")
2771

    
2772
    remaining_time = end_time - _time_fn()
2773

    
2774
    if remaining_time < 0.0:
2775
      # pylint: disable-msg=W0142
2776
      raise RetryTimeout(*retry_args)
2777

    
2778
    assert remaining_time >= 0.0
2779

    
2780
    if calc_delay is None:
2781
      wait_fn(remaining_time)
2782
    else:
2783
      current_delay = calc_delay()
2784
      if current_delay > 0.0:
2785
        wait_fn(current_delay)
2786

    
2787

    
2788
class FileLock(object):
2789
  """Utility class for file locks.
2790

2791
  """
2792
  def __init__(self, fd, filename):
2793
    """Constructor for FileLock.
2794

2795
    @type fd: file
2796
    @param fd: File object
2797
    @type filename: str
2798
    @param filename: Path of the file opened at I{fd}
2799

2800
    """
2801
    self.fd = fd
2802
    self.filename = filename
2803

    
2804
  @classmethod
2805
  def Open(cls, filename):
2806
    """Creates and opens a file to be used as a file-based lock.
2807

2808
    @type filename: string
2809
    @param filename: path to the file to be locked
2810

2811
    """
2812
    # Using "os.open" is necessary to allow both opening existing file
2813
    # read/write and creating if not existing. Vanilla "open" will truncate an
2814
    # existing file -or- allow creating if not existing.
2815
    return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
2816
               filename)
2817

    
2818
  def __del__(self):
2819
    self.Close()
2820

    
2821
  def Close(self):
2822
    """Close the file and release the lock.
2823

2824
    """
2825
    if hasattr(self, "fd") and self.fd:
2826
      self.fd.close()
2827
      self.fd = None
2828

    
2829
  def _flock(self, flag, blocking, timeout, errmsg):
2830
    """Wrapper for fcntl.flock.
2831

2832
    @type flag: int
2833
    @param flag: operation flag
2834
    @type blocking: bool
2835
    @param blocking: whether the operation should be done in blocking mode.
2836
    @type timeout: None or float
2837
    @param timeout: for how long the operation should be retried (implies
2838
                    non-blocking mode).
2839
    @type errmsg: string
2840
    @param errmsg: error message in case operation fails.
2841

2842
    """
2843
    assert self.fd, "Lock was closed"
2844
    assert timeout is None or timeout >= 0, \
2845
      "If specified, timeout must be positive"
2846
    assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
2847

    
2848
    # When a timeout is used, LOCK_NB must always be set
2849
    if not (timeout is None and blocking):
2850
      flag |= fcntl.LOCK_NB
2851

    
2852
    if timeout is None:
2853
      self._Lock(self.fd, flag, timeout)
2854
    else:
2855
      try:
2856
        Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
2857
              args=(self.fd, flag, timeout))
2858
      except RetryTimeout:
2859
        raise errors.LockError(errmsg)
2860

    
2861
  @staticmethod
2862
  def _Lock(fd, flag, timeout):
2863
    try:
2864
      fcntl.flock(fd, flag)
2865
    except IOError, err:
2866
      if timeout is not None and err.errno == errno.EAGAIN:
2867
        raise RetryAgain()
2868

    
2869
      logging.exception("fcntl.flock failed")
2870
      raise
2871

    
2872
  def Exclusive(self, blocking=False, timeout=None):
2873
    """Locks the file in exclusive mode.
2874

2875
    @type blocking: boolean
2876
    @param blocking: whether to block and wait until we
2877
        can lock the file or return immediately
2878
    @type timeout: int or None
2879
    @param timeout: if not None, the duration to wait for the lock
2880
        (in blocking mode)
2881

2882
    """
2883
    self._flock(fcntl.LOCK_EX, blocking, timeout,
2884
                "Failed to lock %s in exclusive mode" % self.filename)
2885

    
2886
  def Shared(self, blocking=False, timeout=None):
2887
    """Locks the file in shared mode.
2888

2889
    @type blocking: boolean
2890
    @param blocking: whether to block and wait until we
2891
        can lock the file or return immediately
2892
    @type timeout: int or None
2893
    @param timeout: if not None, the duration to wait for the lock
2894
        (in blocking mode)
2895

2896
    """
2897
    self._flock(fcntl.LOCK_SH, blocking, timeout,
2898
                "Failed to lock %s in shared mode" % self.filename)
2899

    
2900
  def Unlock(self, blocking=True, timeout=None):
2901
    """Unlocks the file.
2902

2903
    According to C{flock(2)}, unlocking can also be a nonblocking
2904
    operation::
2905

2906
      To make a non-blocking request, include LOCK_NB with any of the above
2907
      operations.
2908

2909
    @type blocking: boolean
2910
    @param blocking: whether to block and wait until we
2911
        can lock the file or return immediately
2912
    @type timeout: int or None
2913
    @param timeout: if not None, the duration to wait for the lock
2914
        (in blocking mode)
2915

2916
    """
2917
    self._flock(fcntl.LOCK_UN, blocking, timeout,
2918
                "Failed to unlock %s" % self.filename)
2919

    
2920

    
2921
class LineSplitter:
2922
  """Splits data chunks into lines separated by newline.
2923

2924
  Instances provide a file-like interface.
2925

2926
  """
2927
  def __init__(self, line_fn, *args):
2928
    """Initializes this class.
2929

2930
    @type line_fn: callable
2931
    @param line_fn: Function called for each line, first parameter is line
2932
    @param args: Extra arguments for L{line_fn}
2933

2934
    """
2935
    assert callable(line_fn)
2936

    
2937
    if args:
2938
      # Python 2.4 doesn't have functools.partial yet
2939
      self._line_fn = \
2940
        lambda line: line_fn(line, *args) # pylint: disable-msg=W0142
2941
    else:
2942
      self._line_fn = line_fn
2943

    
2944
    self._lines = collections.deque()
2945
    self._buffer = ""
2946

    
2947
  def write(self, data):
2948
    parts = (self._buffer + data).split("\n")
2949
    self._buffer = parts.pop()
2950
    self._lines.extend(parts)
2951

    
2952
  def flush(self):
2953
    while self._lines:
2954
      self._line_fn(self._lines.popleft().rstrip("\r\n"))
2955

    
2956
  def close(self):
2957
    self.flush()
2958
    if self._buffer:
2959
      self._line_fn(self._buffer)
2960

    
2961

    
2962
def SignalHandled(signums):
2963
  """Signal Handled decoration.
2964

2965
  This special decorator installs a signal handler and then calls the target
2966
  function. The function must accept a 'signal_handlers' keyword argument,
2967
  which will contain a dict indexed by signal number, with SignalHandler
2968
  objects as values.
2969

2970
  The decorator can be safely stacked with iself, to handle multiple signals
2971
  with different handlers.
2972

2973
  @type signums: list
2974
  @param signums: signals to intercept
2975

2976
  """
2977
  def wrap(fn):
2978
    def sig_function(*args, **kwargs):
2979
      assert 'signal_handlers' not in kwargs or \
2980
             kwargs['signal_handlers'] is None or \
2981
             isinstance(kwargs['signal_handlers'], dict), \
2982
             "Wrong signal_handlers parameter in original function call"
2983
      if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2984
        signal_handlers = kwargs['signal_handlers']
2985
      else:
2986
        signal_handlers = {}
2987
        kwargs['signal_handlers'] = signal_handlers
2988
      sighandler = SignalHandler(signums)
2989
      try:
2990
        for sig in signums:
2991
          signal_handlers[sig] = sighandler
2992
        return fn(*args, **kwargs)
2993
      finally:
2994
        sighandler.Reset()
2995
    return sig_function
2996
  return wrap
2997

    
2998

    
2999
class SignalHandler(object):
3000
  """Generic signal handler class.
3001

3002
  It automatically restores the original handler when deconstructed or
3003
  when L{Reset} is called. You can either pass your own handler
3004
  function in or query the L{called} attribute to detect whether the
3005
  signal was sent.
3006

3007
  @type signum: list
3008
  @ivar signum: the signals we handle
3009
  @type called: boolean
3010
  @ivar called: tracks whether any of the signals have been raised
3011

3012
  """
3013
  def __init__(self, signum):
3014
    """Constructs a new SignalHandler instance.
3015

3016
    @type signum: int or list of ints
3017
    @param signum: Single signal number or set of signal numbers
3018

3019
    """
3020
    self.signum = set(signum)
3021
    self.called = False
3022

    
3023
    self._previous = {}
3024
    try:
3025
      for signum in self.signum:
3026
        # Setup handler
3027
        prev_handler = signal.signal(signum, self._HandleSignal)
3028
        try:
3029
          self._previous[signum] = prev_handler
3030
        except:
3031
          # Restore previous handler
3032
          signal.signal(signum, prev_handler)
3033
          raise
3034
    except:
3035
      # Reset all handlers
3036
      self.Reset()
3037
      # Here we have a race condition: a handler may have already been called,
3038
      # but there's not much we can do about it at this point.
3039
      raise
3040

    
3041
  def __del__(self):
3042
    self.Reset()
3043

    
3044
  def Reset(self):
3045
    """Restore previous handler.
3046

3047
    This will reset all the signals to their previous handlers.
3048

3049
    """
3050
    for signum, prev_handler in self._previous.items():
3051
      signal.signal(signum, prev_handler)
3052
      # If successful, remove from dict
3053
      del self._previous[signum]
3054

    
3055
  def Clear(self):
3056
    """Unsets the L{called} flag.
3057

3058
    This function can be used in case a signal may arrive several times.
3059

3060
    """
3061
    self.called = False
3062

    
3063
  # we don't care about arguments, but we leave them named for the future
3064
  def _HandleSignal(self, signum, frame): # pylint: disable-msg=W0613
3065
    """Actual signal handling function.
3066

3067
    """
3068
    # This is not nice and not absolutely atomic, but it appears to be the only
3069
    # solution in Python -- there are no atomic types.
3070
    self.called = True
3071

    
3072

    
3073
class FieldSet(object):
3074
  """A simple field set.
3075

3076
  Among the features are:
3077
    - checking if a string is among a list of static string or regex objects
3078
    - checking if a whole list of string matches
3079
    - returning the matching groups from a regex match
3080

3081
  Internally, all fields are held as regular expression objects.
3082

3083
  """
3084
  def __init__(self, *items):
3085
    self.items = [re.compile("^%s$" % value) for value in items]
3086

    
3087
  def Extend(self, other_set):
3088
    """Extend the field set with the items from another one"""
3089
    self.items.extend(other_set.items)
3090

    
3091
  def Matches(self, field):
3092
    """Checks if a field matches the current set
3093

3094
    @type field: str
3095
    @param field: the string to match
3096
    @return: either None or a regular expression match object
3097

3098
    """
3099
    for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
3100
      return m
3101
    return None
3102

    
3103
  def NonMatching(self, items):
3104
    """Returns the list of fields not matching the current set
3105

3106
    @type items: list
3107
    @param items: the list of fields to check
3108
    @rtype: list
3109
    @return: list of non-matching fields
3110

3111
    """
3112
    return [val for val in items if not self.Matches(val)]