Statistics
| Branch: | Tag: | Revision:

root / lib / utils.py @ cea881e5

History | View | Annotate | Download (78.7 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Ganeti utility module.
23

24
This module holds functions that can be used in both daemons (all) and
25
the command line scripts.
26

27
"""
28

    
29

    
30
import os
31
import time
32
import subprocess
33
import re
34
import socket
35
import tempfile
36
import shutil
37
import errno
38
import pwd
39
import itertools
40
import select
41
import fcntl
42
import resource
43
import logging
44
import logging.handlers
45
import signal
46
import datetime
47
import calendar
48
import collections
49

    
50
from cStringIO import StringIO
51

    
52
try:
53
  from hashlib import sha1
54
except ImportError:
55
  import sha
56
  sha1 = sha.new
57

    
58
from ganeti import errors
59
from ganeti import constants
60

    
61

    
62
_locksheld = []
63
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
64

    
65
debug_locks = False
66

    
67
#: when set to True, L{RunCmd} is disabled
68
no_fork = False
69

    
70
_RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
71

    
72

    
73
class RunResult(object):
74
  """Holds the result of running external programs.
75

76
  @type exit_code: int
77
  @ivar exit_code: the exit code of the program, or None (if the program
78
      didn't exit())
79
  @type signal: int or None
80
  @ivar signal: the signal that caused the program to finish, or None
81
      (if the program wasn't terminated by a signal)
82
  @type stdout: str
83
  @ivar stdout: the standard output of the program
84
  @type stderr: str
85
  @ivar stderr: the standard error of the program
86
  @type failed: boolean
87
  @ivar failed: True in case the program was
88
      terminated by a signal or exited with a non-zero exit code
89
  @ivar fail_reason: a string detailing the termination reason
90

91
  """
92
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
93
               "failed", "fail_reason", "cmd"]
94

    
95

    
96
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
97
    self.cmd = cmd
98
    self.exit_code = exit_code
99
    self.signal = signal_
100
    self.stdout = stdout
101
    self.stderr = stderr
102
    self.failed = (signal_ is not None or exit_code != 0)
103

    
104
    if self.signal is not None:
105
      self.fail_reason = "terminated by signal %s" % self.signal
106
    elif self.exit_code is not None:
107
      self.fail_reason = "exited with exit code %s" % self.exit_code
108
    else:
109
      self.fail_reason = "unable to determine termination reason"
110

    
111
    if self.failed:
112
      logging.debug("Command '%s' failed (%s); output: %s",
113
                    self.cmd, self.fail_reason, self.output)
114

    
115
  def _GetOutput(self):
116
    """Returns the combined stdout and stderr for easier usage.
117

118
    """
119
    return self.stdout + self.stderr
120

    
121
  output = property(_GetOutput, None, None, "Return full output")
122

    
123

    
124
def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
125
  """Execute a (shell) command.
126

127
  The command should not read from its standard input, as it will be
128
  closed.
129

130
  @type cmd: string or list
131
  @param cmd: Command to run
132
  @type env: dict
133
  @param env: Additional environment
134
  @type output: str
135
  @param output: if desired, the output of the command can be
136
      saved in a file instead of the RunResult instance; this
137
      parameter denotes the file name (if not None)
138
  @type cwd: string
139
  @param cwd: if specified, will be used as the working
140
      directory for the command; the default will be /
141
  @type reset_env: boolean
142
  @param reset_env: whether to reset or keep the default os environment
143
  @rtype: L{RunResult}
144
  @return: RunResult instance
145
  @raise errors.ProgrammerError: if we call this when forks are disabled
146

147
  """
148
  if no_fork:
149
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
150

    
151
  if isinstance(cmd, list):
152
    cmd = [str(val) for val in cmd]
153
    strcmd = " ".join(cmd)
154
    shell = False
155
  else:
156
    strcmd = cmd
157
    shell = True
158
  logging.debug("RunCmd '%s'", strcmd)
159

    
160
  if not reset_env:
161
    cmd_env = os.environ.copy()
162
    cmd_env["LC_ALL"] = "C"
163
  else:
164
    cmd_env = {}
165

    
166
  if env is not None:
167
    cmd_env.update(env)
168

    
169
  try:
170
    if output is None:
171
      out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
172
    else:
173
      status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
174
      out = err = ""
175
  except OSError, err:
176
    if err.errno == errno.ENOENT:
177
      raise errors.OpExecError("Can't execute '%s': not found (%s)" %
178
                               (strcmd, err))
179
    else:
180
      raise
181

    
182
  if status >= 0:
183
    exitcode = status
184
    signal_ = None
185
  else:
186
    exitcode = None
187
    signal_ = -status
188

    
189
  return RunResult(exitcode, signal_, out, err, strcmd)
190

    
191

    
192
def _RunCmdPipe(cmd, env, via_shell, cwd):
193
  """Run a command and return its output.
194

195
  @type  cmd: string or list
196
  @param cmd: Command to run
197
  @type env: dict
198
  @param env: The environment to use
199
  @type via_shell: bool
200
  @param via_shell: if we should run via the shell
201
  @type cwd: string
202
  @param cwd: the working directory for the program
203
  @rtype: tuple
204
  @return: (out, err, status)
205

206
  """
207
  poller = select.poll()
208
  child = subprocess.Popen(cmd, shell=via_shell,
209
                           stderr=subprocess.PIPE,
210
                           stdout=subprocess.PIPE,
211
                           stdin=subprocess.PIPE,
212
                           close_fds=True, env=env,
213
                           cwd=cwd)
214

    
215
  child.stdin.close()
216
  poller.register(child.stdout, select.POLLIN)
217
  poller.register(child.stderr, select.POLLIN)
218
  out = StringIO()
219
  err = StringIO()
220
  fdmap = {
221
    child.stdout.fileno(): (out, child.stdout),
222
    child.stderr.fileno(): (err, child.stderr),
223
    }
224
  for fd in fdmap:
225
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
226
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
227

    
228
  while fdmap:
229
    try:
230
      pollresult = poller.poll()
231
    except EnvironmentError, eerr:
232
      if eerr.errno == errno.EINTR:
233
        continue
234
      raise
235
    except select.error, serr:
236
      if serr[0] == errno.EINTR:
237
        continue
238
      raise
239

    
240
    for fd, event in pollresult:
241
      if event & select.POLLIN or event & select.POLLPRI:
242
        data = fdmap[fd][1].read()
243
        # no data from read signifies EOF (the same as POLLHUP)
244
        if not data:
245
          poller.unregister(fd)
246
          del fdmap[fd]
247
          continue
248
        fdmap[fd][0].write(data)
249
      if (event & select.POLLNVAL or event & select.POLLHUP or
250
          event & select.POLLERR):
251
        poller.unregister(fd)
252
        del fdmap[fd]
253

    
254
  out = out.getvalue()
255
  err = err.getvalue()
256

    
257
  status = child.wait()
258
  return out, err, status
259

    
260

    
261
def _RunCmdFile(cmd, env, via_shell, output, cwd):
262
  """Run a command and save its output to a file.
263

264
  @type  cmd: string or list
265
  @param cmd: Command to run
266
  @type env: dict
267
  @param env: The environment to use
268
  @type via_shell: bool
269
  @param via_shell: if we should run via the shell
270
  @type output: str
271
  @param output: the filename in which to save the output
272
  @type cwd: string
273
  @param cwd: the working directory for the program
274
  @rtype: int
275
  @return: the exit status
276

277
  """
278
  fh = open(output, "a")
279
  try:
280
    child = subprocess.Popen(cmd, shell=via_shell,
281
                             stderr=subprocess.STDOUT,
282
                             stdout=fh,
283
                             stdin=subprocess.PIPE,
284
                             close_fds=True, env=env,
285
                             cwd=cwd)
286

    
287
    child.stdin.close()
288
    status = child.wait()
289
  finally:
290
    fh.close()
291
  return status
292

    
293

    
294
def RunParts(dir_name, env=None, reset_env=False):
295
  """Run Scripts or programs in a directory
296

297
  @type dir_name: string
298
  @param dir_name: absolute path to a directory
299
  @type env: dict
300
  @param env: The environment to use
301
  @type reset_env: boolean
302
  @param reset_env: whether to reset or keep the default os environment
303
  @rtype: list of tuples
304
  @return: list of (name, (one of RUNDIR_STATUS), RunResult)
305

306
  """
307
  rr = []
308

    
309
  try:
310
    dir_contents = ListVisibleFiles(dir_name)
311
  except OSError, err:
312
    logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
313
    return rr
314

    
315
  for relname in sorted(dir_contents):
316
    fname = PathJoin(dir_name, relname)
317
    if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
318
            constants.EXT_PLUGIN_MASK.match(relname) is not None):
319
      rr.append((relname, constants.RUNPARTS_SKIP, None))
320
    else:
321
      try:
322
        result = RunCmd([fname], env=env, reset_env=reset_env)
323
      except Exception, err: # pylint: disable-msg=W0703
324
        rr.append((relname, constants.RUNPARTS_ERR, str(err)))
325
      else:
326
        rr.append((relname, constants.RUNPARTS_RUN, result))
327

    
328
  return rr
329

    
330

    
331
def RemoveFile(filename):
332
  """Remove a file ignoring some errors.
333

334
  Remove a file, ignoring non-existing ones or directories. Other
335
  errors are passed.
336

337
  @type filename: str
338
  @param filename: the file to be removed
339

340
  """
341
  try:
342
    os.unlink(filename)
343
  except OSError, err:
344
    if err.errno not in (errno.ENOENT, errno.EISDIR):
345
      raise
346

    
347

    
348
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
349
  """Renames a file.
350

351
  @type old: string
352
  @param old: Original path
353
  @type new: string
354
  @param new: New path
355
  @type mkdir: bool
356
  @param mkdir: Whether to create target directory if it doesn't exist
357
  @type mkdir_mode: int
358
  @param mkdir_mode: Mode for newly created directories
359

360
  """
361
  try:
362
    return os.rename(old, new)
363
  except OSError, err:
364
    # In at least one use case of this function, the job queue, directory
365
    # creation is very rare. Checking for the directory before renaming is not
366
    # as efficient.
367
    if mkdir and err.errno == errno.ENOENT:
368
      # Create directory and try again
369
      Makedirs(os.path.dirname(new), mode=mkdir_mode)
370

    
371
      return os.rename(old, new)
372

    
373
    raise
374

    
375

    
376
def Makedirs(path, mode=0750):
377
  """Super-mkdir; create a leaf directory and all intermediate ones.
378

379
  This is a wrapper around C{os.makedirs} adding error handling not implemented
380
  before Python 2.5.
381

382
  """
383
  try:
384
    os.makedirs(path, mode)
385
  except OSError, err:
386
    # Ignore EEXIST. This is only handled in os.makedirs as included in
387
    # Python 2.5 and above.
388
    if err.errno != errno.EEXIST or not os.path.exists(path):
389
      raise
390

    
391

    
392
def ResetTempfileModule():
393
  """Resets the random name generator of the tempfile module.
394

395
  This function should be called after C{os.fork} in the child process to
396
  ensure it creates a newly seeded random generator. Otherwise it would
397
  generate the same random parts as the parent process. If several processes
398
  race for the creation of a temporary file, this could lead to one not getting
399
  a temporary name.
400

401
  """
402
  # pylint: disable-msg=W0212
403
  if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
404
    tempfile._once_lock.acquire()
405
    try:
406
      # Reset random name generator
407
      tempfile._name_sequence = None
408
    finally:
409
      tempfile._once_lock.release()
410
  else:
411
    logging.critical("The tempfile module misses at least one of the"
412
                     " '_once_lock' and '_name_sequence' attributes")
413

    
414

    
415
def _FingerprintFile(filename):
416
  """Compute the fingerprint of a file.
417

418
  If the file does not exist, a None will be returned
419
  instead.
420

421
  @type filename: str
422
  @param filename: the filename to checksum
423
  @rtype: str
424
  @return: the hex digest of the sha checksum of the contents
425
      of the file
426

427
  """
428
  if not (os.path.exists(filename) and os.path.isfile(filename)):
429
    return None
430

    
431
  f = open(filename)
432

    
433
  fp = sha1()
434
  while True:
435
    data = f.read(4096)
436
    if not data:
437
      break
438

    
439
    fp.update(data)
440

    
441
  return fp.hexdigest()
442

    
443

    
444
def FingerprintFiles(files):
445
  """Compute fingerprints for a list of files.
446

447
  @type files: list
448
  @param files: the list of filename to fingerprint
449
  @rtype: dict
450
  @return: a dictionary filename: fingerprint, holding only
451
      existing files
452

453
  """
454
  ret = {}
455

    
456
  for filename in files:
457
    cksum = _FingerprintFile(filename)
458
    if cksum:
459
      ret[filename] = cksum
460

    
461
  return ret
462

    
463

    
464
def ForceDictType(target, key_types, allowed_values=None):
465
  """Force the values of a dict to have certain types.
466

467
  @type target: dict
468
  @param target: the dict to update
469
  @type key_types: dict
470
  @param key_types: dict mapping target dict keys to types
471
                    in constants.ENFORCEABLE_TYPES
472
  @type allowed_values: list
473
  @keyword allowed_values: list of specially allowed values
474

475
  """
476
  if allowed_values is None:
477
    allowed_values = []
478

    
479
  if not isinstance(target, dict):
480
    msg = "Expected dictionary, got '%s'" % target
481
    raise errors.TypeEnforcementError(msg)
482

    
483
  for key in target:
484
    if key not in key_types:
485
      msg = "Unknown key '%s'" % key
486
      raise errors.TypeEnforcementError(msg)
487

    
488
    if target[key] in allowed_values:
489
      continue
490

    
491
    ktype = key_types[key]
492
    if ktype not in constants.ENFORCEABLE_TYPES:
493
      msg = "'%s' has non-enforceable type %s" % (key, ktype)
494
      raise errors.ProgrammerError(msg)
495

    
496
    if ktype == constants.VTYPE_STRING:
497
      if not isinstance(target[key], basestring):
498
        if isinstance(target[key], bool) and not target[key]:
499
          target[key] = ''
500
        else:
501
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
502
          raise errors.TypeEnforcementError(msg)
503
    elif ktype == constants.VTYPE_BOOL:
504
      if isinstance(target[key], basestring) and target[key]:
505
        if target[key].lower() == constants.VALUE_FALSE:
506
          target[key] = False
507
        elif target[key].lower() == constants.VALUE_TRUE:
508
          target[key] = True
509
        else:
510
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
511
          raise errors.TypeEnforcementError(msg)
512
      elif target[key]:
513
        target[key] = True
514
      else:
515
        target[key] = False
516
    elif ktype == constants.VTYPE_SIZE:
517
      try:
518
        target[key] = ParseUnit(target[key])
519
      except errors.UnitParseError, err:
520
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
521
              (key, target[key], err)
522
        raise errors.TypeEnforcementError(msg)
523
    elif ktype == constants.VTYPE_INT:
524
      try:
525
        target[key] = int(target[key])
526
      except (ValueError, TypeError):
527
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
528
        raise errors.TypeEnforcementError(msg)
529

    
530

    
531
def IsProcessAlive(pid):
532
  """Check if a given pid exists on the system.
533

534
  @note: zombie status is not handled, so zombie processes
535
      will be returned as alive
536
  @type pid: int
537
  @param pid: the process ID to check
538
  @rtype: boolean
539
  @return: True if the process exists
540

541
  """
542
  if pid <= 0:
543
    return False
544

    
545
  try:
546
    os.stat("/proc/%d/status" % pid)
547
    return True
548
  except EnvironmentError, err:
549
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
550
      return False
551
    raise
552

    
553

    
554
def ReadPidFile(pidfile):
555
  """Read a pid from a file.
556

557
  @type  pidfile: string
558
  @param pidfile: path to the file containing the pid
559
  @rtype: int
560
  @return: The process id, if the file exists and contains a valid PID,
561
           otherwise 0
562

563
  """
564
  try:
565
    raw_data = ReadFile(pidfile)
566
  except EnvironmentError, err:
567
    if err.errno != errno.ENOENT:
568
      logging.exception("Can't read pid file")
569
    return 0
570

    
571
  try:
572
    pid = int(raw_data)
573
  except (TypeError, ValueError), err:
574
    logging.info("Can't parse pid file contents", exc_info=True)
575
    return 0
576

    
577
  return pid
578

    
579

    
580
def MatchNameComponent(key, name_list, case_sensitive=True):
581
  """Try to match a name against a list.
582

583
  This function will try to match a name like test1 against a list
584
  like C{['test1.example.com', 'test2.example.com', ...]}. Against
585
  this list, I{'test1'} as well as I{'test1.example'} will match, but
586
  not I{'test1.ex'}. A multiple match will be considered as no match
587
  at all (e.g. I{'test1'} against C{['test1.example.com',
588
  'test1.example.org']}), except when the key fully matches an entry
589
  (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
590

591
  @type key: str
592
  @param key: the name to be searched
593
  @type name_list: list
594
  @param name_list: the list of strings against which to search the key
595
  @type case_sensitive: boolean
596
  @param case_sensitive: whether to provide a case-sensitive match
597

598
  @rtype: None or str
599
  @return: None if there is no match I{or} if there are multiple matches,
600
      otherwise the element from the list which matches
601

602
  """
603
  if key in name_list:
604
    return key
605

    
606
  re_flags = 0
607
  if not case_sensitive:
608
    re_flags |= re.IGNORECASE
609
    key = key.upper()
610
  mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
611
  names_filtered = []
612
  string_matches = []
613
  for name in name_list:
614
    if mo.match(name) is not None:
615
      names_filtered.append(name)
616
      if not case_sensitive and key == name.upper():
617
        string_matches.append(name)
618

    
619
  if len(string_matches) == 1:
620
    return string_matches[0]
621
  if len(names_filtered) == 1:
622
    return names_filtered[0]
623
  return None
624

    
625

    
626
class HostInfo:
627
  """Class implementing resolver and hostname functionality
628

629
  """
630
  _VALID_NAME_RE = re.compile("^[a-z0-9._-]{1,255}$")
631

    
632
  def __init__(self, name=None):
633
    """Initialize the host name object.
634

635
    If the name argument is not passed, it will use this system's
636
    name.
637

638
    """
639
    if name is None:
640
      name = self.SysName()
641

    
642
    self.query = name
643
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
644
    self.ip = self.ipaddrs[0]
645

    
646
  def ShortName(self):
647
    """Returns the hostname without domain.
648

649
    """
650
    return self.name.split('.')[0]
651

    
652
  @staticmethod
653
  def SysName():
654
    """Return the current system's name.
655

656
    This is simply a wrapper over C{socket.gethostname()}.
657

658
    """
659
    return socket.gethostname()
660

    
661
  @staticmethod
662
  def LookupHostname(hostname):
663
    """Look up hostname
664

665
    @type hostname: str
666
    @param hostname: hostname to look up
667

668
    @rtype: tuple
669
    @return: a tuple (name, aliases, ipaddrs) as returned by
670
        C{socket.gethostbyname_ex}
671
    @raise errors.ResolverError: in case of errors in resolving
672

673
    """
674
    try:
675
      result = socket.gethostbyname_ex(hostname)
676
    except socket.gaierror, err:
677
      # hostname not found in DNS
678
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
679

    
680
    return result
681

    
682
  @classmethod
683
  def NormalizeName(cls, hostname):
684
    """Validate and normalize the given hostname.
685

686
    @attention: the validation is a bit more relaxed than the standards
687
        require; most importantly, we allow underscores in names
688
    @raise errors.OpPrereqError: when the name is not valid
689

690
    """
691
    hostname = hostname.lower()
692
    if (not cls._VALID_NAME_RE.match(hostname) or
693
        # double-dots, meaning empty label
694
        ".." in hostname or
695
        # empty initial label
696
        hostname.startswith(".")):
697
      raise errors.OpPrereqError("Invalid hostname '%s'" % hostname,
698
                                 errors.ECODE_INVAL)
699
    if hostname.endswith("."):
700
      hostname = hostname.rstrip(".")
701
    return hostname
702

    
703

    
704
def GetHostInfo(name=None):
705
  """Lookup host name and raise an OpPrereqError for failures"""
706

    
707
  try:
708
    return HostInfo(name)
709
  except errors.ResolverError, err:
710
    raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
711
                               (err[0], err[2]), errors.ECODE_RESOLVER)
712

    
713

    
714
def ListVolumeGroups():
715
  """List volume groups and their size
716

717
  @rtype: dict
718
  @return:
719
       Dictionary with keys volume name and values
720
       the size of the volume
721

722
  """
723
  command = "vgs --noheadings --units m --nosuffix -o name,size"
724
  result = RunCmd(command)
725
  retval = {}
726
  if result.failed:
727
    return retval
728

    
729
  for line in result.stdout.splitlines():
730
    try:
731
      name, size = line.split()
732
      size = int(float(size))
733
    except (IndexError, ValueError), err:
734
      logging.error("Invalid output from vgs (%s): %s", err, line)
735
      continue
736

    
737
    retval[name] = size
738

    
739
  return retval
740

    
741

    
742
def BridgeExists(bridge):
743
  """Check whether the given bridge exists in the system
744

745
  @type bridge: str
746
  @param bridge: the bridge name to check
747
  @rtype: boolean
748
  @return: True if it does
749

750
  """
751
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
752

    
753

    
754
def NiceSort(name_list):
755
  """Sort a list of strings based on digit and non-digit groupings.
756

757
  Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
758
  will sort the list in the logical order C{['a1', 'a2', 'a10',
759
  'a11']}.
760

761
  The sort algorithm breaks each name in groups of either only-digits
762
  or no-digits. Only the first eight such groups are considered, and
763
  after that we just use what's left of the string.
764

765
  @type name_list: list
766
  @param name_list: the names to be sorted
767
  @rtype: list
768
  @return: a copy of the name list sorted with our algorithm
769

770
  """
771
  _SORTER_BASE = "(\D+|\d+)"
772
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
773
                                                  _SORTER_BASE, _SORTER_BASE,
774
                                                  _SORTER_BASE, _SORTER_BASE,
775
                                                  _SORTER_BASE, _SORTER_BASE)
776
  _SORTER_RE = re.compile(_SORTER_FULL)
777
  _SORTER_NODIGIT = re.compile("^\D*$")
778
  def _TryInt(val):
779
    """Attempts to convert a variable to integer."""
780
    if val is None or _SORTER_NODIGIT.match(val):
781
      return val
782
    rval = int(val)
783
    return rval
784

    
785
  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
786
             for name in name_list]
787
  to_sort.sort()
788
  return [tup[1] for tup in to_sort]
789

    
790

    
791
def TryConvert(fn, val):
792
  """Try to convert a value ignoring errors.
793

794
  This function tries to apply function I{fn} to I{val}. If no
795
  C{ValueError} or C{TypeError} exceptions are raised, it will return
796
  the result, else it will return the original value. Any other
797
  exceptions are propagated to the caller.
798

799
  @type fn: callable
800
  @param fn: function to apply to the value
801
  @param val: the value to be converted
802
  @return: The converted value if the conversion was successful,
803
      otherwise the original value.
804

805
  """
806
  try:
807
    nv = fn(val)
808
  except (ValueError, TypeError):
809
    nv = val
810
  return nv
811

    
812

    
813
def IsValidIP(ip):
814
  """Verifies the syntax of an IPv4 address.
815

816
  This function checks if the IPv4 address passes is valid or not based
817
  on syntax (not IP range, class calculations, etc.).
818

819
  @type ip: str
820
  @param ip: the address to be checked
821
  @rtype: a regular expression match object
822
  @return: a regular expression match object, or None if the
823
      address is not valid
824

825
  """
826
  unit = "(0|[1-9]\d{0,2})"
827
  #TODO: convert and return only boolean
828
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
829

    
830

    
831
def IsValidShellParam(word):
832
  """Verifies is the given word is safe from the shell's p.o.v.
833

834
  This means that we can pass this to a command via the shell and be
835
  sure that it doesn't alter the command line and is passed as such to
836
  the actual command.
837

838
  Note that we are overly restrictive here, in order to be on the safe
839
  side.
840

841
  @type word: str
842
  @param word: the word to check
843
  @rtype: boolean
844
  @return: True if the word is 'safe'
845

846
  """
847
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
848

    
849

    
850
def BuildShellCmd(template, *args):
851
  """Build a safe shell command line from the given arguments.
852

853
  This function will check all arguments in the args list so that they
854
  are valid shell parameters (i.e. they don't contain shell
855
  metacharacters). If everything is ok, it will return the result of
856
  template % args.
857

858
  @type template: str
859
  @param template: the string holding the template for the
860
      string formatting
861
  @rtype: str
862
  @return: the expanded command line
863

864
  """
865
  for word in args:
866
    if not IsValidShellParam(word):
867
      raise errors.ProgrammerError("Shell argument '%s' contains"
868
                                   " invalid characters" % word)
869
  return template % args
870

    
871

    
872
def FormatUnit(value, units):
873
  """Formats an incoming number of MiB with the appropriate unit.
874

875
  @type value: int
876
  @param value: integer representing the value in MiB (1048576)
877
  @type units: char
878
  @param units: the type of formatting we should do:
879
      - 'h' for automatic scaling
880
      - 'm' for MiBs
881
      - 'g' for GiBs
882
      - 't' for TiBs
883
  @rtype: str
884
  @return: the formatted value (with suffix)
885

886
  """
887
  if units not in ('m', 'g', 't', 'h'):
888
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
889

    
890
  suffix = ''
891

    
892
  if units == 'm' or (units == 'h' and value < 1024):
893
    if units == 'h':
894
      suffix = 'M'
895
    return "%d%s" % (round(value, 0), suffix)
896

    
897
  elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
898
    if units == 'h':
899
      suffix = 'G'
900
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
901

    
902
  else:
903
    if units == 'h':
904
      suffix = 'T'
905
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
906

    
907

    
908
def ParseUnit(input_string):
909
  """Tries to extract number and scale from the given string.
910

911
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
912
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
913
  is always an int in MiB.
914

915
  """
916
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
917
  if not m:
918
    raise errors.UnitParseError("Invalid format")
919

    
920
  value = float(m.groups()[0])
921

    
922
  unit = m.groups()[1]
923
  if unit:
924
    lcunit = unit.lower()
925
  else:
926
    lcunit = 'm'
927

    
928
  if lcunit in ('m', 'mb', 'mib'):
929
    # Value already in MiB
930
    pass
931

    
932
  elif lcunit in ('g', 'gb', 'gib'):
933
    value *= 1024
934

    
935
  elif lcunit in ('t', 'tb', 'tib'):
936
    value *= 1024 * 1024
937

    
938
  else:
939
    raise errors.UnitParseError("Unknown unit: %s" % unit)
940

    
941
  # Make sure we round up
942
  if int(value) < value:
943
    value += 1
944

    
945
  # Round up to the next multiple of 4
946
  value = int(value)
947
  if value % 4:
948
    value += 4 - value % 4
949

    
950
  return value
951

    
952

    
953
def AddAuthorizedKey(file_name, key):
954
  """Adds an SSH public key to an authorized_keys file.
955

956
  @type file_name: str
957
  @param file_name: path to authorized_keys file
958
  @type key: str
959
  @param key: string containing key
960

961
  """
962
  key_fields = key.split()
963

    
964
  f = open(file_name, 'a+')
965
  try:
966
    nl = True
967
    for line in f:
968
      # Ignore whitespace changes
969
      if line.split() == key_fields:
970
        break
971
      nl = line.endswith('\n')
972
    else:
973
      if not nl:
974
        f.write("\n")
975
      f.write(key.rstrip('\r\n'))
976
      f.write("\n")
977
      f.flush()
978
  finally:
979
    f.close()
980

    
981

    
982
def RemoveAuthorizedKey(file_name, key):
983
  """Removes an SSH public key from an authorized_keys file.
984

985
  @type file_name: str
986
  @param file_name: path to authorized_keys file
987
  @type key: str
988
  @param key: string containing key
989

990
  """
991
  key_fields = key.split()
992

    
993
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
994
  try:
995
    out = os.fdopen(fd, 'w')
996
    try:
997
      f = open(file_name, 'r')
998
      try:
999
        for line in f:
1000
          # Ignore whitespace changes while comparing lines
1001
          if line.split() != key_fields:
1002
            out.write(line)
1003

    
1004
        out.flush()
1005
        os.rename(tmpname, file_name)
1006
      finally:
1007
        f.close()
1008
    finally:
1009
      out.close()
1010
  except:
1011
    RemoveFile(tmpname)
1012
    raise
1013

    
1014

    
1015
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1016
  """Sets the name of an IP address and hostname in /etc/hosts.
1017

1018
  @type file_name: str
1019
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1020
  @type ip: str
1021
  @param ip: the IP address
1022
  @type hostname: str
1023
  @param hostname: the hostname to be added
1024
  @type aliases: list
1025
  @param aliases: the list of aliases to add for the hostname
1026

1027
  """
1028
  # FIXME: use WriteFile + fn rather than duplicating its efforts
1029
  # Ensure aliases are unique
1030
  aliases = UniqueSequence([hostname] + aliases)[1:]
1031

    
1032
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1033
  try:
1034
    out = os.fdopen(fd, 'w')
1035
    try:
1036
      f = open(file_name, 'r')
1037
      try:
1038
        for line in f:
1039
          fields = line.split()
1040
          if fields and not fields[0].startswith('#') and ip == fields[0]:
1041
            continue
1042
          out.write(line)
1043

    
1044
        out.write("%s\t%s" % (ip, hostname))
1045
        if aliases:
1046
          out.write(" %s" % ' '.join(aliases))
1047
        out.write('\n')
1048

    
1049
        out.flush()
1050
        os.fsync(out)
1051
        os.chmod(tmpname, 0644)
1052
        os.rename(tmpname, file_name)
1053
      finally:
1054
        f.close()
1055
    finally:
1056
      out.close()
1057
  except:
1058
    RemoveFile(tmpname)
1059
    raise
1060

    
1061

    
1062
def AddHostToEtcHosts(hostname):
1063
  """Wrapper around SetEtcHostsEntry.
1064

1065
  @type hostname: str
1066
  @param hostname: a hostname that will be resolved and added to
1067
      L{constants.ETC_HOSTS}
1068

1069
  """
1070
  hi = HostInfo(name=hostname)
1071
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1072

    
1073

    
1074
def RemoveEtcHostsEntry(file_name, hostname):
1075
  """Removes a hostname from /etc/hosts.
1076

1077
  IP addresses without names are removed from the file.
1078

1079
  @type file_name: str
1080
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1081
  @type hostname: str
1082
  @param hostname: the hostname to be removed
1083

1084
  """
1085
  # FIXME: use WriteFile + fn rather than duplicating its efforts
1086
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1087
  try:
1088
    out = os.fdopen(fd, 'w')
1089
    try:
1090
      f = open(file_name, 'r')
1091
      try:
1092
        for line in f:
1093
          fields = line.split()
1094
          if len(fields) > 1 and not fields[0].startswith('#'):
1095
            names = fields[1:]
1096
            if hostname in names:
1097
              while hostname in names:
1098
                names.remove(hostname)
1099
              if names:
1100
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
1101
              continue
1102

    
1103
          out.write(line)
1104

    
1105
        out.flush()
1106
        os.fsync(out)
1107
        os.chmod(tmpname, 0644)
1108
        os.rename(tmpname, file_name)
1109
      finally:
1110
        f.close()
1111
    finally:
1112
      out.close()
1113
  except:
1114
    RemoveFile(tmpname)
1115
    raise
1116

    
1117

    
1118
def RemoveHostFromEtcHosts(hostname):
1119
  """Wrapper around RemoveEtcHostsEntry.
1120

1121
  @type hostname: str
1122
  @param hostname: hostname that will be resolved and its
1123
      full and shot name will be removed from
1124
      L{constants.ETC_HOSTS}
1125

1126
  """
1127
  hi = HostInfo(name=hostname)
1128
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1129
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1130

    
1131

    
1132
def TimestampForFilename():
1133
  """Returns the current time formatted for filenames.
1134

1135
  The format doesn't contain colons as some shells and applications them as
1136
  separators.
1137

1138
  """
1139
  return time.strftime("%Y-%m-%d_%H_%M_%S")
1140

    
1141

    
1142
def CreateBackup(file_name):
1143
  """Creates a backup of a file.
1144

1145
  @type file_name: str
1146
  @param file_name: file to be backed up
1147
  @rtype: str
1148
  @return: the path to the newly created backup
1149
  @raise errors.ProgrammerError: for invalid file names
1150

1151
  """
1152
  if not os.path.isfile(file_name):
1153
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1154
                                file_name)
1155

    
1156
  prefix = ("%s.backup-%s." %
1157
            (os.path.basename(file_name), TimestampForFilename()))
1158
  dir_name = os.path.dirname(file_name)
1159

    
1160
  fsrc = open(file_name, 'rb')
1161
  try:
1162
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1163
    fdst = os.fdopen(fd, 'wb')
1164
    try:
1165
      logging.debug("Backing up %s at %s", file_name, backup_name)
1166
      shutil.copyfileobj(fsrc, fdst)
1167
    finally:
1168
      fdst.close()
1169
  finally:
1170
    fsrc.close()
1171

    
1172
  return backup_name
1173

    
1174

    
1175
def ShellQuote(value):
1176
  """Quotes shell argument according to POSIX.
1177

1178
  @type value: str
1179
  @param value: the argument to be quoted
1180
  @rtype: str
1181
  @return: the quoted value
1182

1183
  """
1184
  if _re_shell_unquoted.match(value):
1185
    return value
1186
  else:
1187
    return "'%s'" % value.replace("'", "'\\''")
1188

    
1189

    
1190
def ShellQuoteArgs(args):
1191
  """Quotes a list of shell arguments.
1192

1193
  @type args: list
1194
  @param args: list of arguments to be quoted
1195
  @rtype: str
1196
  @return: the quoted arguments concatenated with spaces
1197

1198
  """
1199
  return ' '.join([ShellQuote(i) for i in args])
1200

    
1201

    
1202
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1203
  """Simple ping implementation using TCP connect(2).
1204

1205
  Check if the given IP is reachable by doing attempting a TCP connect
1206
  to it.
1207

1208
  @type target: str
1209
  @param target: the IP or hostname to ping
1210
  @type port: int
1211
  @param port: the port to connect to
1212
  @type timeout: int
1213
  @param timeout: the timeout on the connection attempt
1214
  @type live_port_needed: boolean
1215
  @param live_port_needed: whether a closed port will cause the
1216
      function to return failure, as if there was a timeout
1217
  @type source: str or None
1218
  @param source: if specified, will cause the connect to be made
1219
      from this specific source address; failures to bind other
1220
      than C{EADDRNOTAVAIL} will be ignored
1221

1222
  """
1223
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1224

    
1225
  success = False
1226

    
1227
  if source is not None:
1228
    try:
1229
      sock.bind((source, 0))
1230
    except socket.error, (errcode, _):
1231
      if errcode == errno.EADDRNOTAVAIL:
1232
        success = False
1233

    
1234
  sock.settimeout(timeout)
1235

    
1236
  try:
1237
    sock.connect((target, port))
1238
    sock.close()
1239
    success = True
1240
  except socket.timeout:
1241
    success = False
1242
  except socket.error, (errcode, _):
1243
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1244

    
1245
  return success
1246

    
1247

    
1248
def OwnIpAddress(address):
1249
  """Check if the current host has the the given IP address.
1250

1251
  Currently this is done by TCP-pinging the address from the loopback
1252
  address.
1253

1254
  @type address: string
1255
  @param address: the address to check
1256
  @rtype: bool
1257
  @return: True if we own the address
1258

1259
  """
1260
  return TcpPing(address, constants.DEFAULT_NODED_PORT,
1261
                 source=constants.LOCALHOST_IP_ADDRESS)
1262

    
1263

    
1264
def ListVisibleFiles(path):
1265
  """Returns a list of visible files in a directory.
1266

1267
  @type path: str
1268
  @param path: the directory to enumerate
1269
  @rtype: list
1270
  @return: the list of all files not starting with a dot
1271
  @raise ProgrammerError: if L{path} is not an absolue and normalized path
1272

1273
  """
1274
  if not IsNormAbsPath(path):
1275
    raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
1276
                                 " absolute/normalized: '%s'" % path)
1277
  files = [i for i in os.listdir(path) if not i.startswith(".")]
1278
  files.sort()
1279
  return files
1280

    
1281

    
1282
def GetHomeDir(user, default=None):
1283
  """Try to get the homedir of the given user.
1284

1285
  The user can be passed either as a string (denoting the name) or as
1286
  an integer (denoting the user id). If the user is not found, the
1287
  'default' argument is returned, which defaults to None.
1288

1289
  """
1290
  try:
1291
    if isinstance(user, basestring):
1292
      result = pwd.getpwnam(user)
1293
    elif isinstance(user, (int, long)):
1294
      result = pwd.getpwuid(user)
1295
    else:
1296
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1297
                                   type(user))
1298
  except KeyError:
1299
    return default
1300
  return result.pw_dir
1301

    
1302

    
1303
def NewUUID():
1304
  """Returns a random UUID.
1305

1306
  @note: This is a Linux-specific method as it uses the /proc
1307
      filesystem.
1308
  @rtype: str
1309

1310
  """
1311
  return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1312

    
1313

    
1314
def GenerateSecret(numbytes=20):
1315
  """Generates a random secret.
1316

1317
  This will generate a pseudo-random secret returning an hex string
1318
  (so that it can be used where an ASCII string is needed).
1319

1320
  @param numbytes: the number of bytes which will be represented by the returned
1321
      string (defaulting to 20, the length of a SHA1 hash)
1322
  @rtype: str
1323
  @return: an hex representation of the pseudo-random sequence
1324

1325
  """
1326
  return os.urandom(numbytes).encode('hex')
1327

    
1328

    
1329
def EnsureDirs(dirs):
1330
  """Make required directories, if they don't exist.
1331

1332
  @param dirs: list of tuples (dir_name, dir_mode)
1333
  @type dirs: list of (string, integer)
1334

1335
  """
1336
  for dir_name, dir_mode in dirs:
1337
    try:
1338
      os.mkdir(dir_name, dir_mode)
1339
    except EnvironmentError, err:
1340
      if err.errno != errno.EEXIST:
1341
        raise errors.GenericError("Cannot create needed directory"
1342
                                  " '%s': %s" % (dir_name, err))
1343
    if not os.path.isdir(dir_name):
1344
      raise errors.GenericError("%s is not a directory" % dir_name)
1345

    
1346

    
1347
def ReadFile(file_name, size=-1):
1348
  """Reads a file.
1349

1350
  @type size: int
1351
  @param size: Read at most size bytes (if negative, entire file)
1352
  @rtype: str
1353
  @return: the (possibly partial) content of the file
1354

1355
  """
1356
  f = open(file_name, "r")
1357
  try:
1358
    return f.read(size)
1359
  finally:
1360
    f.close()
1361

    
1362

    
1363
def WriteFile(file_name, fn=None, data=None,
1364
              mode=None, uid=-1, gid=-1,
1365
              atime=None, mtime=None, close=True,
1366
              dry_run=False, backup=False,
1367
              prewrite=None, postwrite=None):
1368
  """(Over)write a file atomically.
1369

1370
  The file_name and either fn (a function taking one argument, the
1371
  file descriptor, and which should write the data to it) or data (the
1372
  contents of the file) must be passed. The other arguments are
1373
  optional and allow setting the file mode, owner and group, and the
1374
  mtime/atime of the file.
1375

1376
  If the function doesn't raise an exception, it has succeeded and the
1377
  target file has the new contents. If the function has raised an
1378
  exception, an existing target file should be unmodified and the
1379
  temporary file should be removed.
1380

1381
  @type file_name: str
1382
  @param file_name: the target filename
1383
  @type fn: callable
1384
  @param fn: content writing function, called with
1385
      file descriptor as parameter
1386
  @type data: str
1387
  @param data: contents of the file
1388
  @type mode: int
1389
  @param mode: file mode
1390
  @type uid: int
1391
  @param uid: the owner of the file
1392
  @type gid: int
1393
  @param gid: the group of the file
1394
  @type atime: int
1395
  @param atime: a custom access time to be set on the file
1396
  @type mtime: int
1397
  @param mtime: a custom modification time to be set on the file
1398
  @type close: boolean
1399
  @param close: whether to close file after writing it
1400
  @type prewrite: callable
1401
  @param prewrite: function to be called before writing content
1402
  @type postwrite: callable
1403
  @param postwrite: function to be called after writing content
1404

1405
  @rtype: None or int
1406
  @return: None if the 'close' parameter evaluates to True,
1407
      otherwise the file descriptor
1408

1409
  @raise errors.ProgrammerError: if any of the arguments are not valid
1410

1411
  """
1412
  if not os.path.isabs(file_name):
1413
    raise errors.ProgrammerError("Path passed to WriteFile is not"
1414
                                 " absolute: '%s'" % file_name)
1415

    
1416
  if [fn, data].count(None) != 1:
1417
    raise errors.ProgrammerError("fn or data required")
1418

    
1419
  if [atime, mtime].count(None) == 1:
1420
    raise errors.ProgrammerError("Both atime and mtime must be either"
1421
                                 " set or None")
1422

    
1423
  if backup and not dry_run and os.path.isfile(file_name):
1424
    CreateBackup(file_name)
1425

    
1426
  dir_name, base_name = os.path.split(file_name)
1427
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1428
  do_remove = True
1429
  # here we need to make sure we remove the temp file, if any error
1430
  # leaves it in place
1431
  try:
1432
    if uid != -1 or gid != -1:
1433
      os.chown(new_name, uid, gid)
1434
    if mode:
1435
      os.chmod(new_name, mode)
1436
    if callable(prewrite):
1437
      prewrite(fd)
1438
    if data is not None:
1439
      os.write(fd, data)
1440
    else:
1441
      fn(fd)
1442
    if callable(postwrite):
1443
      postwrite(fd)
1444
    os.fsync(fd)
1445
    if atime is not None and mtime is not None:
1446
      os.utime(new_name, (atime, mtime))
1447
    if not dry_run:
1448
      os.rename(new_name, file_name)
1449
      do_remove = False
1450
  finally:
1451
    if close:
1452
      os.close(fd)
1453
      result = None
1454
    else:
1455
      result = fd
1456
    if do_remove:
1457
      RemoveFile(new_name)
1458

    
1459
  return result
1460

    
1461

    
1462
def FirstFree(seq, base=0):
1463
  """Returns the first non-existing integer from seq.
1464

1465
  The seq argument should be a sorted list of positive integers. The
1466
  first time the index of an element is smaller than the element
1467
  value, the index will be returned.
1468

1469
  The base argument is used to start at a different offset,
1470
  i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1471

1472
  Example: C{[0, 1, 3]} will return I{2}.
1473

1474
  @type seq: sequence
1475
  @param seq: the sequence to be analyzed.
1476
  @type base: int
1477
  @param base: use this value as the base index of the sequence
1478
  @rtype: int
1479
  @return: the first non-used index in the sequence
1480

1481
  """
1482
  for idx, elem in enumerate(seq):
1483
    assert elem >= base, "Passed element is higher than base offset"
1484
    if elem > idx + base:
1485
      # idx is not used
1486
      return idx + base
1487
  return None
1488

    
1489

    
1490
def SingleWaitForFdCondition(fdobj, event, timeout):
1491
  """Waits for a condition to occur on the socket.
1492

1493
  Immediately returns at the first interruption.
1494

1495
  @type fdobj: integer or object supporting a fileno() method
1496
  @param fdobj: entity to wait for events on
1497
  @type event: integer
1498
  @param event: ORed condition (see select module)
1499
  @type timeout: float or None
1500
  @param timeout: Timeout in seconds
1501
  @rtype: int or None
1502
  @return: None for timeout, otherwise occured conditions
1503

1504
  """
1505
  check = (event | select.POLLPRI |
1506
           select.POLLNVAL | select.POLLHUP | select.POLLERR)
1507

    
1508
  if timeout is not None:
1509
    # Poller object expects milliseconds
1510
    timeout *= 1000
1511

    
1512
  poller = select.poll()
1513
  poller.register(fdobj, event)
1514
  try:
1515
    # TODO: If the main thread receives a signal and we have no timeout, we
1516
    # could wait forever. This should check a global "quit" flag or something
1517
    # every so often.
1518
    io_events = poller.poll(timeout)
1519
  except select.error, err:
1520
    if err[0] != errno.EINTR:
1521
      raise
1522
    io_events = []
1523
  if io_events and io_events[0][1] & check:
1524
    return io_events[0][1]
1525
  else:
1526
    return None
1527

    
1528

    
1529
class FdConditionWaiterHelper(object):
1530
  """Retry helper for WaitForFdCondition.
1531

1532
  This class contains the retried and wait functions that make sure
1533
  WaitForFdCondition can continue waiting until the timeout is actually
1534
  expired.
1535

1536
  """
1537

    
1538
  def __init__(self, timeout):
1539
    self.timeout = timeout
1540

    
1541
  def Poll(self, fdobj, event):
1542
    result = SingleWaitForFdCondition(fdobj, event, self.timeout)
1543
    if result is None:
1544
      raise RetryAgain()
1545
    else:
1546
      return result
1547

    
1548
  def UpdateTimeout(self, timeout):
1549
    self.timeout = timeout
1550

    
1551

    
1552
def WaitForFdCondition(fdobj, event, timeout):
1553
  """Waits for a condition to occur on the socket.
1554

1555
  Retries until the timeout is expired, even if interrupted.
1556

1557
  @type fdobj: integer or object supporting a fileno() method
1558
  @param fdobj: entity to wait for events on
1559
  @type event: integer
1560
  @param event: ORed condition (see select module)
1561
  @type timeout: float or None
1562
  @param timeout: Timeout in seconds
1563
  @rtype: int or None
1564
  @return: None for timeout, otherwise occured conditions
1565

1566
  """
1567
  if timeout is not None:
1568
    retrywaiter = FdConditionWaiterHelper(timeout)
1569
    try:
1570
      result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
1571
                     args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
1572
    except RetryTimeout:
1573
      result = None
1574
  else:
1575
    result = None
1576
    while result is None:
1577
      result = SingleWaitForFdCondition(fdobj, event, timeout)
1578
  return result
1579

    
1580

    
1581
def UniqueSequence(seq):
1582
  """Returns a list with unique elements.
1583

1584
  Element order is preserved.
1585

1586
  @type seq: sequence
1587
  @param seq: the sequence with the source elements
1588
  @rtype: list
1589
  @return: list of unique elements from seq
1590

1591
  """
1592
  seen = set()
1593
  return [i for i in seq if i not in seen and not seen.add(i)]
1594

    
1595

    
1596
def NormalizeAndValidateMac(mac):
1597
  """Normalizes and check if a MAC address is valid.
1598

1599
  Checks whether the supplied MAC address is formally correct, only
1600
  accepts colon separated format. Normalize it to all lower.
1601

1602
  @type mac: str
1603
  @param mac: the MAC to be validated
1604
  @rtype: str
1605
  @return: returns the normalized and validated MAC.
1606

1607
  @raise errors.OpPrereqError: If the MAC isn't valid
1608

1609
  """
1610
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1611
  if not mac_check.match(mac):
1612
    raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1613
                               mac, errors.ECODE_INVAL)
1614

    
1615
  return mac.lower()
1616

    
1617

    
1618
def TestDelay(duration):
1619
  """Sleep for a fixed amount of time.
1620

1621
  @type duration: float
1622
  @param duration: the sleep duration
1623
  @rtype: boolean
1624
  @return: False for negative value, True otherwise
1625

1626
  """
1627
  if duration < 0:
1628
    return False, "Invalid sleep duration"
1629
  time.sleep(duration)
1630
  return True, None
1631

    
1632

    
1633
def _CloseFDNoErr(fd, retries=5):
1634
  """Close a file descriptor ignoring errors.
1635

1636
  @type fd: int
1637
  @param fd: the file descriptor
1638
  @type retries: int
1639
  @param retries: how many retries to make, in case we get any
1640
      other error than EBADF
1641

1642
  """
1643
  try:
1644
    os.close(fd)
1645
  except OSError, err:
1646
    if err.errno != errno.EBADF:
1647
      if retries > 0:
1648
        _CloseFDNoErr(fd, retries - 1)
1649
    # else either it's closed already or we're out of retries, so we
1650
    # ignore this and go on
1651

    
1652

    
1653
def CloseFDs(noclose_fds=None):
1654
  """Close file descriptors.
1655

1656
  This closes all file descriptors above 2 (i.e. except
1657
  stdin/out/err).
1658

1659
  @type noclose_fds: list or None
1660
  @param noclose_fds: if given, it denotes a list of file descriptor
1661
      that should not be closed
1662

1663
  """
1664
  # Default maximum for the number of available file descriptors.
1665
  if 'SC_OPEN_MAX' in os.sysconf_names:
1666
    try:
1667
      MAXFD = os.sysconf('SC_OPEN_MAX')
1668
      if MAXFD < 0:
1669
        MAXFD = 1024
1670
    except OSError:
1671
      MAXFD = 1024
1672
  else:
1673
    MAXFD = 1024
1674
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1675
  if (maxfd == resource.RLIM_INFINITY):
1676
    maxfd = MAXFD
1677

    
1678
  # Iterate through and close all file descriptors (except the standard ones)
1679
  for fd in range(3, maxfd):
1680
    if noclose_fds and fd in noclose_fds:
1681
      continue
1682
    _CloseFDNoErr(fd)
1683

    
1684

    
1685
def Daemonize(logfile):
1686
  """Daemonize the current process.
1687

1688
  This detaches the current process from the controlling terminal and
1689
  runs it in the background as a daemon.
1690

1691
  @type logfile: str
1692
  @param logfile: the logfile to which we should redirect stdout/stderr
1693
  @rtype: int
1694
  @return: the value zero
1695

1696
  """
1697
  # pylint: disable-msg=W0212
1698
  # yes, we really want os._exit
1699
  UMASK = 077
1700
  WORKDIR = "/"
1701

    
1702
  # this might fail
1703
  pid = os.fork()
1704
  if (pid == 0):  # The first child.
1705
    os.setsid()
1706
    # this might fail
1707
    pid = os.fork() # Fork a second child.
1708
    if (pid == 0):  # The second child.
1709
      os.chdir(WORKDIR)
1710
      os.umask(UMASK)
1711
    else:
1712
      # exit() or _exit()?  See below.
1713
      os._exit(0) # Exit parent (the first child) of the second child.
1714
  else:
1715
    os._exit(0) # Exit parent of the first child.
1716

    
1717
  for fd in range(3):
1718
    _CloseFDNoErr(fd)
1719
  i = os.open("/dev/null", os.O_RDONLY) # stdin
1720
  assert i == 0, "Can't close/reopen stdin"
1721
  i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1722
  assert i == 1, "Can't close/reopen stdout"
1723
  # Duplicate standard output to standard error.
1724
  os.dup2(1, 2)
1725
  return 0
1726

    
1727

    
1728
def DaemonPidFileName(name):
1729
  """Compute a ganeti pid file absolute path
1730

1731
  @type name: str
1732
  @param name: the daemon name
1733
  @rtype: str
1734
  @return: the full path to the pidfile corresponding to the given
1735
      daemon name
1736

1737
  """
1738
  return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
1739

    
1740

    
1741
def EnsureDaemon(name):
1742
  """Check for and start daemon if not alive.
1743

1744
  """
1745
  result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
1746
  if result.failed:
1747
    logging.error("Can't start daemon '%s', failure %s, output: %s",
1748
                  name, result.fail_reason, result.output)
1749
    return False
1750

    
1751
  return True
1752

    
1753

    
1754
def WritePidFile(name):
1755
  """Write the current process pidfile.
1756

1757
  The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1758

1759
  @type name: str
1760
  @param name: the daemon name to use
1761
  @raise errors.GenericError: if the pid file already exists and
1762
      points to a live process
1763

1764
  """
1765
  pid = os.getpid()
1766
  pidfilename = DaemonPidFileName(name)
1767
  if IsProcessAlive(ReadPidFile(pidfilename)):
1768
    raise errors.GenericError("%s contains a live process" % pidfilename)
1769

    
1770
  WriteFile(pidfilename, data="%d\n" % pid)
1771

    
1772

    
1773
def RemovePidFile(name):
1774
  """Remove the current process pidfile.
1775

1776
  Any errors are ignored.
1777

1778
  @type name: str
1779
  @param name: the daemon name used to derive the pidfile name
1780

1781
  """
1782
  pidfilename = DaemonPidFileName(name)
1783
  # TODO: we could check here that the file contains our pid
1784
  try:
1785
    RemoveFile(pidfilename)
1786
  except: # pylint: disable-msg=W0702
1787
    pass
1788

    
1789

    
1790
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1791
                waitpid=False):
1792
  """Kill a process given by its pid.
1793

1794
  @type pid: int
1795
  @param pid: The PID to terminate.
1796
  @type signal_: int
1797
  @param signal_: The signal to send, by default SIGTERM
1798
  @type timeout: int
1799
  @param timeout: The timeout after which, if the process is still alive,
1800
                  a SIGKILL will be sent. If not positive, no such checking
1801
                  will be done
1802
  @type waitpid: boolean
1803
  @param waitpid: If true, we should waitpid on this process after
1804
      sending signals, since it's our own child and otherwise it
1805
      would remain as zombie
1806

1807
  """
1808
  def _helper(pid, signal_, wait):
1809
    """Simple helper to encapsulate the kill/waitpid sequence"""
1810
    os.kill(pid, signal_)
1811
    if wait:
1812
      try:
1813
        os.waitpid(pid, os.WNOHANG)
1814
      except OSError:
1815
        pass
1816

    
1817
  if pid <= 0:
1818
    # kill with pid=0 == suicide
1819
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1820

    
1821
  if not IsProcessAlive(pid):
1822
    return
1823

    
1824
  _helper(pid, signal_, waitpid)
1825

    
1826
  if timeout <= 0:
1827
    return
1828

    
1829
  def _CheckProcess():
1830
    if not IsProcessAlive(pid):
1831
      return
1832

    
1833
    try:
1834
      (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1835
    except OSError:
1836
      raise RetryAgain()
1837

    
1838
    if result_pid > 0:
1839
      return
1840

    
1841
    raise RetryAgain()
1842

    
1843
  try:
1844
    # Wait up to $timeout seconds
1845
    Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1846
  except RetryTimeout:
1847
    pass
1848

    
1849
  if IsProcessAlive(pid):
1850
    # Kill process if it's still alive
1851
    _helper(pid, signal.SIGKILL, waitpid)
1852

    
1853

    
1854
def FindFile(name, search_path, test=os.path.exists):
1855
  """Look for a filesystem object in a given path.
1856

1857
  This is an abstract method to search for filesystem object (files,
1858
  dirs) under a given search path.
1859

1860
  @type name: str
1861
  @param name: the name to look for
1862
  @type search_path: str
1863
  @param search_path: location to start at
1864
  @type test: callable
1865
  @param test: a function taking one argument that should return True
1866
      if the a given object is valid; the default value is
1867
      os.path.exists, causing only existing files to be returned
1868
  @rtype: str or None
1869
  @return: full path to the object if found, None otherwise
1870

1871
  """
1872
  # validate the filename mask
1873
  if constants.EXT_PLUGIN_MASK.match(name) is None:
1874
    logging.critical("Invalid value passed for external script name: '%s'",
1875
                     name)
1876
    return None
1877

    
1878
  for dir_name in search_path:
1879
    # FIXME: investigate switch to PathJoin
1880
    item_name = os.path.sep.join([dir_name, name])
1881
    # check the user test and that we're indeed resolving to the given
1882
    # basename
1883
    if test(item_name) and os.path.basename(item_name) == name:
1884
      return item_name
1885
  return None
1886

    
1887

    
1888
def CheckVolumeGroupSize(vglist, vgname, minsize):
1889
  """Checks if the volume group list is valid.
1890

1891
  The function will check if a given volume group is in the list of
1892
  volume groups and has a minimum size.
1893

1894
  @type vglist: dict
1895
  @param vglist: dictionary of volume group names and their size
1896
  @type vgname: str
1897
  @param vgname: the volume group we should check
1898
  @type minsize: int
1899
  @param minsize: the minimum size we accept
1900
  @rtype: None or str
1901
  @return: None for success, otherwise the error message
1902

1903
  """
1904
  vgsize = vglist.get(vgname, None)
1905
  if vgsize is None:
1906
    return "volume group '%s' missing" % vgname
1907
  elif vgsize < minsize:
1908
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1909
            (vgname, minsize, vgsize))
1910
  return None
1911

    
1912

    
1913
def SplitTime(value):
1914
  """Splits time as floating point number into a tuple.
1915

1916
  @param value: Time in seconds
1917
  @type value: int or float
1918
  @return: Tuple containing (seconds, microseconds)
1919

1920
  """
1921
  (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1922

    
1923
  assert 0 <= seconds, \
1924
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1925
  assert 0 <= microseconds <= 999999, \
1926
    "Microseconds must be 0-999999, but are %s" % microseconds
1927

    
1928
  return (int(seconds), int(microseconds))
1929

    
1930

    
1931
def MergeTime(timetuple):
1932
  """Merges a tuple into time as a floating point number.
1933

1934
  @param timetuple: Time as tuple, (seconds, microseconds)
1935
  @type timetuple: tuple
1936
  @return: Time as a floating point number expressed in seconds
1937

1938
  """
1939
  (seconds, microseconds) = timetuple
1940

    
1941
  assert 0 <= seconds, \
1942
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1943
  assert 0 <= microseconds <= 999999, \
1944
    "Microseconds must be 0-999999, but are %s" % microseconds
1945

    
1946
  return float(seconds) + (float(microseconds) * 0.000001)
1947

    
1948

    
1949
def GetDaemonPort(daemon_name):
1950
  """Get the daemon port for this cluster.
1951

1952
  Note that this routine does not read a ganeti-specific file, but
1953
  instead uses C{socket.getservbyname} to allow pre-customization of
1954
  this parameter outside of Ganeti.
1955

1956
  @type daemon_name: string
1957
  @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1958
  @rtype: int
1959

1960
  """
1961
  if daemon_name not in constants.DAEMONS_PORTS:
1962
    raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1963

    
1964
  (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1965
  try:
1966
    port = socket.getservbyname(daemon_name, proto)
1967
  except socket.error:
1968
    port = default_port
1969

    
1970
  return port
1971

    
1972

    
1973
def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
1974
                 multithreaded=False, syslog=constants.SYSLOG_USAGE):
1975
  """Configures the logging module.
1976

1977
  @type logfile: str
1978
  @param logfile: the filename to which we should log
1979
  @type debug: integer
1980
  @param debug: if greater than zero, enable debug messages, otherwise
1981
      only those at C{INFO} and above level
1982
  @type stderr_logging: boolean
1983
  @param stderr_logging: whether we should also log to the standard error
1984
  @type program: str
1985
  @param program: the name under which we should log messages
1986
  @type multithreaded: boolean
1987
  @param multithreaded: if True, will add the thread name to the log file
1988
  @type syslog: string
1989
  @param syslog: one of 'no', 'yes', 'only':
1990
      - if no, syslog is not used
1991
      - if yes, syslog is used (in addition to file-logging)
1992
      - if only, only syslog is used
1993
  @raise EnvironmentError: if we can't open the log file and
1994
      syslog/stderr logging is disabled
1995

1996
  """
1997
  fmt = "%(asctime)s: " + program + " pid=%(process)d"
1998
  sft = program + "[%(process)d]:"
1999
  if multithreaded:
2000
    fmt += "/%(threadName)s"
2001
    sft += " (%(threadName)s)"
2002
  if debug:
2003
    fmt += " %(module)s:%(lineno)s"
2004
    # no debug info for syslog loggers
2005
  fmt += " %(levelname)s %(message)s"
2006
  # yes, we do want the textual level, as remote syslog will probably
2007
  # lose the error level, and it's easier to grep for it
2008
  sft += " %(levelname)s %(message)s"
2009
  formatter = logging.Formatter(fmt)
2010
  sys_fmt = logging.Formatter(sft)
2011

    
2012
  root_logger = logging.getLogger("")
2013
  root_logger.setLevel(logging.NOTSET)
2014

    
2015
  # Remove all previously setup handlers
2016
  for handler in root_logger.handlers:
2017
    handler.close()
2018
    root_logger.removeHandler(handler)
2019

    
2020
  if stderr_logging:
2021
    stderr_handler = logging.StreamHandler()
2022
    stderr_handler.setFormatter(formatter)
2023
    if debug:
2024
      stderr_handler.setLevel(logging.NOTSET)
2025
    else:
2026
      stderr_handler.setLevel(logging.CRITICAL)
2027
    root_logger.addHandler(stderr_handler)
2028

    
2029
  if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
2030
    facility = logging.handlers.SysLogHandler.LOG_DAEMON
2031
    syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
2032
                                                    facility)
2033
    syslog_handler.setFormatter(sys_fmt)
2034
    # Never enable debug over syslog
2035
    syslog_handler.setLevel(logging.INFO)
2036
    root_logger.addHandler(syslog_handler)
2037

    
2038
  if syslog != constants.SYSLOG_ONLY:
2039
    # this can fail, if the logging directories are not setup or we have
2040
    # a permisssion problem; in this case, it's best to log but ignore
2041
    # the error if stderr_logging is True, and if false we re-raise the
2042
    # exception since otherwise we could run but without any logs at all
2043
    try:
2044
      logfile_handler = logging.FileHandler(logfile)
2045
      logfile_handler.setFormatter(formatter)
2046
      if debug:
2047
        logfile_handler.setLevel(logging.DEBUG)
2048
      else:
2049
        logfile_handler.setLevel(logging.INFO)
2050
      root_logger.addHandler(logfile_handler)
2051
    except EnvironmentError:
2052
      if stderr_logging or syslog == constants.SYSLOG_YES:
2053
        logging.exception("Failed to enable logging to file '%s'", logfile)
2054
      else:
2055
        # we need to re-raise the exception
2056
        raise
2057

    
2058

    
2059
def IsNormAbsPath(path):
2060
  """Check whether a path is absolute and also normalized
2061

2062
  This avoids things like /dir/../../other/path to be valid.
2063

2064
  """
2065
  return os.path.normpath(path) == path and os.path.isabs(path)
2066

    
2067

    
2068
def PathJoin(*args):
2069
  """Safe-join a list of path components.
2070

2071
  Requirements:
2072
      - the first argument must be an absolute path
2073
      - no component in the path must have backtracking (e.g. /../),
2074
        since we check for normalization at the end
2075

2076
  @param args: the path components to be joined
2077
  @raise ValueError: for invalid paths
2078

2079
  """
2080
  # ensure we're having at least one path passed in
2081
  assert args
2082
  # ensure the first component is an absolute and normalized path name
2083
  root = args[0]
2084
  if not IsNormAbsPath(root):
2085
    raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
2086
  result = os.path.join(*args)
2087
  # ensure that the whole path is normalized
2088
  if not IsNormAbsPath(result):
2089
    raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
2090
  # check that we're still under the original prefix
2091
  prefix = os.path.commonprefix([root, result])
2092
  if prefix != root:
2093
    raise ValueError("Error: path joining resulted in different prefix"
2094
                     " (%s != %s)" % (prefix, root))
2095
  return result
2096

    
2097

    
2098
def TailFile(fname, lines=20):
2099
  """Return the last lines from a file.
2100

2101
  @note: this function will only read and parse the last 4KB of
2102
      the file; if the lines are very long, it could be that less
2103
      than the requested number of lines are returned
2104

2105
  @param fname: the file name
2106
  @type lines: int
2107
  @param lines: the (maximum) number of lines to return
2108

2109
  """
2110
  fd = open(fname, "r")
2111
  try:
2112
    fd.seek(0, 2)
2113
    pos = fd.tell()
2114
    pos = max(0, pos-4096)
2115
    fd.seek(pos, 0)
2116
    raw_data = fd.read()
2117
  finally:
2118
    fd.close()
2119

    
2120
  rows = raw_data.splitlines()
2121
  return rows[-lines:]
2122

    
2123

    
2124
def _ParseAsn1Generalizedtime(value):
2125
  """Parses an ASN1 GENERALIZEDTIME timestamp as used by pyOpenSSL.
2126

2127
  @type value: string
2128
  @param value: ASN1 GENERALIZEDTIME timestamp
2129

2130
  """
2131
  m = re.match(r"^(\d+)([-+]\d\d)(\d\d)$", value)
2132
  if m:
2133
    # We have an offset
2134
    asn1time = m.group(1)
2135
    hours = int(m.group(2))
2136
    minutes = int(m.group(3))
2137
    utcoffset = (60 * hours) + minutes
2138
  else:
2139
    if not value.endswith("Z"):
2140
      raise ValueError("Missing timezone")
2141
    asn1time = value[:-1]
2142
    utcoffset = 0
2143

    
2144
  parsed = time.strptime(asn1time, "%Y%m%d%H%M%S")
2145

    
2146
  tt = datetime.datetime(*(parsed[:7])) - datetime.timedelta(minutes=utcoffset)
2147

    
2148
  return calendar.timegm(tt.utctimetuple())
2149

    
2150

    
2151
def GetX509CertValidity(cert):
2152
  """Returns the validity period of the certificate.
2153

2154
  @type cert: OpenSSL.crypto.X509
2155
  @param cert: X509 certificate object
2156

2157
  """
2158
  # The get_notBefore and get_notAfter functions are only supported in
2159
  # pyOpenSSL 0.7 and above.
2160
  try:
2161
    get_notbefore_fn = cert.get_notBefore
2162
  except AttributeError:
2163
    not_before = None
2164
  else:
2165
    not_before_asn1 = get_notbefore_fn()
2166

    
2167
    if not_before_asn1 is None:
2168
      not_before = None
2169
    else:
2170
      not_before = _ParseAsn1Generalizedtime(not_before_asn1)
2171

    
2172
  try:
2173
    get_notafter_fn = cert.get_notAfter
2174
  except AttributeError:
2175
    not_after = None
2176
  else:
2177
    not_after_asn1 = get_notafter_fn()
2178

    
2179
    if not_after_asn1 is None:
2180
      not_after = None
2181
    else:
2182
      not_after = _ParseAsn1Generalizedtime(not_after_asn1)
2183

    
2184
  return (not_before, not_after)
2185

    
2186

    
2187
def SafeEncode(text):
2188
  """Return a 'safe' version of a source string.
2189

2190
  This function mangles the input string and returns a version that
2191
  should be safe to display/encode as ASCII. To this end, we first
2192
  convert it to ASCII using the 'backslashreplace' encoding which
2193
  should get rid of any non-ASCII chars, and then we process it
2194
  through a loop copied from the string repr sources in the python; we
2195
  don't use string_escape anymore since that escape single quotes and
2196
  backslashes too, and that is too much; and that escaping is not
2197
  stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2198

2199
  @type text: str or unicode
2200
  @param text: input data
2201
  @rtype: str
2202
  @return: a safe version of text
2203

2204
  """
2205
  if isinstance(text, unicode):
2206
    # only if unicode; if str already, we handle it below
2207
    text = text.encode('ascii', 'backslashreplace')
2208
  resu = ""
2209
  for char in text:
2210
    c = ord(char)
2211
    if char  == '\t':
2212
      resu += r'\t'
2213
    elif char == '\n':
2214
      resu += r'\n'
2215
    elif char == '\r':
2216
      resu += r'\'r'
2217
    elif c < 32 or c >= 127: # non-printable
2218
      resu += "\\x%02x" % (c & 0xff)
2219
    else:
2220
      resu += char
2221
  return resu
2222

    
2223

    
2224
def UnescapeAndSplit(text, sep=","):
2225
  """Split and unescape a string based on a given separator.
2226

2227
  This function splits a string based on a separator where the
2228
  separator itself can be escape in order to be an element of the
2229
  elements. The escaping rules are (assuming coma being the
2230
  separator):
2231
    - a plain , separates the elements
2232
    - a sequence \\\\, (double backslash plus comma) is handled as a
2233
      backslash plus a separator comma
2234
    - a sequence \, (backslash plus comma) is handled as a
2235
      non-separator comma
2236

2237
  @type text: string
2238
  @param text: the string to split
2239
  @type sep: string
2240
  @param text: the separator
2241
  @rtype: string
2242
  @return: a list of strings
2243

2244
  """
2245
  # we split the list by sep (with no escaping at this stage)
2246
  slist = text.split(sep)
2247
  # next, we revisit the elements and if any of them ended with an odd
2248
  # number of backslashes, then we join it with the next
2249
  rlist = []
2250
  while slist:
2251
    e1 = slist.pop(0)
2252
    if e1.endswith("\\"):
2253
      num_b = len(e1) - len(e1.rstrip("\\"))
2254
      if num_b % 2 == 1:
2255
        e2 = slist.pop(0)
2256
        # here the backslashes remain (all), and will be reduced in
2257
        # the next step
2258
        rlist.append(e1 + sep + e2)
2259
        continue
2260
    rlist.append(e1)
2261
  # finally, replace backslash-something with something
2262
  rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2263
  return rlist
2264

    
2265

    
2266
def CommaJoin(names):
2267
  """Nicely join a set of identifiers.
2268

2269
  @param names: set, list or tuple
2270
  @return: a string with the formatted results
2271

2272
  """
2273
  return ", ".join([str(val) for val in names])
2274

    
2275

    
2276
def BytesToMebibyte(value):
2277
  """Converts bytes to mebibytes.
2278

2279
  @type value: int
2280
  @param value: Value in bytes
2281
  @rtype: int
2282
  @return: Value in mebibytes
2283

2284
  """
2285
  return int(round(value / (1024.0 * 1024.0), 0))
2286

    
2287

    
2288
def CalculateDirectorySize(path):
2289
  """Calculates the size of a directory recursively.
2290

2291
  @type path: string
2292
  @param path: Path to directory
2293
  @rtype: int
2294
  @return: Size in mebibytes
2295

2296
  """
2297
  size = 0
2298

    
2299
  for (curpath, _, files) in os.walk(path):
2300
    for filename in files:
2301
      st = os.lstat(PathJoin(curpath, filename))
2302
      size += st.st_size
2303

    
2304
  return BytesToMebibyte(size)
2305

    
2306

    
2307
def GetFilesystemStats(path):
2308
  """Returns the total and free space on a filesystem.
2309

2310
  @type path: string
2311
  @param path: Path on filesystem to be examined
2312
  @rtype: int
2313
  @return: tuple of (Total space, Free space) in mebibytes
2314

2315
  """
2316
  st = os.statvfs(path)
2317

    
2318
  fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2319
  tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2320
  return (tsize, fsize)
2321

    
2322

    
2323
def RunInSeparateProcess(fn, *args):
2324
  """Runs a function in a separate process.
2325

2326
  Note: Only boolean return values are supported.
2327

2328
  @type fn: callable
2329
  @param fn: Function to be called
2330
  @rtype: bool
2331
  @return: Function's result
2332

2333
  """
2334
  pid = os.fork()
2335
  if pid == 0:
2336
    # Child process
2337
    try:
2338
      # In case the function uses temporary files
2339
      ResetTempfileModule()
2340

    
2341
      # Call function
2342
      result = int(bool(fn(*args)))
2343
      assert result in (0, 1)
2344
    except: # pylint: disable-msg=W0702
2345
      logging.exception("Error while calling function in separate process")
2346
      # 0 and 1 are reserved for the return value
2347
      result = 33
2348

    
2349
    os._exit(result) # pylint: disable-msg=W0212
2350

    
2351
  # Parent process
2352

    
2353
  # Avoid zombies and check exit code
2354
  (_, status) = os.waitpid(pid, 0)
2355

    
2356
  if os.WIFSIGNALED(status):
2357
    exitcode = None
2358
    signum = os.WTERMSIG(status)
2359
  else:
2360
    exitcode = os.WEXITSTATUS(status)
2361
    signum = None
2362

    
2363
  if not (exitcode in (0, 1) and signum is None):
2364
    raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2365
                              (exitcode, signum))
2366

    
2367
  return bool(exitcode)
2368

    
2369

    
2370
def LockedMethod(fn):
2371
  """Synchronized object access decorator.
2372

2373
  This decorator is intended to protect access to an object using the
2374
  object's own lock which is hardcoded to '_lock'.
2375

2376
  """
2377
  def _LockDebug(*args, **kwargs):
2378
    if debug_locks:
2379
      logging.debug(*args, **kwargs)
2380

    
2381
  def wrapper(self, *args, **kwargs):
2382
    # pylint: disable-msg=W0212
2383
    assert hasattr(self, '_lock')
2384
    lock = self._lock
2385
    _LockDebug("Waiting for %s", lock)
2386
    lock.acquire()
2387
    try:
2388
      _LockDebug("Acquired %s", lock)
2389
      result = fn(self, *args, **kwargs)
2390
    finally:
2391
      _LockDebug("Releasing %s", lock)
2392
      lock.release()
2393
      _LockDebug("Released %s", lock)
2394
    return result
2395
  return wrapper
2396

    
2397

    
2398
def LockFile(fd):
2399
  """Locks a file using POSIX locks.
2400

2401
  @type fd: int
2402
  @param fd: the file descriptor we need to lock
2403

2404
  """
2405
  try:
2406
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2407
  except IOError, err:
2408
    if err.errno == errno.EAGAIN:
2409
      raise errors.LockError("File already locked")
2410
    raise
2411

    
2412

    
2413
def FormatTime(val):
2414
  """Formats a time value.
2415

2416
  @type val: float or None
2417
  @param val: the timestamp as returned by time.time()
2418
  @return: a string value or N/A if we don't have a valid timestamp
2419

2420
  """
2421
  if val is None or not isinstance(val, (int, float)):
2422
    return "N/A"
2423
  # these two codes works on Linux, but they are not guaranteed on all
2424
  # platforms
2425
  return time.strftime("%F %T", time.localtime(val))
2426

    
2427

    
2428
def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2429
  """Reads the watcher pause file.
2430

2431
  @type filename: string
2432
  @param filename: Path to watcher pause file
2433
  @type now: None, float or int
2434
  @param now: Current time as Unix timestamp
2435
  @type remove_after: int
2436
  @param remove_after: Remove watcher pause file after specified amount of
2437
    seconds past the pause end time
2438

2439
  """
2440
  if now is None:
2441
    now = time.time()
2442

    
2443
  try:
2444
    value = ReadFile(filename)
2445
  except IOError, err:
2446
    if err.errno != errno.ENOENT:
2447
      raise
2448
    value = None
2449

    
2450
  if value is not None:
2451
    try:
2452
      value = int(value)
2453
    except ValueError:
2454
      logging.warning(("Watcher pause file (%s) contains invalid value,"
2455
                       " removing it"), filename)
2456
      RemoveFile(filename)
2457
      value = None
2458

    
2459
    if value is not None:
2460
      # Remove file if it's outdated
2461
      if now > (value + remove_after):
2462
        RemoveFile(filename)
2463
        value = None
2464

    
2465
      elif now > value:
2466
        value = None
2467

    
2468
  return value
2469

    
2470

    
2471
class RetryTimeout(Exception):
2472
  """Retry loop timed out.
2473

2474
  """
2475

    
2476

    
2477
class RetryAgain(Exception):
2478
  """Retry again.
2479

2480
  """
2481

    
2482

    
2483
class _RetryDelayCalculator(object):
2484
  """Calculator for increasing delays.
2485

2486
  """
2487
  __slots__ = [
2488
    "_factor",
2489
    "_limit",
2490
    "_next",
2491
    "_start",
2492
    ]
2493

    
2494
  def __init__(self, start, factor, limit):
2495
    """Initializes this class.
2496

2497
    @type start: float
2498
    @param start: Initial delay
2499
    @type factor: float
2500
    @param factor: Factor for delay increase
2501
    @type limit: float or None
2502
    @param limit: Upper limit for delay or None for no limit
2503

2504
    """
2505
    assert start > 0.0
2506
    assert factor >= 1.0
2507
    assert limit is None or limit >= 0.0
2508

    
2509
    self._start = start
2510
    self._factor = factor
2511
    self._limit = limit
2512

    
2513
    self._next = start
2514

    
2515
  def __call__(self):
2516
    """Returns current delay and calculates the next one.
2517

2518
    """
2519
    current = self._next
2520

    
2521
    # Update for next run
2522
    if self._limit is None or self._next < self._limit:
2523
      self._next = min(self._limit, self._next * self._factor)
2524

    
2525
    return current
2526

    
2527

    
2528
#: Special delay to specify whole remaining timeout
2529
RETRY_REMAINING_TIME = object()
2530

    
2531

    
2532
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2533
          _time_fn=time.time):
2534
  """Call a function repeatedly until it succeeds.
2535

2536
  The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2537
  anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2538
  total of C{timeout} seconds, this function throws L{RetryTimeout}.
2539

2540
  C{delay} can be one of the following:
2541
    - callable returning the delay length as a float
2542
    - Tuple of (start, factor, limit)
2543
    - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2544
      useful when overriding L{wait_fn} to wait for an external event)
2545
    - A static delay as a number (int or float)
2546

2547
  @type fn: callable
2548
  @param fn: Function to be called
2549
  @param delay: Either a callable (returning the delay), a tuple of (start,
2550
                factor, limit) (see L{_RetryDelayCalculator}),
2551
                L{RETRY_REMAINING_TIME} or a number (int or float)
2552
  @type timeout: float
2553
  @param timeout: Total timeout
2554
  @type wait_fn: callable
2555
  @param wait_fn: Waiting function
2556
  @return: Return value of function
2557

2558
  """
2559
  assert callable(fn)
2560
  assert callable(wait_fn)
2561
  assert callable(_time_fn)
2562

    
2563
  if args is None:
2564
    args = []
2565

    
2566
  end_time = _time_fn() + timeout
2567

    
2568
  if callable(delay):
2569
    # External function to calculate delay
2570
    calc_delay = delay
2571

    
2572
  elif isinstance(delay, (tuple, list)):
2573
    # Increasing delay with optional upper boundary
2574
    (start, factor, limit) = delay
2575
    calc_delay = _RetryDelayCalculator(start, factor, limit)
2576

    
2577
  elif delay is RETRY_REMAINING_TIME:
2578
    # Always use the remaining time
2579
    calc_delay = None
2580

    
2581
  else:
2582
    # Static delay
2583
    calc_delay = lambda: delay
2584

    
2585
  assert calc_delay is None or callable(calc_delay)
2586

    
2587
  while True:
2588
    try:
2589
      # pylint: disable-msg=W0142
2590
      return fn(*args)
2591
    except RetryAgain:
2592
      pass
2593
    except RetryTimeout:
2594
      raise errors.ProgrammerError("Nested retry loop detected that didn't"
2595
                                   " handle RetryTimeout")
2596

    
2597
    remaining_time = end_time - _time_fn()
2598

    
2599
    if remaining_time < 0.0:
2600
      raise RetryTimeout()
2601

    
2602
    assert remaining_time >= 0.0
2603

    
2604
    if calc_delay is None:
2605
      wait_fn(remaining_time)
2606
    else:
2607
      current_delay = calc_delay()
2608
      if current_delay > 0.0:
2609
        wait_fn(current_delay)
2610

    
2611

    
2612
class FileLock(object):
2613
  """Utility class for file locks.
2614

2615
  """
2616
  def __init__(self, fd, filename):
2617
    """Constructor for FileLock.
2618

2619
    @type fd: file
2620
    @param fd: File object
2621
    @type filename: str
2622
    @param filename: Path of the file opened at I{fd}
2623

2624
    """
2625
    self.fd = fd
2626
    self.filename = filename
2627

    
2628
  @classmethod
2629
  def Open(cls, filename):
2630
    """Creates and opens a file to be used as a file-based lock.
2631

2632
    @type filename: string
2633
    @param filename: path to the file to be locked
2634

2635
    """
2636
    # Using "os.open" is necessary to allow both opening existing file
2637
    # read/write and creating if not existing. Vanilla "open" will truncate an
2638
    # existing file -or- allow creating if not existing.
2639
    return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
2640
               filename)
2641

    
2642
  def __del__(self):
2643
    self.Close()
2644

    
2645
  def Close(self):
2646
    """Close the file and release the lock.
2647

2648
    """
2649
    if hasattr(self, "fd") and self.fd:
2650
      self.fd.close()
2651
      self.fd = None
2652

    
2653
  def _flock(self, flag, blocking, timeout, errmsg):
2654
    """Wrapper for fcntl.flock.
2655

2656
    @type flag: int
2657
    @param flag: operation flag
2658
    @type blocking: bool
2659
    @param blocking: whether the operation should be done in blocking mode.
2660
    @type timeout: None or float
2661
    @param timeout: for how long the operation should be retried (implies
2662
                    non-blocking mode).
2663
    @type errmsg: string
2664
    @param errmsg: error message in case operation fails.
2665

2666
    """
2667
    assert self.fd, "Lock was closed"
2668
    assert timeout is None or timeout >= 0, \
2669
      "If specified, timeout must be positive"
2670
    assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
2671

    
2672
    # When a timeout is used, LOCK_NB must always be set
2673
    if not (timeout is None and blocking):
2674
      flag |= fcntl.LOCK_NB
2675

    
2676
    if timeout is None:
2677
      self._Lock(self.fd, flag, timeout)
2678
    else:
2679
      try:
2680
        Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
2681
              args=(self.fd, flag, timeout))
2682
      except RetryTimeout:
2683
        raise errors.LockError(errmsg)
2684

    
2685
  @staticmethod
2686
  def _Lock(fd, flag, timeout):
2687
    try:
2688
      fcntl.flock(fd, flag)
2689
    except IOError, err:
2690
      if timeout is not None and err.errno == errno.EAGAIN:
2691
        raise RetryAgain()
2692

    
2693
      logging.exception("fcntl.flock failed")
2694
      raise
2695

    
2696
  def Exclusive(self, blocking=False, timeout=None):
2697
    """Locks the file in exclusive mode.
2698

2699
    @type blocking: boolean
2700
    @param blocking: whether to block and wait until we
2701
        can lock the file or return immediately
2702
    @type timeout: int or None
2703
    @param timeout: if not None, the duration to wait for the lock
2704
        (in blocking mode)
2705

2706
    """
2707
    self._flock(fcntl.LOCK_EX, blocking, timeout,
2708
                "Failed to lock %s in exclusive mode" % self.filename)
2709

    
2710
  def Shared(self, blocking=False, timeout=None):
2711
    """Locks the file in shared mode.
2712

2713
    @type blocking: boolean
2714
    @param blocking: whether to block and wait until we
2715
        can lock the file or return immediately
2716
    @type timeout: int or None
2717
    @param timeout: if not None, the duration to wait for the lock
2718
        (in blocking mode)
2719

2720
    """
2721
    self._flock(fcntl.LOCK_SH, blocking, timeout,
2722
                "Failed to lock %s in shared mode" % self.filename)
2723

    
2724
  def Unlock(self, blocking=True, timeout=None):
2725
    """Unlocks the file.
2726

2727
    According to C{flock(2)}, unlocking can also be a nonblocking
2728
    operation::
2729

2730
      To make a non-blocking request, include LOCK_NB with any of the above
2731
      operations.
2732

2733
    @type blocking: boolean
2734
    @param blocking: whether to block and wait until we
2735
        can lock the file or return immediately
2736
    @type timeout: int or None
2737
    @param timeout: if not None, the duration to wait for the lock
2738
        (in blocking mode)
2739

2740
    """
2741
    self._flock(fcntl.LOCK_UN, blocking, timeout,
2742
                "Failed to unlock %s" % self.filename)
2743

    
2744

    
2745
class LineSplitter:
2746
  """Splits data chunks into lines separated by newline.
2747

2748
  Instances provide a file-like interface.
2749

2750
  """
2751
  def __init__(self, line_fn, *args):
2752
    """Initializes this class.
2753

2754
    @type line_fn: callable
2755
    @param line_fn: Function called for each line, first parameter is line
2756
    @param args: Extra arguments for L{line_fn}
2757

2758
    """
2759
    assert callable(line_fn)
2760

    
2761
    if args:
2762
      # Python 2.4 doesn't have functools.partial yet
2763
      self._line_fn = \
2764
        lambda line: line_fn(line, *args) # pylint: disable-msg=W0142
2765
    else:
2766
      self._line_fn = line_fn
2767

    
2768
    self._lines = collections.deque()
2769
    self._buffer = ""
2770

    
2771
  def write(self, data):
2772
    parts = (self._buffer + data).split("\n")
2773
    self._buffer = parts.pop()
2774
    self._lines.extend(parts)
2775

    
2776
  def flush(self):
2777
    while self._lines:
2778
      self._line_fn(self._lines.popleft().rstrip("\r\n"))
2779

    
2780
  def close(self):
2781
    self.flush()
2782
    if self._buffer:
2783
      self._line_fn(self._buffer)
2784

    
2785

    
2786
def SignalHandled(signums):
2787
  """Signal Handled decoration.
2788

2789
  This special decorator installs a signal handler and then calls the target
2790
  function. The function must accept a 'signal_handlers' keyword argument,
2791
  which will contain a dict indexed by signal number, with SignalHandler
2792
  objects as values.
2793

2794
  The decorator can be safely stacked with iself, to handle multiple signals
2795
  with different handlers.
2796

2797
  @type signums: list
2798
  @param signums: signals to intercept
2799

2800
  """
2801
  def wrap(fn):
2802
    def sig_function(*args, **kwargs):
2803
      assert 'signal_handlers' not in kwargs or \
2804
             kwargs['signal_handlers'] is None or \
2805
             isinstance(kwargs['signal_handlers'], dict), \
2806
             "Wrong signal_handlers parameter in original function call"
2807
      if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2808
        signal_handlers = kwargs['signal_handlers']
2809
      else:
2810
        signal_handlers = {}
2811
        kwargs['signal_handlers'] = signal_handlers
2812
      sighandler = SignalHandler(signums)
2813
      try:
2814
        for sig in signums:
2815
          signal_handlers[sig] = sighandler
2816
        return fn(*args, **kwargs)
2817
      finally:
2818
        sighandler.Reset()
2819
    return sig_function
2820
  return wrap
2821

    
2822

    
2823
class SignalHandler(object):
2824
  """Generic signal handler class.
2825

2826
  It automatically restores the original handler when deconstructed or
2827
  when L{Reset} is called. You can either pass your own handler
2828
  function in or query the L{called} attribute to detect whether the
2829
  signal was sent.
2830

2831
  @type signum: list
2832
  @ivar signum: the signals we handle
2833
  @type called: boolean
2834
  @ivar called: tracks whether any of the signals have been raised
2835

2836
  """
2837
  def __init__(self, signum):
2838
    """Constructs a new SignalHandler instance.
2839

2840
    @type signum: int or list of ints
2841
    @param signum: Single signal number or set of signal numbers
2842

2843
    """
2844
    self.signum = set(signum)
2845
    self.called = False
2846

    
2847
    self._previous = {}
2848
    try:
2849
      for signum in self.signum:
2850
        # Setup handler
2851
        prev_handler = signal.signal(signum, self._HandleSignal)
2852
        try:
2853
          self._previous[signum] = prev_handler
2854
        except:
2855
          # Restore previous handler
2856
          signal.signal(signum, prev_handler)
2857
          raise
2858
    except:
2859
      # Reset all handlers
2860
      self.Reset()
2861
      # Here we have a race condition: a handler may have already been called,
2862
      # but there's not much we can do about it at this point.
2863
      raise
2864

    
2865
  def __del__(self):
2866
    self.Reset()
2867

    
2868
  def Reset(self):
2869
    """Restore previous handler.
2870

2871
    This will reset all the signals to their previous handlers.
2872

2873
    """
2874
    for signum, prev_handler in self._previous.items():
2875
      signal.signal(signum, prev_handler)
2876
      # If successful, remove from dict
2877
      del self._previous[signum]
2878

    
2879
  def Clear(self):
2880
    """Unsets the L{called} flag.
2881

2882
    This function can be used in case a signal may arrive several times.
2883

2884
    """
2885
    self.called = False
2886

    
2887
  # we don't care about arguments, but we leave them named for the future
2888
  def _HandleSignal(self, signum, frame): # pylint: disable-msg=W0613
2889
    """Actual signal handling function.
2890

2891
    """
2892
    # This is not nice and not absolutely atomic, but it appears to be the only
2893
    # solution in Python -- there are no atomic types.
2894
    self.called = True
2895

    
2896

    
2897
class FieldSet(object):
2898
  """A simple field set.
2899

2900
  Among the features are:
2901
    - checking if a string is among a list of static string or regex objects
2902
    - checking if a whole list of string matches
2903
    - returning the matching groups from a regex match
2904

2905
  Internally, all fields are held as regular expression objects.
2906

2907
  """
2908
  def __init__(self, *items):
2909
    self.items = [re.compile("^%s$" % value) for value in items]
2910

    
2911
  def Extend(self, other_set):
2912
    """Extend the field set with the items from another one"""
2913
    self.items.extend(other_set.items)
2914

    
2915
  def Matches(self, field):
2916
    """Checks if a field matches the current set
2917

2918
    @type field: str
2919
    @param field: the string to match
2920
    @return: either None or a regular expression match object
2921

2922
    """
2923
    for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2924
      return m
2925
    return None
2926

    
2927
  def NonMatching(self, items):
2928
    """Returns the list of fields not matching the current set
2929

2930
    @type items: list
2931
    @param items: the list of fields to check
2932
    @rtype: list
2933
    @return: list of non-matching fields
2934

2935
    """
2936
    return [val for val in items if not self.Matches(val)]