Statistics
| Branch: | Tag: | Revision:

root / lib / utils.py @ 2de64672

History | View | Annotate | Download (73.6 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Ganeti utility module.
23

24
This module holds functions that can be used in both daemons (all) and
25
the command line scripts.
26

27
"""
28

    
29

    
30
import os
31
import time
32
import subprocess
33
import re
34
import socket
35
import tempfile
36
import shutil
37
import errno
38
import pwd
39
import itertools
40
import select
41
import fcntl
42
import resource
43
import logging
44
import logging.handlers
45
import signal
46

    
47
from cStringIO import StringIO
48

    
49
try:
50
  from hashlib import sha1
51
except ImportError:
52
  import sha
53
  sha1 = sha.new
54

    
55
from ganeti import errors
56
from ganeti import constants
57

    
58

    
59
_locksheld = []
60
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
61

    
62
debug_locks = False
63

    
64
#: when set to True, L{RunCmd} is disabled
65
no_fork = False
66

    
67
_RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
68

    
69

    
70
class RunResult(object):
71
  """Holds the result of running external programs.
72

73
  @type exit_code: int
74
  @ivar exit_code: the exit code of the program, or None (if the program
75
      didn't exit())
76
  @type signal: int or None
77
  @ivar signal: the signal that caused the program to finish, or None
78
      (if the program wasn't terminated by a signal)
79
  @type stdout: str
80
  @ivar stdout: the standard output of the program
81
  @type stderr: str
82
  @ivar stderr: the standard error of the program
83
  @type failed: boolean
84
  @ivar failed: True in case the program was
85
      terminated by a signal or exited with a non-zero exit code
86
  @ivar fail_reason: a string detailing the termination reason
87

88
  """
89
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
90
               "failed", "fail_reason", "cmd"]
91

    
92

    
93
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
94
    self.cmd = cmd
95
    self.exit_code = exit_code
96
    self.signal = signal_
97
    self.stdout = stdout
98
    self.stderr = stderr
99
    self.failed = (signal_ is not None or exit_code != 0)
100

    
101
    if self.signal is not None:
102
      self.fail_reason = "terminated by signal %s" % self.signal
103
    elif self.exit_code is not None:
104
      self.fail_reason = "exited with exit code %s" % self.exit_code
105
    else:
106
      self.fail_reason = "unable to determine termination reason"
107

    
108
    if self.failed:
109
      logging.debug("Command '%s' failed (%s); output: %s",
110
                    self.cmd, self.fail_reason, self.output)
111

    
112
  def _GetOutput(self):
113
    """Returns the combined stdout and stderr for easier usage.
114

115
    """
116
    return self.stdout + self.stderr
117

    
118
  output = property(_GetOutput, None, None, "Return full output")
119

    
120

    
121
def RunCmd(cmd, env=None, output=None, cwd='/', reset_env=False):
122
  """Execute a (shell) command.
123

124
  The command should not read from its standard input, as it will be
125
  closed.
126

127
  @type cmd: string or list
128
  @param cmd: Command to run
129
  @type env: dict
130
  @param env: Additional environment
131
  @type output: str
132
  @param output: if desired, the output of the command can be
133
      saved in a file instead of the RunResult instance; this
134
      parameter denotes the file name (if not None)
135
  @type cwd: string
136
  @param cwd: if specified, will be used as the working
137
      directory for the command; the default will be /
138
  @type reset_env: boolean
139
  @param reset_env: whether to reset or keep the default os environment
140
  @rtype: L{RunResult}
141
  @return: RunResult instance
142
  @raise errors.ProgrammerError: if we call this when forks are disabled
143

144
  """
145
  if no_fork:
146
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
147

    
148
  if isinstance(cmd, list):
149
    cmd = [str(val) for val in cmd]
150
    strcmd = " ".join(cmd)
151
    shell = False
152
  else:
153
    strcmd = cmd
154
    shell = True
155
  logging.debug("RunCmd '%s'", strcmd)
156

    
157
  if not reset_env:
158
    cmd_env = os.environ.copy()
159
    cmd_env["LC_ALL"] = "C"
160
  else:
161
    cmd_env = {}
162

    
163
  if env is not None:
164
    cmd_env.update(env)
165

    
166
  try:
167
    if output is None:
168
      out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
169
    else:
170
      status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
171
      out = err = ""
172
  except OSError, err:
173
    if err.errno == errno.ENOENT:
174
      raise errors.OpExecError("Can't execute '%s': not found (%s)" %
175
                               (strcmd, err))
176
    else:
177
      raise
178

    
179
  if status >= 0:
180
    exitcode = status
181
    signal_ = None
182
  else:
183
    exitcode = None
184
    signal_ = -status
185

    
186
  return RunResult(exitcode, signal_, out, err, strcmd)
187

    
188

    
189
def _RunCmdPipe(cmd, env, via_shell, cwd):
190
  """Run a command and return its output.
191

192
  @type  cmd: string or list
193
  @param cmd: Command to run
194
  @type env: dict
195
  @param env: The environment to use
196
  @type via_shell: bool
197
  @param via_shell: if we should run via the shell
198
  @type cwd: string
199
  @param cwd: the working directory for the program
200
  @rtype: tuple
201
  @return: (out, err, status)
202

203
  """
204
  poller = select.poll()
205
  child = subprocess.Popen(cmd, shell=via_shell,
206
                           stderr=subprocess.PIPE,
207
                           stdout=subprocess.PIPE,
208
                           stdin=subprocess.PIPE,
209
                           close_fds=True, env=env,
210
                           cwd=cwd)
211

    
212
  child.stdin.close()
213
  poller.register(child.stdout, select.POLLIN)
214
  poller.register(child.stderr, select.POLLIN)
215
  out = StringIO()
216
  err = StringIO()
217
  fdmap = {
218
    child.stdout.fileno(): (out, child.stdout),
219
    child.stderr.fileno(): (err, child.stderr),
220
    }
221
  for fd in fdmap:
222
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
223
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
224

    
225
  while fdmap:
226
    try:
227
      pollresult = poller.poll()
228
    except EnvironmentError, eerr:
229
      if eerr.errno == errno.EINTR:
230
        continue
231
      raise
232
    except select.error, serr:
233
      if serr[0] == errno.EINTR:
234
        continue
235
      raise
236

    
237
    for fd, event in pollresult:
238
      if event & select.POLLIN or event & select.POLLPRI:
239
        data = fdmap[fd][1].read()
240
        # no data from read signifies EOF (the same as POLLHUP)
241
        if not data:
242
          poller.unregister(fd)
243
          del fdmap[fd]
244
          continue
245
        fdmap[fd][0].write(data)
246
      if (event & select.POLLNVAL or event & select.POLLHUP or
247
          event & select.POLLERR):
248
        poller.unregister(fd)
249
        del fdmap[fd]
250

    
251
  out = out.getvalue()
252
  err = err.getvalue()
253

    
254
  status = child.wait()
255
  return out, err, status
256

    
257

    
258
def _RunCmdFile(cmd, env, via_shell, output, cwd):
259
  """Run a command and save its output to a file.
260

261
  @type  cmd: string or list
262
  @param cmd: Command to run
263
  @type env: dict
264
  @param env: The environment to use
265
  @type via_shell: bool
266
  @param via_shell: if we should run via the shell
267
  @type output: str
268
  @param output: the filename in which to save the output
269
  @type cwd: string
270
  @param cwd: the working directory for the program
271
  @rtype: int
272
  @return: the exit status
273

274
  """
275
  fh = open(output, "a")
276
  try:
277
    child = subprocess.Popen(cmd, shell=via_shell,
278
                             stderr=subprocess.STDOUT,
279
                             stdout=fh,
280
                             stdin=subprocess.PIPE,
281
                             close_fds=True, env=env,
282
                             cwd=cwd)
283

    
284
    child.stdin.close()
285
    status = child.wait()
286
  finally:
287
    fh.close()
288
  return status
289

    
290

    
291
def RunParts(dir_name, env=None, reset_env=False):
292
  """Run Scripts or programs in a directory
293

294
  @type dir_name: string
295
  @param dir_name: absolute path to a directory
296
  @type env: dict
297
  @param env: The environment to use
298
  @type reset_env: boolean
299
  @param reset_env: whether to reset or keep the default os environment
300
  @rtype: list of tuples
301
  @return: list of (name, (one of RUNDIR_STATUS), RunResult)
302

303
  """
304
  rr = []
305

    
306
  try:
307
    dir_contents = ListVisibleFiles(dir_name)
308
  except OSError, err:
309
    logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
310
    return rr
311

    
312
  for relname in sorted(dir_contents):
313
    fname = PathJoin(dir_name, relname)
314
    if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
315
            constants.EXT_PLUGIN_MASK.match(relname) is not None):
316
      rr.append((relname, constants.RUNPARTS_SKIP, None))
317
    else:
318
      try:
319
        result = RunCmd([fname], env=env, reset_env=reset_env)
320
      except Exception, err: # pylint: disable-msg=W0703
321
        rr.append((relname, constants.RUNPARTS_ERR, str(err)))
322
      else:
323
        rr.append((relname, constants.RUNPARTS_RUN, result))
324

    
325
  return rr
326

    
327

    
328
def RemoveFile(filename):
329
  """Remove a file ignoring some errors.
330

331
  Remove a file, ignoring non-existing ones or directories. Other
332
  errors are passed.
333

334
  @type filename: str
335
  @param filename: the file to be removed
336

337
  """
338
  try:
339
    os.unlink(filename)
340
  except OSError, err:
341
    if err.errno not in (errno.ENOENT, errno.EISDIR):
342
      raise
343

    
344

    
345
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
346
  """Renames a file.
347

348
  @type old: string
349
  @param old: Original path
350
  @type new: string
351
  @param new: New path
352
  @type mkdir: bool
353
  @param mkdir: Whether to create target directory if it doesn't exist
354
  @type mkdir_mode: int
355
  @param mkdir_mode: Mode for newly created directories
356

357
  """
358
  try:
359
    return os.rename(old, new)
360
  except OSError, err:
361
    # In at least one use case of this function, the job queue, directory
362
    # creation is very rare. Checking for the directory before renaming is not
363
    # as efficient.
364
    if mkdir and err.errno == errno.ENOENT:
365
      # Create directory and try again
366
      dirname = os.path.dirname(new)
367
      try:
368
        os.makedirs(dirname, mode=mkdir_mode)
369
      except OSError, err:
370
        # Ignore EEXIST. This is only handled in os.makedirs as included in
371
        # Python 2.5 and above.
372
        if err.errno != errno.EEXIST or not os.path.exists(dirname):
373
          raise
374

    
375
      return os.rename(old, new)
376

    
377
    raise
378

    
379

    
380
def ResetTempfileModule():
381
  """Resets the random name generator of the tempfile module.
382

383
  This function should be called after C{os.fork} in the child process to
384
  ensure it creates a newly seeded random generator. Otherwise it would
385
  generate the same random parts as the parent process. If several processes
386
  race for the creation of a temporary file, this could lead to one not getting
387
  a temporary name.
388

389
  """
390
  # pylint: disable-msg=W0212
391
  if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
392
    tempfile._once_lock.acquire()
393
    try:
394
      # Reset random name generator
395
      tempfile._name_sequence = None
396
    finally:
397
      tempfile._once_lock.release()
398
  else:
399
    logging.critical("The tempfile module misses at least one of the"
400
                     " '_once_lock' and '_name_sequence' attributes")
401

    
402

    
403
def _FingerprintFile(filename):
404
  """Compute the fingerprint of a file.
405

406
  If the file does not exist, a None will be returned
407
  instead.
408

409
  @type filename: str
410
  @param filename: the filename to checksum
411
  @rtype: str
412
  @return: the hex digest of the sha checksum of the contents
413
      of the file
414

415
  """
416
  if not (os.path.exists(filename) and os.path.isfile(filename)):
417
    return None
418

    
419
  f = open(filename)
420

    
421
  fp = sha1()
422
  while True:
423
    data = f.read(4096)
424
    if not data:
425
      break
426

    
427
    fp.update(data)
428

    
429
  return fp.hexdigest()
430

    
431

    
432
def FingerprintFiles(files):
433
  """Compute fingerprints for a list of files.
434

435
  @type files: list
436
  @param files: the list of filename to fingerprint
437
  @rtype: dict
438
  @return: a dictionary filename: fingerprint, holding only
439
      existing files
440

441
  """
442
  ret = {}
443

    
444
  for filename in files:
445
    cksum = _FingerprintFile(filename)
446
    if cksum:
447
      ret[filename] = cksum
448

    
449
  return ret
450

    
451

    
452
def ForceDictType(target, key_types, allowed_values=None):
453
  """Force the values of a dict to have certain types.
454

455
  @type target: dict
456
  @param target: the dict to update
457
  @type key_types: dict
458
  @param key_types: dict mapping target dict keys to types
459
                    in constants.ENFORCEABLE_TYPES
460
  @type allowed_values: list
461
  @keyword allowed_values: list of specially allowed values
462

463
  """
464
  if allowed_values is None:
465
    allowed_values = []
466

    
467
  if not isinstance(target, dict):
468
    msg = "Expected dictionary, got '%s'" % target
469
    raise errors.TypeEnforcementError(msg)
470

    
471
  for key in target:
472
    if key not in key_types:
473
      msg = "Unknown key '%s'" % key
474
      raise errors.TypeEnforcementError(msg)
475

    
476
    if target[key] in allowed_values:
477
      continue
478

    
479
    ktype = key_types[key]
480
    if ktype not in constants.ENFORCEABLE_TYPES:
481
      msg = "'%s' has non-enforceable type %s" % (key, ktype)
482
      raise errors.ProgrammerError(msg)
483

    
484
    if ktype == constants.VTYPE_STRING:
485
      if not isinstance(target[key], basestring):
486
        if isinstance(target[key], bool) and not target[key]:
487
          target[key] = ''
488
        else:
489
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
490
          raise errors.TypeEnforcementError(msg)
491
    elif ktype == constants.VTYPE_BOOL:
492
      if isinstance(target[key], basestring) and target[key]:
493
        if target[key].lower() == constants.VALUE_FALSE:
494
          target[key] = False
495
        elif target[key].lower() == constants.VALUE_TRUE:
496
          target[key] = True
497
        else:
498
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
499
          raise errors.TypeEnforcementError(msg)
500
      elif target[key]:
501
        target[key] = True
502
      else:
503
        target[key] = False
504
    elif ktype == constants.VTYPE_SIZE:
505
      try:
506
        target[key] = ParseUnit(target[key])
507
      except errors.UnitParseError, err:
508
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
509
              (key, target[key], err)
510
        raise errors.TypeEnforcementError(msg)
511
    elif ktype == constants.VTYPE_INT:
512
      try:
513
        target[key] = int(target[key])
514
      except (ValueError, TypeError):
515
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
516
        raise errors.TypeEnforcementError(msg)
517

    
518

    
519
def IsProcessAlive(pid):
520
  """Check if a given pid exists on the system.
521

522
  @note: zombie status is not handled, so zombie processes
523
      will be returned as alive
524
  @type pid: int
525
  @param pid: the process ID to check
526
  @rtype: boolean
527
  @return: True if the process exists
528

529
  """
530
  if pid <= 0:
531
    return False
532

    
533
  try:
534
    os.stat("/proc/%d/status" % pid)
535
    return True
536
  except EnvironmentError, err:
537
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
538
      return False
539
    raise
540

    
541

    
542
def ReadPidFile(pidfile):
543
  """Read a pid from a file.
544

545
  @type  pidfile: string
546
  @param pidfile: path to the file containing the pid
547
  @rtype: int
548
  @return: The process id, if the file exists and contains a valid PID,
549
           otherwise 0
550

551
  """
552
  try:
553
    raw_data = ReadFile(pidfile)
554
  except EnvironmentError, err:
555
    if err.errno != errno.ENOENT:
556
      logging.exception("Can't read pid file")
557
    return 0
558

    
559
  try:
560
    pid = int(raw_data)
561
  except (TypeError, ValueError), err:
562
    logging.info("Can't parse pid file contents", exc_info=True)
563
    return 0
564

    
565
  return pid
566

    
567

    
568
def MatchNameComponent(key, name_list, case_sensitive=True):
569
  """Try to match a name against a list.
570

571
  This function will try to match a name like test1 against a list
572
  like C{['test1.example.com', 'test2.example.com', ...]}. Against
573
  this list, I{'test1'} as well as I{'test1.example'} will match, but
574
  not I{'test1.ex'}. A multiple match will be considered as no match
575
  at all (e.g. I{'test1'} against C{['test1.example.com',
576
  'test1.example.org']}), except when the key fully matches an entry
577
  (e.g. I{'test1'} against C{['test1', 'test1.example.com']}).
578

579
  @type key: str
580
  @param key: the name to be searched
581
  @type name_list: list
582
  @param name_list: the list of strings against which to search the key
583
  @type case_sensitive: boolean
584
  @param case_sensitive: whether to provide a case-sensitive match
585

586
  @rtype: None or str
587
  @return: None if there is no match I{or} if there are multiple matches,
588
      otherwise the element from the list which matches
589

590
  """
591
  if key in name_list:
592
    return key
593

    
594
  re_flags = 0
595
  if not case_sensitive:
596
    re_flags |= re.IGNORECASE
597
    key = key.upper()
598
  mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
599
  names_filtered = []
600
  string_matches = []
601
  for name in name_list:
602
    if mo.match(name) is not None:
603
      names_filtered.append(name)
604
      if not case_sensitive and key == name.upper():
605
        string_matches.append(name)
606

    
607
  if len(string_matches) == 1:
608
    return string_matches[0]
609
  if len(names_filtered) == 1:
610
    return names_filtered[0]
611
  return None
612

    
613

    
614
class HostInfo:
615
  """Class implementing resolver and hostname functionality
616

617
  """
618
  _VALID_NAME_RE = re.compile("^[a-z0-9._-]{1,255}$")
619

    
620
  def __init__(self, name=None):
621
    """Initialize the host name object.
622

623
    If the name argument is not passed, it will use this system's
624
    name.
625

626
    """
627
    if name is None:
628
      name = self.SysName()
629

    
630
    self.query = name
631
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
632
    self.ip = self.ipaddrs[0]
633

    
634
  def ShortName(self):
635
    """Returns the hostname without domain.
636

637
    """
638
    return self.name.split('.')[0]
639

    
640
  @staticmethod
641
  def SysName():
642
    """Return the current system's name.
643

644
    This is simply a wrapper over C{socket.gethostname()}.
645

646
    """
647
    return socket.gethostname()
648

    
649
  @staticmethod
650
  def LookupHostname(hostname):
651
    """Look up hostname
652

653
    @type hostname: str
654
    @param hostname: hostname to look up
655

656
    @rtype: tuple
657
    @return: a tuple (name, aliases, ipaddrs) as returned by
658
        C{socket.gethostbyname_ex}
659
    @raise errors.ResolverError: in case of errors in resolving
660

661
    """
662
    try:
663
      result = socket.gethostbyname_ex(hostname)
664
    except socket.gaierror, err:
665
      # hostname not found in DNS
666
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
667

    
668
    return result
669

    
670
  @classmethod
671
  def NormalizeName(cls, hostname):
672
    """Validate and normalize the given hostname.
673

674
    @attention: the validation is a bit more relaxed than the standards
675
        require; most importantly, we allow underscores in names
676
    @raise errors.OpPrereqError: when the name is not valid
677

678
    """
679
    hostname = hostname.lower()
680
    if (not cls._VALID_NAME_RE.match(hostname) or
681
        # double-dots, meaning empty label
682
        ".." in hostname or
683
        # empty initial label
684
        hostname.startswith(".")):
685
      raise errors.OpPrereqError("Invalid hostname '%s'" % hostname,
686
                                 errors.ECODE_INVAL)
687
    if hostname.endswith("."):
688
      hostname = hostname.rstrip(".")
689
    return hostname
690

    
691

    
692
def GetHostInfo(name=None):
693
  """Lookup host name and raise an OpPrereqError for failures"""
694

    
695
  try:
696
    return HostInfo(name)
697
  except errors.ResolverError, err:
698
    raise errors.OpPrereqError("The given name (%s) does not resolve: %s" %
699
                               (err[0], err[2]), errors.ECODE_RESOLVER)
700

    
701

    
702
def ListVolumeGroups():
703
  """List volume groups and their size
704

705
  @rtype: dict
706
  @return:
707
       Dictionary with keys volume name and values
708
       the size of the volume
709

710
  """
711
  command = "vgs --noheadings --units m --nosuffix -o name,size"
712
  result = RunCmd(command)
713
  retval = {}
714
  if result.failed:
715
    return retval
716

    
717
  for line in result.stdout.splitlines():
718
    try:
719
      name, size = line.split()
720
      size = int(float(size))
721
    except (IndexError, ValueError), err:
722
      logging.error("Invalid output from vgs (%s): %s", err, line)
723
      continue
724

    
725
    retval[name] = size
726

    
727
  return retval
728

    
729

    
730
def BridgeExists(bridge):
731
  """Check whether the given bridge exists in the system
732

733
  @type bridge: str
734
  @param bridge: the bridge name to check
735
  @rtype: boolean
736
  @return: True if it does
737

738
  """
739
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
740

    
741

    
742
def NiceSort(name_list):
743
  """Sort a list of strings based on digit and non-digit groupings.
744

745
  Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
746
  will sort the list in the logical order C{['a1', 'a2', 'a10',
747
  'a11']}.
748

749
  The sort algorithm breaks each name in groups of either only-digits
750
  or no-digits. Only the first eight such groups are considered, and
751
  after that we just use what's left of the string.
752

753
  @type name_list: list
754
  @param name_list: the names to be sorted
755
  @rtype: list
756
  @return: a copy of the name list sorted with our algorithm
757

758
  """
759
  _SORTER_BASE = "(\D+|\d+)"
760
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
761
                                                  _SORTER_BASE, _SORTER_BASE,
762
                                                  _SORTER_BASE, _SORTER_BASE,
763
                                                  _SORTER_BASE, _SORTER_BASE)
764
  _SORTER_RE = re.compile(_SORTER_FULL)
765
  _SORTER_NODIGIT = re.compile("^\D*$")
766
  def _TryInt(val):
767
    """Attempts to convert a variable to integer."""
768
    if val is None or _SORTER_NODIGIT.match(val):
769
      return val
770
    rval = int(val)
771
    return rval
772

    
773
  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
774
             for name in name_list]
775
  to_sort.sort()
776
  return [tup[1] for tup in to_sort]
777

    
778

    
779
def TryConvert(fn, val):
780
  """Try to convert a value ignoring errors.
781

782
  This function tries to apply function I{fn} to I{val}. If no
783
  C{ValueError} or C{TypeError} exceptions are raised, it will return
784
  the result, else it will return the original value. Any other
785
  exceptions are propagated to the caller.
786

787
  @type fn: callable
788
  @param fn: function to apply to the value
789
  @param val: the value to be converted
790
  @return: The converted value if the conversion was successful,
791
      otherwise the original value.
792

793
  """
794
  try:
795
    nv = fn(val)
796
  except (ValueError, TypeError):
797
    nv = val
798
  return nv
799

    
800

    
801
def IsValidIP(ip):
802
  """Verifies the syntax of an IPv4 address.
803

804
  This function checks if the IPv4 address passes is valid or not based
805
  on syntax (not IP range, class calculations, etc.).
806

807
  @type ip: str
808
  @param ip: the address to be checked
809
  @rtype: a regular expression match object
810
  @return: a regular expression match object, or None if the
811
      address is not valid
812

813
  """
814
  unit = "(0|[1-9]\d{0,2})"
815
  #TODO: convert and return only boolean
816
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
817

    
818

    
819
def IsValidShellParam(word):
820
  """Verifies is the given word is safe from the shell's p.o.v.
821

822
  This means that we can pass this to a command via the shell and be
823
  sure that it doesn't alter the command line and is passed as such to
824
  the actual command.
825

826
  Note that we are overly restrictive here, in order to be on the safe
827
  side.
828

829
  @type word: str
830
  @param word: the word to check
831
  @rtype: boolean
832
  @return: True if the word is 'safe'
833

834
  """
835
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
836

    
837

    
838
def BuildShellCmd(template, *args):
839
  """Build a safe shell command line from the given arguments.
840

841
  This function will check all arguments in the args list so that they
842
  are valid shell parameters (i.e. they don't contain shell
843
  metacharacters). If everything is ok, it will return the result of
844
  template % args.
845

846
  @type template: str
847
  @param template: the string holding the template for the
848
      string formatting
849
  @rtype: str
850
  @return: the expanded command line
851

852
  """
853
  for word in args:
854
    if not IsValidShellParam(word):
855
      raise errors.ProgrammerError("Shell argument '%s' contains"
856
                                   " invalid characters" % word)
857
  return template % args
858

    
859

    
860
def FormatUnit(value, units):
861
  """Formats an incoming number of MiB with the appropriate unit.
862

863
  @type value: int
864
  @param value: integer representing the value in MiB (1048576)
865
  @type units: char
866
  @param units: the type of formatting we should do:
867
      - 'h' for automatic scaling
868
      - 'm' for MiBs
869
      - 'g' for GiBs
870
      - 't' for TiBs
871
  @rtype: str
872
  @return: the formatted value (with suffix)
873

874
  """
875
  if units not in ('m', 'g', 't', 'h'):
876
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
877

    
878
  suffix = ''
879

    
880
  if units == 'm' or (units == 'h' and value < 1024):
881
    if units == 'h':
882
      suffix = 'M'
883
    return "%d%s" % (round(value, 0), suffix)
884

    
885
  elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
886
    if units == 'h':
887
      suffix = 'G'
888
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
889

    
890
  else:
891
    if units == 'h':
892
      suffix = 'T'
893
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
894

    
895

    
896
def ParseUnit(input_string):
897
  """Tries to extract number and scale from the given string.
898

899
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
900
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
901
  is always an int in MiB.
902

903
  """
904
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
905
  if not m:
906
    raise errors.UnitParseError("Invalid format")
907

    
908
  value = float(m.groups()[0])
909

    
910
  unit = m.groups()[1]
911
  if unit:
912
    lcunit = unit.lower()
913
  else:
914
    lcunit = 'm'
915

    
916
  if lcunit in ('m', 'mb', 'mib'):
917
    # Value already in MiB
918
    pass
919

    
920
  elif lcunit in ('g', 'gb', 'gib'):
921
    value *= 1024
922

    
923
  elif lcunit in ('t', 'tb', 'tib'):
924
    value *= 1024 * 1024
925

    
926
  else:
927
    raise errors.UnitParseError("Unknown unit: %s" % unit)
928

    
929
  # Make sure we round up
930
  if int(value) < value:
931
    value += 1
932

    
933
  # Round up to the next multiple of 4
934
  value = int(value)
935
  if value % 4:
936
    value += 4 - value % 4
937

    
938
  return value
939

    
940

    
941
def AddAuthorizedKey(file_name, key):
942
  """Adds an SSH public key to an authorized_keys file.
943

944
  @type file_name: str
945
  @param file_name: path to authorized_keys file
946
  @type key: str
947
  @param key: string containing key
948

949
  """
950
  key_fields = key.split()
951

    
952
  f = open(file_name, 'a+')
953
  try:
954
    nl = True
955
    for line in f:
956
      # Ignore whitespace changes
957
      if line.split() == key_fields:
958
        break
959
      nl = line.endswith('\n')
960
    else:
961
      if not nl:
962
        f.write("\n")
963
      f.write(key.rstrip('\r\n'))
964
      f.write("\n")
965
      f.flush()
966
  finally:
967
    f.close()
968

    
969

    
970
def RemoveAuthorizedKey(file_name, key):
971
  """Removes an SSH public key from an authorized_keys file.
972

973
  @type file_name: str
974
  @param file_name: path to authorized_keys file
975
  @type key: str
976
  @param key: string containing key
977

978
  """
979
  key_fields = key.split()
980

    
981
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
982
  try:
983
    out = os.fdopen(fd, 'w')
984
    try:
985
      f = open(file_name, 'r')
986
      try:
987
        for line in f:
988
          # Ignore whitespace changes while comparing lines
989
          if line.split() != key_fields:
990
            out.write(line)
991

    
992
        out.flush()
993
        os.rename(tmpname, file_name)
994
      finally:
995
        f.close()
996
    finally:
997
      out.close()
998
  except:
999
    RemoveFile(tmpname)
1000
    raise
1001

    
1002

    
1003
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1004
  """Sets the name of an IP address and hostname in /etc/hosts.
1005

1006
  @type file_name: str
1007
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1008
  @type ip: str
1009
  @param ip: the IP address
1010
  @type hostname: str
1011
  @param hostname: the hostname to be added
1012
  @type aliases: list
1013
  @param aliases: the list of aliases to add for the hostname
1014

1015
  """
1016
  # FIXME: use WriteFile + fn rather than duplicating its efforts
1017
  # Ensure aliases are unique
1018
  aliases = UniqueSequence([hostname] + aliases)[1:]
1019

    
1020
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1021
  try:
1022
    out = os.fdopen(fd, 'w')
1023
    try:
1024
      f = open(file_name, 'r')
1025
      try:
1026
        for line in f:
1027
          fields = line.split()
1028
          if fields and not fields[0].startswith('#') and ip == fields[0]:
1029
            continue
1030
          out.write(line)
1031

    
1032
        out.write("%s\t%s" % (ip, hostname))
1033
        if aliases:
1034
          out.write(" %s" % ' '.join(aliases))
1035
        out.write('\n')
1036

    
1037
        out.flush()
1038
        os.fsync(out)
1039
        os.chmod(tmpname, 0644)
1040
        os.rename(tmpname, file_name)
1041
      finally:
1042
        f.close()
1043
    finally:
1044
      out.close()
1045
  except:
1046
    RemoveFile(tmpname)
1047
    raise
1048

    
1049

    
1050
def AddHostToEtcHosts(hostname):
1051
  """Wrapper around SetEtcHostsEntry.
1052

1053
  @type hostname: str
1054
  @param hostname: a hostname that will be resolved and added to
1055
      L{constants.ETC_HOSTS}
1056

1057
  """
1058
  hi = HostInfo(name=hostname)
1059
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
1060

    
1061

    
1062
def RemoveEtcHostsEntry(file_name, hostname):
1063
  """Removes a hostname from /etc/hosts.
1064

1065
  IP addresses without names are removed from the file.
1066

1067
  @type file_name: str
1068
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1069
  @type hostname: str
1070
  @param hostname: the hostname to be removed
1071

1072
  """
1073
  # FIXME: use WriteFile + fn rather than duplicating its efforts
1074
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1075
  try:
1076
    out = os.fdopen(fd, 'w')
1077
    try:
1078
      f = open(file_name, 'r')
1079
      try:
1080
        for line in f:
1081
          fields = line.split()
1082
          if len(fields) > 1 and not fields[0].startswith('#'):
1083
            names = fields[1:]
1084
            if hostname in names:
1085
              while hostname in names:
1086
                names.remove(hostname)
1087
              if names:
1088
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
1089
              continue
1090

    
1091
          out.write(line)
1092

    
1093
        out.flush()
1094
        os.fsync(out)
1095
        os.chmod(tmpname, 0644)
1096
        os.rename(tmpname, file_name)
1097
      finally:
1098
        f.close()
1099
    finally:
1100
      out.close()
1101
  except:
1102
    RemoveFile(tmpname)
1103
    raise
1104

    
1105

    
1106
def RemoveHostFromEtcHosts(hostname):
1107
  """Wrapper around RemoveEtcHostsEntry.
1108

1109
  @type hostname: str
1110
  @param hostname: hostname that will be resolved and its
1111
      full and shot name will be removed from
1112
      L{constants.ETC_HOSTS}
1113

1114
  """
1115
  hi = HostInfo(name=hostname)
1116
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
1117
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1118

    
1119

    
1120
def CreateBackup(file_name):
1121
  """Creates a backup of a file.
1122

1123
  @type file_name: str
1124
  @param file_name: file to be backed up
1125
  @rtype: str
1126
  @return: the path to the newly created backup
1127
  @raise errors.ProgrammerError: for invalid file names
1128

1129
  """
1130
  if not os.path.isfile(file_name):
1131
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1132
                                file_name)
1133

    
1134
  prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1135
  dir_name = os.path.dirname(file_name)
1136

    
1137
  fsrc = open(file_name, 'rb')
1138
  try:
1139
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1140
    fdst = os.fdopen(fd, 'wb')
1141
    try:
1142
      shutil.copyfileobj(fsrc, fdst)
1143
    finally:
1144
      fdst.close()
1145
  finally:
1146
    fsrc.close()
1147

    
1148
  return backup_name
1149

    
1150

    
1151
def ShellQuote(value):
1152
  """Quotes shell argument according to POSIX.
1153

1154
  @type value: str
1155
  @param value: the argument to be quoted
1156
  @rtype: str
1157
  @return: the quoted value
1158

1159
  """
1160
  if _re_shell_unquoted.match(value):
1161
    return value
1162
  else:
1163
    return "'%s'" % value.replace("'", "'\\''")
1164

    
1165

    
1166
def ShellQuoteArgs(args):
1167
  """Quotes a list of shell arguments.
1168

1169
  @type args: list
1170
  @param args: list of arguments to be quoted
1171
  @rtype: str
1172
  @return: the quoted arguments concatenated with spaces
1173

1174
  """
1175
  return ' '.join([ShellQuote(i) for i in args])
1176

    
1177

    
1178
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1179
  """Simple ping implementation using TCP connect(2).
1180

1181
  Check if the given IP is reachable by doing attempting a TCP connect
1182
  to it.
1183

1184
  @type target: str
1185
  @param target: the IP or hostname to ping
1186
  @type port: int
1187
  @param port: the port to connect to
1188
  @type timeout: int
1189
  @param timeout: the timeout on the connection attempt
1190
  @type live_port_needed: boolean
1191
  @param live_port_needed: whether a closed port will cause the
1192
      function to return failure, as if there was a timeout
1193
  @type source: str or None
1194
  @param source: if specified, will cause the connect to be made
1195
      from this specific source address; failures to bind other
1196
      than C{EADDRNOTAVAIL} will be ignored
1197

1198
  """
1199
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1200

    
1201
  success = False
1202

    
1203
  if source is not None:
1204
    try:
1205
      sock.bind((source, 0))
1206
    except socket.error, (errcode, _):
1207
      if errcode == errno.EADDRNOTAVAIL:
1208
        success = False
1209

    
1210
  sock.settimeout(timeout)
1211

    
1212
  try:
1213
    sock.connect((target, port))
1214
    sock.close()
1215
    success = True
1216
  except socket.timeout:
1217
    success = False
1218
  except socket.error, (errcode, _):
1219
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1220

    
1221
  return success
1222

    
1223

    
1224
def OwnIpAddress(address):
1225
  """Check if the current host has the the given IP address.
1226

1227
  Currently this is done by TCP-pinging the address from the loopback
1228
  address.
1229

1230
  @type address: string
1231
  @param address: the address to check
1232
  @rtype: bool
1233
  @return: True if we own the address
1234

1235
  """
1236
  return TcpPing(address, constants.DEFAULT_NODED_PORT,
1237
                 source=constants.LOCALHOST_IP_ADDRESS)
1238

    
1239

    
1240
def ListVisibleFiles(path):
1241
  """Returns a list of visible files in a directory.
1242

1243
  @type path: str
1244
  @param path: the directory to enumerate
1245
  @rtype: list
1246
  @return: the list of all files not starting with a dot
1247
  @raise ProgrammerError: if L{path} is not an absolue and normalized path
1248

1249
  """
1250
  if not IsNormAbsPath(path):
1251
    raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
1252
                                 " absolute/normalized: '%s'" % path)
1253
  files = [i for i in os.listdir(path) if not i.startswith(".")]
1254
  files.sort()
1255
  return files
1256

    
1257

    
1258
def GetHomeDir(user, default=None):
1259
  """Try to get the homedir of the given user.
1260

1261
  The user can be passed either as a string (denoting the name) or as
1262
  an integer (denoting the user id). If the user is not found, the
1263
  'default' argument is returned, which defaults to None.
1264

1265
  """
1266
  try:
1267
    if isinstance(user, basestring):
1268
      result = pwd.getpwnam(user)
1269
    elif isinstance(user, (int, long)):
1270
      result = pwd.getpwuid(user)
1271
    else:
1272
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1273
                                   type(user))
1274
  except KeyError:
1275
    return default
1276
  return result.pw_dir
1277

    
1278

    
1279
def NewUUID():
1280
  """Returns a random UUID.
1281

1282
  @note: This is a Linux-specific method as it uses the /proc
1283
      filesystem.
1284
  @rtype: str
1285

1286
  """
1287
  return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1288

    
1289

    
1290
def GenerateSecret(numbytes=20):
1291
  """Generates a random secret.
1292

1293
  This will generate a pseudo-random secret returning an hex string
1294
  (so that it can be used where an ASCII string is needed).
1295

1296
  @param numbytes: the number of bytes which will be represented by the returned
1297
      string (defaulting to 20, the length of a SHA1 hash)
1298
  @rtype: str
1299
  @return: an hex representation of the pseudo-random sequence
1300

1301
  """
1302
  return os.urandom(numbytes).encode('hex')
1303

    
1304

    
1305
def EnsureDirs(dirs):
1306
  """Make required directories, if they don't exist.
1307

1308
  @param dirs: list of tuples (dir_name, dir_mode)
1309
  @type dirs: list of (string, integer)
1310

1311
  """
1312
  for dir_name, dir_mode in dirs:
1313
    try:
1314
      os.mkdir(dir_name, dir_mode)
1315
    except EnvironmentError, err:
1316
      if err.errno != errno.EEXIST:
1317
        raise errors.GenericError("Cannot create needed directory"
1318
                                  " '%s': %s" % (dir_name, err))
1319
    if not os.path.isdir(dir_name):
1320
      raise errors.GenericError("%s is not a directory" % dir_name)
1321

    
1322

    
1323
def ReadFile(file_name, size=-1):
1324
  """Reads a file.
1325

1326
  @type size: int
1327
  @param size: Read at most size bytes (if negative, entire file)
1328
  @rtype: str
1329
  @return: the (possibly partial) content of the file
1330

1331
  """
1332
  f = open(file_name, "r")
1333
  try:
1334
    return f.read(size)
1335
  finally:
1336
    f.close()
1337

    
1338

    
1339
def WriteFile(file_name, fn=None, data=None,
1340
              mode=None, uid=-1, gid=-1,
1341
              atime=None, mtime=None, close=True,
1342
              dry_run=False, backup=False,
1343
              prewrite=None, postwrite=None):
1344
  """(Over)write a file atomically.
1345

1346
  The file_name and either fn (a function taking one argument, the
1347
  file descriptor, and which should write the data to it) or data (the
1348
  contents of the file) must be passed. The other arguments are
1349
  optional and allow setting the file mode, owner and group, and the
1350
  mtime/atime of the file.
1351

1352
  If the function doesn't raise an exception, it has succeeded and the
1353
  target file has the new contents. If the function has raised an
1354
  exception, an existing target file should be unmodified and the
1355
  temporary file should be removed.
1356

1357
  @type file_name: str
1358
  @param file_name: the target filename
1359
  @type fn: callable
1360
  @param fn: content writing function, called with
1361
      file descriptor as parameter
1362
  @type data: str
1363
  @param data: contents of the file
1364
  @type mode: int
1365
  @param mode: file mode
1366
  @type uid: int
1367
  @param uid: the owner of the file
1368
  @type gid: int
1369
  @param gid: the group of the file
1370
  @type atime: int
1371
  @param atime: a custom access time to be set on the file
1372
  @type mtime: int
1373
  @param mtime: a custom modification time to be set on the file
1374
  @type close: boolean
1375
  @param close: whether to close file after writing it
1376
  @type prewrite: callable
1377
  @param prewrite: function to be called before writing content
1378
  @type postwrite: callable
1379
  @param postwrite: function to be called after writing content
1380

1381
  @rtype: None or int
1382
  @return: None if the 'close' parameter evaluates to True,
1383
      otherwise the file descriptor
1384

1385
  @raise errors.ProgrammerError: if any of the arguments are not valid
1386

1387
  """
1388
  if not os.path.isabs(file_name):
1389
    raise errors.ProgrammerError("Path passed to WriteFile is not"
1390
                                 " absolute: '%s'" % file_name)
1391

    
1392
  if [fn, data].count(None) != 1:
1393
    raise errors.ProgrammerError("fn or data required")
1394

    
1395
  if [atime, mtime].count(None) == 1:
1396
    raise errors.ProgrammerError("Both atime and mtime must be either"
1397
                                 " set or None")
1398

    
1399
  if backup and not dry_run and os.path.isfile(file_name):
1400
    CreateBackup(file_name)
1401

    
1402
  dir_name, base_name = os.path.split(file_name)
1403
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1404
  do_remove = True
1405
  # here we need to make sure we remove the temp file, if any error
1406
  # leaves it in place
1407
  try:
1408
    if uid != -1 or gid != -1:
1409
      os.chown(new_name, uid, gid)
1410
    if mode:
1411
      os.chmod(new_name, mode)
1412
    if callable(prewrite):
1413
      prewrite(fd)
1414
    if data is not None:
1415
      os.write(fd, data)
1416
    else:
1417
      fn(fd)
1418
    if callable(postwrite):
1419
      postwrite(fd)
1420
    os.fsync(fd)
1421
    if atime is not None and mtime is not None:
1422
      os.utime(new_name, (atime, mtime))
1423
    if not dry_run:
1424
      os.rename(new_name, file_name)
1425
      do_remove = False
1426
  finally:
1427
    if close:
1428
      os.close(fd)
1429
      result = None
1430
    else:
1431
      result = fd
1432
    if do_remove:
1433
      RemoveFile(new_name)
1434

    
1435
  return result
1436

    
1437

    
1438
def FirstFree(seq, base=0):
1439
  """Returns the first non-existing integer from seq.
1440

1441
  The seq argument should be a sorted list of positive integers. The
1442
  first time the index of an element is smaller than the element
1443
  value, the index will be returned.
1444

1445
  The base argument is used to start at a different offset,
1446
  i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1447

1448
  Example: C{[0, 1, 3]} will return I{2}.
1449

1450
  @type seq: sequence
1451
  @param seq: the sequence to be analyzed.
1452
  @type base: int
1453
  @param base: use this value as the base index of the sequence
1454
  @rtype: int
1455
  @return: the first non-used index in the sequence
1456

1457
  """
1458
  for idx, elem in enumerate(seq):
1459
    assert elem >= base, "Passed element is higher than base offset"
1460
    if elem > idx + base:
1461
      # idx is not used
1462
      return idx + base
1463
  return None
1464

    
1465

    
1466
def all(seq, pred=bool): # pylint: disable-msg=W0622
1467
  "Returns True if pred(x) is True for every element in the iterable"
1468
  for _ in itertools.ifilterfalse(pred, seq):
1469
    return False
1470
  return True
1471

    
1472

    
1473
def any(seq, pred=bool): # pylint: disable-msg=W0622
1474
  "Returns True if pred(x) is True for at least one element in the iterable"
1475
  for _ in itertools.ifilter(pred, seq):
1476
    return True
1477
  return False
1478

    
1479

    
1480
def partition(seq, pred=bool): # # pylint: disable-msg=W0622
1481
  "Partition a list in two, based on the given predicate"
1482
  return (list(itertools.ifilter(pred, seq)),
1483
          list(itertools.ifilterfalse(pred, seq)))
1484

    
1485

    
1486
def UniqueSequence(seq):
1487
  """Returns a list with unique elements.
1488

1489
  Element order is preserved.
1490

1491
  @type seq: sequence
1492
  @param seq: the sequence with the source elements
1493
  @rtype: list
1494
  @return: list of unique elements from seq
1495

1496
  """
1497
  seen = set()
1498
  return [i for i in seq if i not in seen and not seen.add(i)]
1499

    
1500

    
1501
def NormalizeAndValidateMac(mac):
1502
  """Normalizes and check if a MAC address is valid.
1503

1504
  Checks whether the supplied MAC address is formally correct, only
1505
  accepts colon separated format. Normalize it to all lower.
1506

1507
  @type mac: str
1508
  @param mac: the MAC to be validated
1509
  @rtype: str
1510
  @return: returns the normalized and validated MAC.
1511

1512
  @raise errors.OpPrereqError: If the MAC isn't valid
1513

1514
  """
1515
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
1516
  if not mac_check.match(mac):
1517
    raise errors.OpPrereqError("Invalid MAC address specified: %s" %
1518
                               mac, errors.ECODE_INVAL)
1519

    
1520
  return mac.lower()
1521

    
1522

    
1523
def TestDelay(duration):
1524
  """Sleep for a fixed amount of time.
1525

1526
  @type duration: float
1527
  @param duration: the sleep duration
1528
  @rtype: boolean
1529
  @return: False for negative value, True otherwise
1530

1531
  """
1532
  if duration < 0:
1533
    return False, "Invalid sleep duration"
1534
  time.sleep(duration)
1535
  return True, None
1536

    
1537

    
1538
def _CloseFDNoErr(fd, retries=5):
1539
  """Close a file descriptor ignoring errors.
1540

1541
  @type fd: int
1542
  @param fd: the file descriptor
1543
  @type retries: int
1544
  @param retries: how many retries to make, in case we get any
1545
      other error than EBADF
1546

1547
  """
1548
  try:
1549
    os.close(fd)
1550
  except OSError, err:
1551
    if err.errno != errno.EBADF:
1552
      if retries > 0:
1553
        _CloseFDNoErr(fd, retries - 1)
1554
    # else either it's closed already or we're out of retries, so we
1555
    # ignore this and go on
1556

    
1557

    
1558
def CloseFDs(noclose_fds=None):
1559
  """Close file descriptors.
1560

1561
  This closes all file descriptors above 2 (i.e. except
1562
  stdin/out/err).
1563

1564
  @type noclose_fds: list or None
1565
  @param noclose_fds: if given, it denotes a list of file descriptor
1566
      that should not be closed
1567

1568
  """
1569
  # Default maximum for the number of available file descriptors.
1570
  if 'SC_OPEN_MAX' in os.sysconf_names:
1571
    try:
1572
      MAXFD = os.sysconf('SC_OPEN_MAX')
1573
      if MAXFD < 0:
1574
        MAXFD = 1024
1575
    except OSError:
1576
      MAXFD = 1024
1577
  else:
1578
    MAXFD = 1024
1579
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1580
  if (maxfd == resource.RLIM_INFINITY):
1581
    maxfd = MAXFD
1582

    
1583
  # Iterate through and close all file descriptors (except the standard ones)
1584
  for fd in range(3, maxfd):
1585
    if noclose_fds and fd in noclose_fds:
1586
      continue
1587
    _CloseFDNoErr(fd)
1588

    
1589

    
1590
def Daemonize(logfile):
1591
  """Daemonize the current process.
1592

1593
  This detaches the current process from the controlling terminal and
1594
  runs it in the background as a daemon.
1595

1596
  @type logfile: str
1597
  @param logfile: the logfile to which we should redirect stdout/stderr
1598
  @rtype: int
1599
  @return: the value zero
1600

1601
  """
1602
  # pylint: disable-msg=W0212
1603
  # yes, we really want os._exit
1604
  UMASK = 077
1605
  WORKDIR = "/"
1606

    
1607
  # this might fail
1608
  pid = os.fork()
1609
  if (pid == 0):  # The first child.
1610
    os.setsid()
1611
    # this might fail
1612
    pid = os.fork() # Fork a second child.
1613
    if (pid == 0):  # The second child.
1614
      os.chdir(WORKDIR)
1615
      os.umask(UMASK)
1616
    else:
1617
      # exit() or _exit()?  See below.
1618
      os._exit(0) # Exit parent (the first child) of the second child.
1619
  else:
1620
    os._exit(0) # Exit parent of the first child.
1621

    
1622
  for fd in range(3):
1623
    _CloseFDNoErr(fd)
1624
  i = os.open("/dev/null", os.O_RDONLY) # stdin
1625
  assert i == 0, "Can't close/reopen stdin"
1626
  i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1627
  assert i == 1, "Can't close/reopen stdout"
1628
  # Duplicate standard output to standard error.
1629
  os.dup2(1, 2)
1630
  return 0
1631

    
1632

    
1633
def DaemonPidFileName(name):
1634
  """Compute a ganeti pid file absolute path
1635

1636
  @type name: str
1637
  @param name: the daemon name
1638
  @rtype: str
1639
  @return: the full path to the pidfile corresponding to the given
1640
      daemon name
1641

1642
  """
1643
  return PathJoin(constants.RUN_GANETI_DIR, "%s.pid" % name)
1644

    
1645

    
1646
def EnsureDaemon(name):
1647
  """Check for and start daemon if not alive.
1648

1649
  """
1650
  result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
1651
  if result.failed:
1652
    logging.error("Can't start daemon '%s', failure %s, output: %s",
1653
                  name, result.fail_reason, result.output)
1654
    return False
1655

    
1656
  return True
1657

    
1658

    
1659
def WritePidFile(name):
1660
  """Write the current process pidfile.
1661

1662
  The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1663

1664
  @type name: str
1665
  @param name: the daemon name to use
1666
  @raise errors.GenericError: if the pid file already exists and
1667
      points to a live process
1668

1669
  """
1670
  pid = os.getpid()
1671
  pidfilename = DaemonPidFileName(name)
1672
  if IsProcessAlive(ReadPidFile(pidfilename)):
1673
    raise errors.GenericError("%s contains a live process" % pidfilename)
1674

    
1675
  WriteFile(pidfilename, data="%d\n" % pid)
1676

    
1677

    
1678
def RemovePidFile(name):
1679
  """Remove the current process pidfile.
1680

1681
  Any errors are ignored.
1682

1683
  @type name: str
1684
  @param name: the daemon name used to derive the pidfile name
1685

1686
  """
1687
  pidfilename = DaemonPidFileName(name)
1688
  # TODO: we could check here that the file contains our pid
1689
  try:
1690
    RemoveFile(pidfilename)
1691
  except: # pylint: disable-msg=W0702
1692
    pass
1693

    
1694

    
1695
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1696
                waitpid=False):
1697
  """Kill a process given by its pid.
1698

1699
  @type pid: int
1700
  @param pid: The PID to terminate.
1701
  @type signal_: int
1702
  @param signal_: The signal to send, by default SIGTERM
1703
  @type timeout: int
1704
  @param timeout: The timeout after which, if the process is still alive,
1705
                  a SIGKILL will be sent. If not positive, no such checking
1706
                  will be done
1707
  @type waitpid: boolean
1708
  @param waitpid: If true, we should waitpid on this process after
1709
      sending signals, since it's our own child and otherwise it
1710
      would remain as zombie
1711

1712
  """
1713
  def _helper(pid, signal_, wait):
1714
    """Simple helper to encapsulate the kill/waitpid sequence"""
1715
    os.kill(pid, signal_)
1716
    if wait:
1717
      try:
1718
        os.waitpid(pid, os.WNOHANG)
1719
      except OSError:
1720
        pass
1721

    
1722
  if pid <= 0:
1723
    # kill with pid=0 == suicide
1724
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1725

    
1726
  if not IsProcessAlive(pid):
1727
    return
1728

    
1729
  _helper(pid, signal_, waitpid)
1730

    
1731
  if timeout <= 0:
1732
    return
1733

    
1734
  def _CheckProcess():
1735
    if not IsProcessAlive(pid):
1736
      return
1737

    
1738
    try:
1739
      (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1740
    except OSError:
1741
      raise RetryAgain()
1742

    
1743
    if result_pid > 0:
1744
      return
1745

    
1746
    raise RetryAgain()
1747

    
1748
  try:
1749
    # Wait up to $timeout seconds
1750
    Retry(_CheckProcess, (0.01, 1.5, 0.1), timeout)
1751
  except RetryTimeout:
1752
    pass
1753

    
1754
  if IsProcessAlive(pid):
1755
    # Kill process if it's still alive
1756
    _helper(pid, signal.SIGKILL, waitpid)
1757

    
1758

    
1759
def FindFile(name, search_path, test=os.path.exists):
1760
  """Look for a filesystem object in a given path.
1761

1762
  This is an abstract method to search for filesystem object (files,
1763
  dirs) under a given search path.
1764

1765
  @type name: str
1766
  @param name: the name to look for
1767
  @type search_path: str
1768
  @param search_path: location to start at
1769
  @type test: callable
1770
  @param test: a function taking one argument that should return True
1771
      if the a given object is valid; the default value is
1772
      os.path.exists, causing only existing files to be returned
1773
  @rtype: str or None
1774
  @return: full path to the object if found, None otherwise
1775

1776
  """
1777
  # validate the filename mask
1778
  if constants.EXT_PLUGIN_MASK.match(name) is None:
1779
    logging.critical("Invalid value passed for external script name: '%s'",
1780
                     name)
1781
    return None
1782

    
1783
  for dir_name in search_path:
1784
    # FIXME: investigate switch to PathJoin
1785
    item_name = os.path.sep.join([dir_name, name])
1786
    # check the user test and that we're indeed resolving to the given
1787
    # basename
1788
    if test(item_name) and os.path.basename(item_name) == name:
1789
      return item_name
1790
  return None
1791

    
1792

    
1793
def CheckVolumeGroupSize(vglist, vgname, minsize):
1794
  """Checks if the volume group list is valid.
1795

1796
  The function will check if a given volume group is in the list of
1797
  volume groups and has a minimum size.
1798

1799
  @type vglist: dict
1800
  @param vglist: dictionary of volume group names and their size
1801
  @type vgname: str
1802
  @param vgname: the volume group we should check
1803
  @type minsize: int
1804
  @param minsize: the minimum size we accept
1805
  @rtype: None or str
1806
  @return: None for success, otherwise the error message
1807

1808
  """
1809
  vgsize = vglist.get(vgname, None)
1810
  if vgsize is None:
1811
    return "volume group '%s' missing" % vgname
1812
  elif vgsize < minsize:
1813
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1814
            (vgname, minsize, vgsize))
1815
  return None
1816

    
1817

    
1818
def SplitTime(value):
1819
  """Splits time as floating point number into a tuple.
1820

1821
  @param value: Time in seconds
1822
  @type value: int or float
1823
  @return: Tuple containing (seconds, microseconds)
1824

1825
  """
1826
  (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1827

    
1828
  assert 0 <= seconds, \
1829
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1830
  assert 0 <= microseconds <= 999999, \
1831
    "Microseconds must be 0-999999, but are %s" % microseconds
1832

    
1833
  return (int(seconds), int(microseconds))
1834

    
1835

    
1836
def MergeTime(timetuple):
1837
  """Merges a tuple into time as a floating point number.
1838

1839
  @param timetuple: Time as tuple, (seconds, microseconds)
1840
  @type timetuple: tuple
1841
  @return: Time as a floating point number expressed in seconds
1842

1843
  """
1844
  (seconds, microseconds) = timetuple
1845

    
1846
  assert 0 <= seconds, \
1847
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1848
  assert 0 <= microseconds <= 999999, \
1849
    "Microseconds must be 0-999999, but are %s" % microseconds
1850

    
1851
  return float(seconds) + (float(microseconds) * 0.000001)
1852

    
1853

    
1854
def GetDaemonPort(daemon_name):
1855
  """Get the daemon port for this cluster.
1856

1857
  Note that this routine does not read a ganeti-specific file, but
1858
  instead uses C{socket.getservbyname} to allow pre-customization of
1859
  this parameter outside of Ganeti.
1860

1861
  @type daemon_name: string
1862
  @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1863
  @rtype: int
1864

1865
  """
1866
  if daemon_name not in constants.DAEMONS_PORTS:
1867
    raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1868

    
1869
  (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1870
  try:
1871
    port = socket.getservbyname(daemon_name, proto)
1872
  except socket.error:
1873
    port = default_port
1874

    
1875
  return port
1876

    
1877

    
1878
def SetupLogging(logfile, debug=0, stderr_logging=False, program="",
1879
                 multithreaded=False, syslog=constants.SYSLOG_USAGE):
1880
  """Configures the logging module.
1881

1882
  @type logfile: str
1883
  @param logfile: the filename to which we should log
1884
  @type debug: integer
1885
  @param debug: if greater than zero, enable debug messages, otherwise
1886
      only those at C{INFO} and above level
1887
  @type stderr_logging: boolean
1888
  @param stderr_logging: whether we should also log to the standard error
1889
  @type program: str
1890
  @param program: the name under which we should log messages
1891
  @type multithreaded: boolean
1892
  @param multithreaded: if True, will add the thread name to the log file
1893
  @type syslog: string
1894
  @param syslog: one of 'no', 'yes', 'only':
1895
      - if no, syslog is not used
1896
      - if yes, syslog is used (in addition to file-logging)
1897
      - if only, only syslog is used
1898
  @raise EnvironmentError: if we can't open the log file and
1899
      syslog/stderr logging is disabled
1900

1901
  """
1902
  fmt = "%(asctime)s: " + program + " pid=%(process)d"
1903
  sft = program + "[%(process)d]:"
1904
  if multithreaded:
1905
    fmt += "/%(threadName)s"
1906
    sft += " (%(threadName)s)"
1907
  if debug:
1908
    fmt += " %(module)s:%(lineno)s"
1909
    # no debug info for syslog loggers
1910
  fmt += " %(levelname)s %(message)s"
1911
  # yes, we do want the textual level, as remote syslog will probably
1912
  # lose the error level, and it's easier to grep for it
1913
  sft += " %(levelname)s %(message)s"
1914
  formatter = logging.Formatter(fmt)
1915
  sys_fmt = logging.Formatter(sft)
1916

    
1917
  root_logger = logging.getLogger("")
1918
  root_logger.setLevel(logging.NOTSET)
1919

    
1920
  # Remove all previously setup handlers
1921
  for handler in root_logger.handlers:
1922
    handler.close()
1923
    root_logger.removeHandler(handler)
1924

    
1925
  if stderr_logging:
1926
    stderr_handler = logging.StreamHandler()
1927
    stderr_handler.setFormatter(formatter)
1928
    if debug:
1929
      stderr_handler.setLevel(logging.NOTSET)
1930
    else:
1931
      stderr_handler.setLevel(logging.CRITICAL)
1932
    root_logger.addHandler(stderr_handler)
1933

    
1934
  if syslog in (constants.SYSLOG_YES, constants.SYSLOG_ONLY):
1935
    facility = logging.handlers.SysLogHandler.LOG_DAEMON
1936
    syslog_handler = logging.handlers.SysLogHandler(constants.SYSLOG_SOCKET,
1937
                                                    facility)
1938
    syslog_handler.setFormatter(sys_fmt)
1939
    # Never enable debug over syslog
1940
    syslog_handler.setLevel(logging.INFO)
1941
    root_logger.addHandler(syslog_handler)
1942

    
1943
  if syslog != constants.SYSLOG_ONLY:
1944
    # this can fail, if the logging directories are not setup or we have
1945
    # a permisssion problem; in this case, it's best to log but ignore
1946
    # the error if stderr_logging is True, and if false we re-raise the
1947
    # exception since otherwise we could run but without any logs at all
1948
    try:
1949
      logfile_handler = logging.FileHandler(logfile)
1950
      logfile_handler.setFormatter(formatter)
1951
      if debug:
1952
        logfile_handler.setLevel(logging.DEBUG)
1953
      else:
1954
        logfile_handler.setLevel(logging.INFO)
1955
      root_logger.addHandler(logfile_handler)
1956
    except EnvironmentError:
1957
      if stderr_logging or syslog == constants.SYSLOG_YES:
1958
        logging.exception("Failed to enable logging to file '%s'", logfile)
1959
      else:
1960
        # we need to re-raise the exception
1961
        raise
1962

    
1963

    
1964
def IsNormAbsPath(path):
1965
  """Check whether a path is absolute and also normalized
1966

1967
  This avoids things like /dir/../../other/path to be valid.
1968

1969
  """
1970
  return os.path.normpath(path) == path and os.path.isabs(path)
1971

    
1972

    
1973
def PathJoin(*args):
1974
  """Safe-join a list of path components.
1975

1976
  Requirements:
1977
      - the first argument must be an absolute path
1978
      - no component in the path must have backtracking (e.g. /../),
1979
        since we check for normalization at the end
1980

1981
  @param args: the path components to be joined
1982
  @raise ValueError: for invalid paths
1983

1984
  """
1985
  # ensure we're having at least one path passed in
1986
  assert args
1987
  # ensure the first component is an absolute and normalized path name
1988
  root = args[0]
1989
  if not IsNormAbsPath(root):
1990
    raise ValueError("Invalid parameter to PathJoin: '%s'" % str(args[0]))
1991
  result = os.path.join(*args)
1992
  # ensure that the whole path is normalized
1993
  if not IsNormAbsPath(result):
1994
    raise ValueError("Invalid parameters to PathJoin: '%s'" % str(args))
1995
  # check that we're still under the original prefix
1996
  prefix = os.path.commonprefix([root, result])
1997
  if prefix != root:
1998
    raise ValueError("Error: path joining resulted in different prefix"
1999
                     " (%s != %s)" % (prefix, root))
2000
  return result
2001

    
2002

    
2003
def TailFile(fname, lines=20):
2004
  """Return the last lines from a file.
2005

2006
  @note: this function will only read and parse the last 4KB of
2007
      the file; if the lines are very long, it could be that less
2008
      than the requested number of lines are returned
2009

2010
  @param fname: the file name
2011
  @type lines: int
2012
  @param lines: the (maximum) number of lines to return
2013

2014
  """
2015
  fd = open(fname, "r")
2016
  try:
2017
    fd.seek(0, 2)
2018
    pos = fd.tell()
2019
    pos = max(0, pos-4096)
2020
    fd.seek(pos, 0)
2021
    raw_data = fd.read()
2022
  finally:
2023
    fd.close()
2024

    
2025
  rows = raw_data.splitlines()
2026
  return rows[-lines:]
2027

    
2028

    
2029
def SafeEncode(text):
2030
  """Return a 'safe' version of a source string.
2031

2032
  This function mangles the input string and returns a version that
2033
  should be safe to display/encode as ASCII. To this end, we first
2034
  convert it to ASCII using the 'backslashreplace' encoding which
2035
  should get rid of any non-ASCII chars, and then we process it
2036
  through a loop copied from the string repr sources in the python; we
2037
  don't use string_escape anymore since that escape single quotes and
2038
  backslashes too, and that is too much; and that escaping is not
2039
  stable, i.e. string_escape(string_escape(x)) != string_escape(x).
2040

2041
  @type text: str or unicode
2042
  @param text: input data
2043
  @rtype: str
2044
  @return: a safe version of text
2045

2046
  """
2047
  if isinstance(text, unicode):
2048
    # only if unicode; if str already, we handle it below
2049
    text = text.encode('ascii', 'backslashreplace')
2050
  resu = ""
2051
  for char in text:
2052
    c = ord(char)
2053
    if char  == '\t':
2054
      resu += r'\t'
2055
    elif char == '\n':
2056
      resu += r'\n'
2057
    elif char == '\r':
2058
      resu += r'\'r'
2059
    elif c < 32 or c >= 127: # non-printable
2060
      resu += "\\x%02x" % (c & 0xff)
2061
    else:
2062
      resu += char
2063
  return resu
2064

    
2065

    
2066
def UnescapeAndSplit(text, sep=","):
2067
  """Split and unescape a string based on a given separator.
2068

2069
  This function splits a string based on a separator where the
2070
  separator itself can be escape in order to be an element of the
2071
  elements. The escaping rules are (assuming coma being the
2072
  separator):
2073
    - a plain , separates the elements
2074
    - a sequence \\\\, (double backslash plus comma) is handled as a
2075
      backslash plus a separator comma
2076
    - a sequence \, (backslash plus comma) is handled as a
2077
      non-separator comma
2078

2079
  @type text: string
2080
  @param text: the string to split
2081
  @type sep: string
2082
  @param text: the separator
2083
  @rtype: string
2084
  @return: a list of strings
2085

2086
  """
2087
  # we split the list by sep (with no escaping at this stage)
2088
  slist = text.split(sep)
2089
  # next, we revisit the elements and if any of them ended with an odd
2090
  # number of backslashes, then we join it with the next
2091
  rlist = []
2092
  while slist:
2093
    e1 = slist.pop(0)
2094
    if e1.endswith("\\"):
2095
      num_b = len(e1) - len(e1.rstrip("\\"))
2096
      if num_b % 2 == 1:
2097
        e2 = slist.pop(0)
2098
        # here the backslashes remain (all), and will be reduced in
2099
        # the next step
2100
        rlist.append(e1 + sep + e2)
2101
        continue
2102
    rlist.append(e1)
2103
  # finally, replace backslash-something with something
2104
  rlist = [re.sub(r"\\(.)", r"\1", v) for v in rlist]
2105
  return rlist
2106

    
2107

    
2108
def CommaJoin(names):
2109
  """Nicely join a set of identifiers.
2110

2111
  @param names: set, list or tuple
2112
  @return: a string with the formatted results
2113

2114
  """
2115
  return ", ".join([str(val) for val in names])
2116

    
2117

    
2118
def BytesToMebibyte(value):
2119
  """Converts bytes to mebibytes.
2120

2121
  @type value: int
2122
  @param value: Value in bytes
2123
  @rtype: int
2124
  @return: Value in mebibytes
2125

2126
  """
2127
  return int(round(value / (1024.0 * 1024.0), 0))
2128

    
2129

    
2130
def CalculateDirectorySize(path):
2131
  """Calculates the size of a directory recursively.
2132

2133
  @type path: string
2134
  @param path: Path to directory
2135
  @rtype: int
2136
  @return: Size in mebibytes
2137

2138
  """
2139
  size = 0
2140

    
2141
  for (curpath, _, files) in os.walk(path):
2142
    for filename in files:
2143
      st = os.lstat(PathJoin(curpath, filename))
2144
      size += st.st_size
2145

    
2146
  return BytesToMebibyte(size)
2147

    
2148

    
2149
def GetFilesystemStats(path):
2150
  """Returns the total and free space on a filesystem.
2151

2152
  @type path: string
2153
  @param path: Path on filesystem to be examined
2154
  @rtype: int
2155
  @return: tuple of (Total space, Free space) in mebibytes
2156

2157
  """
2158
  st = os.statvfs(path)
2159

    
2160
  fsize = BytesToMebibyte(st.f_bavail * st.f_frsize)
2161
  tsize = BytesToMebibyte(st.f_blocks * st.f_frsize)
2162
  return (tsize, fsize)
2163

    
2164

    
2165
def RunInSeparateProcess(fn, *args):
2166
  """Runs a function in a separate process.
2167

2168
  Note: Only boolean return values are supported.
2169

2170
  @type fn: callable
2171
  @param fn: Function to be called
2172
  @rtype: bool
2173
  @return: Function's result
2174

2175
  """
2176
  pid = os.fork()
2177
  if pid == 0:
2178
    # Child process
2179
    try:
2180
      # In case the function uses temporary files
2181
      ResetTempfileModule()
2182

    
2183
      # Call function
2184
      result = int(bool(fn(*args)))
2185
      assert result in (0, 1)
2186
    except: # pylint: disable-msg=W0702
2187
      logging.exception("Error while calling function in separate process")
2188
      # 0 and 1 are reserved for the return value
2189
      result = 33
2190

    
2191
    os._exit(result) # pylint: disable-msg=W0212
2192

    
2193
  # Parent process
2194

    
2195
  # Avoid zombies and check exit code
2196
  (_, status) = os.waitpid(pid, 0)
2197

    
2198
  if os.WIFSIGNALED(status):
2199
    exitcode = None
2200
    signum = os.WTERMSIG(status)
2201
  else:
2202
    exitcode = os.WEXITSTATUS(status)
2203
    signum = None
2204

    
2205
  if not (exitcode in (0, 1) and signum is None):
2206
    raise errors.GenericError("Child program failed (code=%s, signal=%s)" %
2207
                              (exitcode, signum))
2208

    
2209
  return bool(exitcode)
2210

    
2211

    
2212
def LockedMethod(fn):
2213
  """Synchronized object access decorator.
2214

2215
  This decorator is intended to protect access to an object using the
2216
  object's own lock which is hardcoded to '_lock'.
2217

2218
  """
2219
  def _LockDebug(*args, **kwargs):
2220
    if debug_locks:
2221
      logging.debug(*args, **kwargs)
2222

    
2223
  def wrapper(self, *args, **kwargs):
2224
    # pylint: disable-msg=W0212
2225
    assert hasattr(self, '_lock')
2226
    lock = self._lock
2227
    _LockDebug("Waiting for %s", lock)
2228
    lock.acquire()
2229
    try:
2230
      _LockDebug("Acquired %s", lock)
2231
      result = fn(self, *args, **kwargs)
2232
    finally:
2233
      _LockDebug("Releasing %s", lock)
2234
      lock.release()
2235
      _LockDebug("Released %s", lock)
2236
    return result
2237
  return wrapper
2238

    
2239

    
2240
def LockFile(fd):
2241
  """Locks a file using POSIX locks.
2242

2243
  @type fd: int
2244
  @param fd: the file descriptor we need to lock
2245

2246
  """
2247
  try:
2248
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2249
  except IOError, err:
2250
    if err.errno == errno.EAGAIN:
2251
      raise errors.LockError("File already locked")
2252
    raise
2253

    
2254

    
2255
def FormatTime(val):
2256
  """Formats a time value.
2257

2258
  @type val: float or None
2259
  @param val: the timestamp as returned by time.time()
2260
  @return: a string value or N/A if we don't have a valid timestamp
2261

2262
  """
2263
  if val is None or not isinstance(val, (int, float)):
2264
    return "N/A"
2265
  # these two codes works on Linux, but they are not guaranteed on all
2266
  # platforms
2267
  return time.strftime("%F %T", time.localtime(val))
2268

    
2269

    
2270
def ReadWatcherPauseFile(filename, now=None, remove_after=3600):
2271
  """Reads the watcher pause file.
2272

2273
  @type filename: string
2274
  @param filename: Path to watcher pause file
2275
  @type now: None, float or int
2276
  @param now: Current time as Unix timestamp
2277
  @type remove_after: int
2278
  @param remove_after: Remove watcher pause file after specified amount of
2279
    seconds past the pause end time
2280

2281
  """
2282
  if now is None:
2283
    now = time.time()
2284

    
2285
  try:
2286
    value = ReadFile(filename)
2287
  except IOError, err:
2288
    if err.errno != errno.ENOENT:
2289
      raise
2290
    value = None
2291

    
2292
  if value is not None:
2293
    try:
2294
      value = int(value)
2295
    except ValueError:
2296
      logging.warning(("Watcher pause file (%s) contains invalid value,"
2297
                       " removing it"), filename)
2298
      RemoveFile(filename)
2299
      value = None
2300

    
2301
    if value is not None:
2302
      # Remove file if it's outdated
2303
      if now > (value + remove_after):
2304
        RemoveFile(filename)
2305
        value = None
2306

    
2307
      elif now > value:
2308
        value = None
2309

    
2310
  return value
2311

    
2312

    
2313
class RetryTimeout(Exception):
2314
  """Retry loop timed out.
2315

2316
  """
2317

    
2318

    
2319
class RetryAgain(Exception):
2320
  """Retry again.
2321

2322
  """
2323

    
2324

    
2325
class _RetryDelayCalculator(object):
2326
  """Calculator for increasing delays.
2327

2328
  """
2329
  __slots__ = [
2330
    "_factor",
2331
    "_limit",
2332
    "_next",
2333
    "_start",
2334
    ]
2335

    
2336
  def __init__(self, start, factor, limit):
2337
    """Initializes this class.
2338

2339
    @type start: float
2340
    @param start: Initial delay
2341
    @type factor: float
2342
    @param factor: Factor for delay increase
2343
    @type limit: float or None
2344
    @param limit: Upper limit for delay or None for no limit
2345

2346
    """
2347
    assert start > 0.0
2348
    assert factor >= 1.0
2349
    assert limit is None or limit >= 0.0
2350

    
2351
    self._start = start
2352
    self._factor = factor
2353
    self._limit = limit
2354

    
2355
    self._next = start
2356

    
2357
  def __call__(self):
2358
    """Returns current delay and calculates the next one.
2359

2360
    """
2361
    current = self._next
2362

    
2363
    # Update for next run
2364
    if self._limit is None or self._next < self._limit:
2365
      self._next = min(self._limit, self._next * self._factor)
2366

    
2367
    return current
2368

    
2369

    
2370
#: Special delay to specify whole remaining timeout
2371
RETRY_REMAINING_TIME = object()
2372

    
2373

    
2374
def Retry(fn, delay, timeout, args=None, wait_fn=time.sleep,
2375
          _time_fn=time.time):
2376
  """Call a function repeatedly until it succeeds.
2377

2378
  The function C{fn} is called repeatedly until it doesn't throw L{RetryAgain}
2379
  anymore. Between calls a delay, specified by C{delay}, is inserted. After a
2380
  total of C{timeout} seconds, this function throws L{RetryTimeout}.
2381

2382
  C{delay} can be one of the following:
2383
    - callable returning the delay length as a float
2384
    - Tuple of (start, factor, limit)
2385
    - L{RETRY_REMAINING_TIME} to sleep until the timeout expires (this is
2386
      useful when overriding L{wait_fn} to wait for an external event)
2387
    - A static delay as a number (int or float)
2388

2389
  @type fn: callable
2390
  @param fn: Function to be called
2391
  @param delay: Either a callable (returning the delay), a tuple of (start,
2392
                factor, limit) (see L{_RetryDelayCalculator}),
2393
                L{RETRY_REMAINING_TIME} or a number (int or float)
2394
  @type timeout: float
2395
  @param timeout: Total timeout
2396
  @type wait_fn: callable
2397
  @param wait_fn: Waiting function
2398
  @return: Return value of function
2399

2400
  """
2401
  assert callable(fn)
2402
  assert callable(wait_fn)
2403
  assert callable(_time_fn)
2404

    
2405
  if args is None:
2406
    args = []
2407

    
2408
  end_time = _time_fn() + timeout
2409

    
2410
  if callable(delay):
2411
    # External function to calculate delay
2412
    calc_delay = delay
2413

    
2414
  elif isinstance(delay, (tuple, list)):
2415
    # Increasing delay with optional upper boundary
2416
    (start, factor, limit) = delay
2417
    calc_delay = _RetryDelayCalculator(start, factor, limit)
2418

    
2419
  elif delay is RETRY_REMAINING_TIME:
2420
    # Always use the remaining time
2421
    calc_delay = None
2422

    
2423
  else:
2424
    # Static delay
2425
    calc_delay = lambda: delay
2426

    
2427
  assert calc_delay is None or callable(calc_delay)
2428

    
2429
  while True:
2430
    try:
2431
      # pylint: disable-msg=W0142
2432
      return fn(*args)
2433
    except RetryAgain:
2434
      pass
2435

    
2436
    remaining_time = end_time - _time_fn()
2437

    
2438
    if remaining_time < 0.0:
2439
      raise RetryTimeout()
2440

    
2441
    assert remaining_time >= 0.0
2442

    
2443
    if calc_delay is None:
2444
      wait_fn(remaining_time)
2445
    else:
2446
      current_delay = calc_delay()
2447
      if current_delay > 0.0:
2448
        wait_fn(current_delay)
2449

    
2450

    
2451
class FileLock(object):
2452
  """Utility class for file locks.
2453

2454
  """
2455
  def __init__(self, fd, filename):
2456
    """Constructor for FileLock.
2457

2458
    @type fd: file
2459
    @param fd: File object
2460
    @type filename: str
2461
    @param filename: Path of the file opened at I{fd}
2462

2463
    """
2464
    self.fd = fd
2465
    self.filename = filename
2466

    
2467
  @classmethod
2468
  def Open(cls, filename):
2469
    """Creates and opens a file to be used as a file-based lock.
2470

2471
    @type filename: string
2472
    @param filename: path to the file to be locked
2473

2474
    """
2475
    # Using "os.open" is necessary to allow both opening existing file
2476
    # read/write and creating if not existing. Vanilla "open" will truncate an
2477
    # existing file -or- allow creating if not existing.
2478
    return cls(os.fdopen(os.open(filename, os.O_RDWR | os.O_CREAT), "w+"),
2479
               filename)
2480

    
2481
  def __del__(self):
2482
    self.Close()
2483

    
2484
  def Close(self):
2485
    """Close the file and release the lock.
2486

2487
    """
2488
    if hasattr(self, "fd") and self.fd:
2489
      self.fd.close()
2490
      self.fd = None
2491

    
2492
  def _flock(self, flag, blocking, timeout, errmsg):
2493
    """Wrapper for fcntl.flock.
2494

2495
    @type flag: int
2496
    @param flag: operation flag
2497
    @type blocking: bool
2498
    @param blocking: whether the operation should be done in blocking mode.
2499
    @type timeout: None or float
2500
    @param timeout: for how long the operation should be retried (implies
2501
                    non-blocking mode).
2502
    @type errmsg: string
2503
    @param errmsg: error message in case operation fails.
2504

2505
    """
2506
    assert self.fd, "Lock was closed"
2507
    assert timeout is None or timeout >= 0, \
2508
      "If specified, timeout must be positive"
2509
    assert not (flag & fcntl.LOCK_NB), "LOCK_NB must not be set"
2510

    
2511
    # When a timeout is used, LOCK_NB must always be set
2512
    if not (timeout is None and blocking):
2513
      flag |= fcntl.LOCK_NB
2514

    
2515
    if timeout is None:
2516
      self._Lock(self.fd, flag, timeout)
2517
    else:
2518
      try:
2519
        Retry(self._Lock, (0.1, 1.2, 1.0), timeout,
2520
              args=(self.fd, flag, timeout))
2521
      except RetryTimeout:
2522
        raise errors.LockError(errmsg)
2523

    
2524
  @staticmethod
2525
  def _Lock(fd, flag, timeout):
2526
    try:
2527
      fcntl.flock(fd, flag)
2528
    except IOError, err:
2529
      if timeout is not None and err.errno == errno.EAGAIN:
2530
        raise RetryAgain()
2531

    
2532
      logging.exception("fcntl.flock failed")
2533
      raise
2534

    
2535
  def Exclusive(self, blocking=False, timeout=None):
2536
    """Locks the file in exclusive mode.
2537

2538
    @type blocking: boolean
2539
    @param blocking: whether to block and wait until we
2540
        can lock the file or return immediately
2541
    @type timeout: int or None
2542
    @param timeout: if not None, the duration to wait for the lock
2543
        (in blocking mode)
2544

2545
    """
2546
    self._flock(fcntl.LOCK_EX, blocking, timeout,
2547
                "Failed to lock %s in exclusive mode" % self.filename)
2548

    
2549
  def Shared(self, blocking=False, timeout=None):
2550
    """Locks the file in shared mode.
2551

2552
    @type blocking: boolean
2553
    @param blocking: whether to block and wait until we
2554
        can lock the file or return immediately
2555
    @type timeout: int or None
2556
    @param timeout: if not None, the duration to wait for the lock
2557
        (in blocking mode)
2558

2559
    """
2560
    self._flock(fcntl.LOCK_SH, blocking, timeout,
2561
                "Failed to lock %s in shared mode" % self.filename)
2562

    
2563
  def Unlock(self, blocking=True, timeout=None):
2564
    """Unlocks the file.
2565

2566
    According to C{flock(2)}, unlocking can also be a nonblocking
2567
    operation::
2568

2569
      To make a non-blocking request, include LOCK_NB with any of the above
2570
      operations.
2571

2572
    @type blocking: boolean
2573
    @param blocking: whether to block and wait until we
2574
        can lock the file or return immediately
2575
    @type timeout: int or None
2576
    @param timeout: if not None, the duration to wait for the lock
2577
        (in blocking mode)
2578

2579
    """
2580
    self._flock(fcntl.LOCK_UN, blocking, timeout,
2581
                "Failed to unlock %s" % self.filename)
2582

    
2583

    
2584
def SignalHandled(signums):
2585
  """Signal Handled decoration.
2586

2587
  This special decorator installs a signal handler and then calls the target
2588
  function. The function must accept a 'signal_handlers' keyword argument,
2589
  which will contain a dict indexed by signal number, with SignalHandler
2590
  objects as values.
2591

2592
  The decorator can be safely stacked with iself, to handle multiple signals
2593
  with different handlers.
2594

2595
  @type signums: list
2596
  @param signums: signals to intercept
2597

2598
  """
2599
  def wrap(fn):
2600
    def sig_function(*args, **kwargs):
2601
      assert 'signal_handlers' not in kwargs or \
2602
             kwargs['signal_handlers'] is None or \
2603
             isinstance(kwargs['signal_handlers'], dict), \
2604
             "Wrong signal_handlers parameter in original function call"
2605
      if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2606
        signal_handlers = kwargs['signal_handlers']
2607
      else:
2608
        signal_handlers = {}
2609
        kwargs['signal_handlers'] = signal_handlers
2610
      sighandler = SignalHandler(signums)
2611
      try:
2612
        for sig in signums:
2613
          signal_handlers[sig] = sighandler
2614
        return fn(*args, **kwargs)
2615
      finally:
2616
        sighandler.Reset()
2617
    return sig_function
2618
  return wrap
2619

    
2620

    
2621
class SignalHandler(object):
2622
  """Generic signal handler class.
2623

2624
  It automatically restores the original handler when deconstructed or
2625
  when L{Reset} is called. You can either pass your own handler
2626
  function in or query the L{called} attribute to detect whether the
2627
  signal was sent.
2628

2629
  @type signum: list
2630
  @ivar signum: the signals we handle
2631
  @type called: boolean
2632
  @ivar called: tracks whether any of the signals have been raised
2633

2634
  """
2635
  def __init__(self, signum):
2636
    """Constructs a new SignalHandler instance.
2637

2638
    @type signum: int or list of ints
2639
    @param signum: Single signal number or set of signal numbers
2640

2641
    """
2642
    self.signum = set(signum)
2643
    self.called = False
2644

    
2645
    self._previous = {}
2646
    try:
2647
      for signum in self.signum:
2648
        # Setup handler
2649
        prev_handler = signal.signal(signum, self._HandleSignal)
2650
        try:
2651
          self._previous[signum] = prev_handler
2652
        except:
2653
          # Restore previous handler
2654
          signal.signal(signum, prev_handler)
2655
          raise
2656
    except:
2657
      # Reset all handlers
2658
      self.Reset()
2659
      # Here we have a race condition: a handler may have already been called,
2660
      # but there's not much we can do about it at this point.
2661
      raise
2662

    
2663
  def __del__(self):
2664
    self.Reset()
2665

    
2666
  def Reset(self):
2667
    """Restore previous handler.
2668

2669
    This will reset all the signals to their previous handlers.
2670

2671
    """
2672
    for signum, prev_handler in self._previous.items():
2673
      signal.signal(signum, prev_handler)
2674
      # If successful, remove from dict
2675
      del self._previous[signum]
2676

    
2677
  def Clear(self):
2678
    """Unsets the L{called} flag.
2679

2680
    This function can be used in case a signal may arrive several times.
2681

2682
    """
2683
    self.called = False
2684

    
2685
  # we don't care about arguments, but we leave them named for the future
2686
  def _HandleSignal(self, signum, frame): # pylint: disable-msg=W0613
2687
    """Actual signal handling function.
2688

2689
    """
2690
    # This is not nice and not absolutely atomic, but it appears to be the only
2691
    # solution in Python -- there are no atomic types.
2692
    self.called = True
2693

    
2694

    
2695
class FieldSet(object):
2696
  """A simple field set.
2697

2698
  Among the features are:
2699
    - checking if a string is among a list of static string or regex objects
2700
    - checking if a whole list of string matches
2701
    - returning the matching groups from a regex match
2702

2703
  Internally, all fields are held as regular expression objects.
2704

2705
  """
2706
  def __init__(self, *items):
2707
    self.items = [re.compile("^%s$" % value) for value in items]
2708

    
2709
  def Extend(self, other_set):
2710
    """Extend the field set with the items from another one"""
2711
    self.items.extend(other_set.items)
2712

    
2713
  def Matches(self, field):
2714
    """Checks if a field matches the current set
2715

2716
    @type field: str
2717
    @param field: the string to match
2718
    @return: either None or a regular expression match object
2719

2720
    """
2721
    for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2722
      return m
2723
    return None
2724

    
2725
  def NonMatching(self, items):
2726
    """Returns the list of fields not matching the current set
2727

2728
    @type items: list
2729
    @param items: the list of fields to check
2730
    @rtype: list
2731
    @return: list of non-matching fields
2732

2733
    """
2734
    return [val for val in items if not self.Matches(val)]