Statistics
| Branch: | Tag: | Revision:

root / lib / utils.py @ 69efe319

History | View | Annotate | Download (54.6 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Ganeti utility module.
23

24
This module holds functions that can be used in both daemons (all) and
25
the command line scripts.
26

27
"""
28

    
29

    
30
import sys
31
import os
32
import sha
33
import time
34
import subprocess
35
import re
36
import socket
37
import tempfile
38
import shutil
39
import errno
40
import pwd
41
import itertools
42
import select
43
import fcntl
44
import resource
45
import logging
46
import signal
47

    
48
from cStringIO import StringIO
49

    
50
from ganeti import errors
51
from ganeti import constants
52

    
53

    
54
_locksheld = []
55
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
56

    
57
debug = False
58
debug_locks = False
59

    
60
#: when set to True, L{RunCmd} is disabled
61
no_fork = False
62

    
63

    
64
class RunResult(object):
65
  """Holds the result of running external programs.
66

67
  @type exit_code: int
68
  @ivar exit_code: the exit code of the program, or None (if the program
69
      didn't exit())
70
  @type signal: int or None
71
  @ivar signal: the signal that caused the program to finish, or None
72
      (if the program wasn't terminated by a signal)
73
  @type stdout: str
74
  @ivar stdout: the standard output of the program
75
  @type stderr: str
76
  @ivar stderr: the standard error of the program
77
  @type failed: boolean
78
  @ivar failed: True in case the program was
79
      terminated by a signal or exited with a non-zero exit code
80
  @ivar fail_reason: a string detailing the termination reason
81

82
  """
83
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
84
               "failed", "fail_reason", "cmd"]
85

    
86

    
87
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
88
    self.cmd = cmd
89
    self.exit_code = exit_code
90
    self.signal = signal_
91
    self.stdout = stdout
92
    self.stderr = stderr
93
    self.failed = (signal_ is not None or exit_code != 0)
94

    
95
    if self.signal is not None:
96
      self.fail_reason = "terminated by signal %s" % self.signal
97
    elif self.exit_code is not None:
98
      self.fail_reason = "exited with exit code %s" % self.exit_code
99
    else:
100
      self.fail_reason = "unable to determine termination reason"
101

    
102
    if self.failed:
103
      logging.debug("Command '%s' failed (%s); output: %s",
104
                    self.cmd, self.fail_reason, self.output)
105

    
106
  def _GetOutput(self):
107
    """Returns the combined stdout and stderr for easier usage.
108

109
    """
110
    return self.stdout + self.stderr
111

    
112
  output = property(_GetOutput, None, None, "Return full output")
113

    
114

    
115
def RunCmd(cmd, env=None, output=None, cwd='/'):
116
  """Execute a (shell) command.
117

118
  The command should not read from its standard input, as it will be
119
  closed.
120

121
  @type  cmd: string or list
122
  @param cmd: Command to run
123
  @type env: dict
124
  @param env: Additional environment
125
  @type output: str
126
  @param output: if desired, the output of the command can be
127
      saved in a file instead of the RunResult instance; this
128
      parameter denotes the file name (if not None)
129
  @type cwd: string
130
  @param cwd: if specified, will be used as the working
131
      directory for the command; the default will be /
132
  @rtype: L{RunResult}
133
  @return: RunResult instance
134
  @raise erors.ProgrammerError: if we call this when forks are disabled
135

136
  """
137
  if no_fork:
138
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
139

    
140
  if isinstance(cmd, list):
141
    cmd = [str(val) for val in cmd]
142
    strcmd = " ".join(cmd)
143
    shell = False
144
  else:
145
    strcmd = cmd
146
    shell = True
147
  logging.debug("RunCmd '%s'", strcmd)
148

    
149
  cmd_env = os.environ.copy()
150
  cmd_env["LC_ALL"] = "C"
151
  if env is not None:
152
    cmd_env.update(env)
153

    
154
  if output is None:
155
    out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
156
  else:
157
    status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
158
    out = err = ""
159

    
160
  if status >= 0:
161
    exitcode = status
162
    signal_ = None
163
  else:
164
    exitcode = None
165
    signal_ = -status
166

    
167
  return RunResult(exitcode, signal_, out, err, strcmd)
168

    
169

    
170
def _RunCmdPipe(cmd, env, via_shell, cwd):
171
  """Run a command and return its output.
172

173
  @type  cmd: string or list
174
  @param cmd: Command to run
175
  @type env: dict
176
  @param env: The environment to use
177
  @type via_shell: bool
178
  @param via_shell: if we should run via the shell
179
  @type cwd: string
180
  @param cwd: the working directory for the program
181
  @rtype: tuple
182
  @return: (out, err, status)
183

184
  """
185
  poller = select.poll()
186
  child = subprocess.Popen(cmd, shell=via_shell,
187
                           stderr=subprocess.PIPE,
188
                           stdout=subprocess.PIPE,
189
                           stdin=subprocess.PIPE,
190
                           close_fds=True, env=env,
191
                           cwd=cwd)
192

    
193
  child.stdin.close()
194
  poller.register(child.stdout, select.POLLIN)
195
  poller.register(child.stderr, select.POLLIN)
196
  out = StringIO()
197
  err = StringIO()
198
  fdmap = {
199
    child.stdout.fileno(): (out, child.stdout),
200
    child.stderr.fileno(): (err, child.stderr),
201
    }
202
  for fd in fdmap:
203
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
204
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
205

    
206
  while fdmap:
207
    try:
208
      pollresult = poller.poll()
209
    except EnvironmentError, eerr:
210
      if eerr.errno == errno.EINTR:
211
        continue
212
      raise
213
    except select.error, serr:
214
      if serr[0] == errno.EINTR:
215
        continue
216
      raise
217

    
218
    for fd, event in pollresult:
219
      if event & select.POLLIN or event & select.POLLPRI:
220
        data = fdmap[fd][1].read()
221
        # no data from read signifies EOF (the same as POLLHUP)
222
        if not data:
223
          poller.unregister(fd)
224
          del fdmap[fd]
225
          continue
226
        fdmap[fd][0].write(data)
227
      if (event & select.POLLNVAL or event & select.POLLHUP or
228
          event & select.POLLERR):
229
        poller.unregister(fd)
230
        del fdmap[fd]
231

    
232
  out = out.getvalue()
233
  err = err.getvalue()
234

    
235
  status = child.wait()
236
  return out, err, status
237

    
238

    
239
def _RunCmdFile(cmd, env, via_shell, output, cwd):
240
  """Run a command and save its output to a file.
241

242
  @type  cmd: string or list
243
  @param cmd: Command to run
244
  @type env: dict
245
  @param env: The environment to use
246
  @type via_shell: bool
247
  @param via_shell: if we should run via the shell
248
  @type output: str
249
  @param output: the filename in which to save the output
250
  @type cwd: string
251
  @param cwd: the working directory for the program
252
  @rtype: int
253
  @return: the exit status
254

255
  """
256
  fh = open(output, "a")
257
  try:
258
    child = subprocess.Popen(cmd, shell=via_shell,
259
                             stderr=subprocess.STDOUT,
260
                             stdout=fh,
261
                             stdin=subprocess.PIPE,
262
                             close_fds=True, env=env,
263
                             cwd=cwd)
264

    
265
    child.stdin.close()
266
    status = child.wait()
267
  finally:
268
    fh.close()
269
  return status
270

    
271

    
272
def RemoveFile(filename):
273
  """Remove a file ignoring some errors.
274

275
  Remove a file, ignoring non-existing ones or directories. Other
276
  errors are passed.
277

278
  @type filename: str
279
  @param filename: the file to be removed
280

281
  """
282
  try:
283
    os.unlink(filename)
284
  except OSError, err:
285
    if err.errno not in (errno.ENOENT, errno.EISDIR):
286
      raise
287

    
288

    
289
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
290
  """Renames a file.
291

292
  @type old: string
293
  @param old: Original path
294
  @type new: string
295
  @param new: New path
296
  @type mkdir: bool
297
  @param mkdir: Whether to create target directory if it doesn't exist
298
  @type mkdir_mode: int
299
  @param mkdir_mode: Mode for newly created directories
300

301
  """
302
  try:
303
    return os.rename(old, new)
304
  except OSError, err:
305
    # In at least one use case of this function, the job queue, directory
306
    # creation is very rare. Checking for the directory before renaming is not
307
    # as efficient.
308
    if mkdir and err.errno == errno.ENOENT:
309
      # Create directory and try again
310
      os.makedirs(os.path.dirname(new), mkdir_mode)
311
      return os.rename(old, new)
312
    raise
313

    
314

    
315
def _FingerprintFile(filename):
316
  """Compute the fingerprint of a file.
317

318
  If the file does not exist, a None will be returned
319
  instead.
320

321
  @type filename: str
322
  @param filename: the filename to checksum
323
  @rtype: str
324
  @return: the hex digest of the sha checksum of the contents
325
      of the file
326

327
  """
328
  if not (os.path.exists(filename) and os.path.isfile(filename)):
329
    return None
330

    
331
  f = open(filename)
332

    
333
  fp = sha.sha()
334
  while True:
335
    data = f.read(4096)
336
    if not data:
337
      break
338

    
339
    fp.update(data)
340

    
341
  return fp.hexdigest()
342

    
343

    
344
def FingerprintFiles(files):
345
  """Compute fingerprints for a list of files.
346

347
  @type files: list
348
  @param files: the list of filename to fingerprint
349
  @rtype: dict
350
  @return: a dictionary filename: fingerprint, holding only
351
      existing files
352

353
  """
354
  ret = {}
355

    
356
  for filename in files:
357
    cksum = _FingerprintFile(filename)
358
    if cksum:
359
      ret[filename] = cksum
360

    
361
  return ret
362

    
363

    
364
def CheckDict(target, template, logname=None):
365
  """Ensure a dictionary has a required set of keys.
366

367
  For the given dictionaries I{target} and I{template}, ensure
368
  I{target} has all the keys from I{template}. Missing keys are added
369
  with values from template.
370

371
  @type target: dict
372
  @param target: the dictionary to update
373
  @type template: dict
374
  @param template: the dictionary holding the default values
375
  @type logname: str or None
376
  @param logname: if not None, causes the missing keys to be
377
      logged with this name
378

379
  """
380
  missing = []
381
  for k in template:
382
    if k not in target:
383
      missing.append(k)
384
      target[k] = template[k]
385

    
386
  if missing and logname:
387
    logging.warning('%s missing keys %s', logname, ', '.join(missing))
388

    
389

    
390
def ForceDictType(target, key_types, allowed_values=None):
391
  """Force the values of a dict to have certain types.
392

393
  @type target: dict
394
  @param target: the dict to update
395
  @type key_types: dict
396
  @param key_types: dict mapping target dict keys to types
397
                    in constants.ENFORCEABLE_TYPES
398
  @type allowed_values: list
399
  @keyword allowed_values: list of specially allowed values
400

401
  """
402
  if allowed_values is None:
403
    allowed_values = []
404

    
405
  for key in target:
406
    if key not in key_types:
407
      msg = "Unknown key '%s'" % key
408
      raise errors.TypeEnforcementError(msg)
409

    
410
    if target[key] in allowed_values:
411
      continue
412

    
413
    type = key_types[key]
414
    if type not in constants.ENFORCEABLE_TYPES:
415
      msg = "'%s' has non-enforceable type %s" % (key, type)
416
      raise errors.ProgrammerError(msg)
417

    
418
    if type == constants.VTYPE_STRING:
419
      if not isinstance(target[key], basestring):
420
        if isinstance(target[key], bool) and not target[key]:
421
          target[key] = ''
422
        else:
423
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
424
          raise errors.TypeEnforcementError(msg)
425
    elif type == constants.VTYPE_BOOL:
426
      if isinstance(target[key], basestring) and target[key]:
427
        if target[key].lower() == constants.VALUE_FALSE:
428
          target[key] = False
429
        elif target[key].lower() == constants.VALUE_TRUE:
430
          target[key] = True
431
        else:
432
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
433
          raise errors.TypeEnforcementError(msg)
434
      elif target[key]:
435
        target[key] = True
436
      else:
437
        target[key] = False
438
    elif type == constants.VTYPE_SIZE:
439
      try:
440
        target[key] = ParseUnit(target[key])
441
      except errors.UnitParseError, err:
442
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
443
              (key, target[key], err)
444
        raise errors.TypeEnforcementError(msg)
445
    elif type == constants.VTYPE_INT:
446
      try:
447
        target[key] = int(target[key])
448
      except (ValueError, TypeError):
449
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
450
        raise errors.TypeEnforcementError(msg)
451

    
452

    
453
def IsProcessAlive(pid):
454
  """Check if a given pid exists on the system.
455

456
  @note: zombie status is not handled, so zombie processes
457
      will be returned as alive
458
  @type pid: int
459
  @param pid: the process ID to check
460
  @rtype: boolean
461
  @return: True if the process exists
462

463
  """
464
  if pid <= 0:
465
    return False
466

    
467
  try:
468
    os.stat("/proc/%d/status" % pid)
469
    return True
470
  except EnvironmentError, err:
471
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
472
      return False
473
    raise
474

    
475

    
476
def ReadPidFile(pidfile):
477
  """Read a pid from a file.
478

479
  @type  pidfile: string
480
  @param pidfile: path to the file containing the pid
481
  @rtype: int
482
  @return: The process id, if the file exists and contains a valid PID,
483
           otherwise 0
484

485
  """
486
  try:
487
    pf = open(pidfile, 'r')
488
  except EnvironmentError, err:
489
    if err.errno != errno.ENOENT:
490
      logging.exception("Can't read pid file?!")
491
    return 0
492

    
493
  try:
494
    pid = int(pf.read())
495
  except ValueError, err:
496
    logging.info("Can't parse pid file contents", exc_info=True)
497
    return 0
498

    
499
  return pid
500

    
501

    
502
def MatchNameComponent(key, name_list):
503
  """Try to match a name against a list.
504

505
  This function will try to match a name like test1 against a list
506
  like C{['test1.example.com', 'test2.example.com', ...]}. Against
507
  this list, I{'test1'} as well as I{'test1.example'} will match, but
508
  not I{'test1.ex'}. A multiple match will be considered as no match
509
  at all (e.g. I{'test1'} against C{['test1.example.com',
510
  'test1.example.org']}).
511

512
  @type key: str
513
  @param key: the name to be searched
514
  @type name_list: list
515
  @param name_list: the list of strings against which to search the key
516

517
  @rtype: None or str
518
  @return: None if there is no match I{or} if there are multiple matches,
519
      otherwise the element from the list which matches
520

521
  """
522
  mo = re.compile("^%s(\..*)?$" % re.escape(key))
523
  names_filtered = [name for name in name_list if mo.match(name) is not None]
524
  if len(names_filtered) != 1:
525
    return None
526
  return names_filtered[0]
527

    
528

    
529
class HostInfo:
530
  """Class implementing resolver and hostname functionality
531

532
  """
533
  def __init__(self, name=None):
534
    """Initialize the host name object.
535

536
    If the name argument is not passed, it will use this system's
537
    name.
538

539
    """
540
    if name is None:
541
      name = self.SysName()
542

    
543
    self.query = name
544
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
545
    self.ip = self.ipaddrs[0]
546

    
547
  def ShortName(self):
548
    """Returns the hostname without domain.
549

550
    """
551
    return self.name.split('.')[0]
552

    
553
  @staticmethod
554
  def SysName():
555
    """Return the current system's name.
556

557
    This is simply a wrapper over C{socket.gethostname()}.
558

559
    """
560
    return socket.gethostname()
561

    
562
  @staticmethod
563
  def LookupHostname(hostname):
564
    """Look up hostname
565

566
    @type hostname: str
567
    @param hostname: hostname to look up
568

569
    @rtype: tuple
570
    @return: a tuple (name, aliases, ipaddrs) as returned by
571
        C{socket.gethostbyname_ex}
572
    @raise errors.ResolverError: in case of errors in resolving
573

574
    """
575
    try:
576
      result = socket.gethostbyname_ex(hostname)
577
    except socket.gaierror, err:
578
      # hostname not found in DNS
579
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
580

    
581
    return result
582

    
583

    
584
def ListVolumeGroups():
585
  """List volume groups and their size
586

587
  @rtype: dict
588
  @return:
589
       Dictionary with keys volume name and values
590
       the size of the volume
591

592
  """
593
  command = "vgs --noheadings --units m --nosuffix -o name,size"
594
  result = RunCmd(command)
595
  retval = {}
596
  if result.failed:
597
    return retval
598

    
599
  for line in result.stdout.splitlines():
600
    try:
601
      name, size = line.split()
602
      size = int(float(size))
603
    except (IndexError, ValueError), err:
604
      logging.error("Invalid output from vgs (%s): %s", err, line)
605
      continue
606

    
607
    retval[name] = size
608

    
609
  return retval
610

    
611

    
612
def BridgeExists(bridge):
613
  """Check whether the given bridge exists in the system
614

615
  @type bridge: str
616
  @param bridge: the bridge name to check
617
  @rtype: boolean
618
  @return: True if it does
619

620
  """
621
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
622

    
623

    
624
def NiceSort(name_list):
625
  """Sort a list of strings based on digit and non-digit groupings.
626

627
  Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
628
  will sort the list in the logical order C{['a1', 'a2', 'a10',
629
  'a11']}.
630

631
  The sort algorithm breaks each name in groups of either only-digits
632
  or no-digits. Only the first eight such groups are considered, and
633
  after that we just use what's left of the string.
634

635
  @type name_list: list
636
  @param name_list: the names to be sorted
637
  @rtype: list
638
  @return: a copy of the name list sorted with our algorithm
639

640
  """
641
  _SORTER_BASE = "(\D+|\d+)"
642
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
643
                                                  _SORTER_BASE, _SORTER_BASE,
644
                                                  _SORTER_BASE, _SORTER_BASE,
645
                                                  _SORTER_BASE, _SORTER_BASE)
646
  _SORTER_RE = re.compile(_SORTER_FULL)
647
  _SORTER_NODIGIT = re.compile("^\D*$")
648
  def _TryInt(val):
649
    """Attempts to convert a variable to integer."""
650
    if val is None or _SORTER_NODIGIT.match(val):
651
      return val
652
    rval = int(val)
653
    return rval
654

    
655
  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
656
             for name in name_list]
657
  to_sort.sort()
658
  return [tup[1] for tup in to_sort]
659

    
660

    
661
def TryConvert(fn, val):
662
  """Try to convert a value ignoring errors.
663

664
  This function tries to apply function I{fn} to I{val}. If no
665
  C{ValueError} or C{TypeError} exceptions are raised, it will return
666
  the result, else it will return the original value. Any other
667
  exceptions are propagated to the caller.
668

669
  @type fn: callable
670
  @param fn: function to apply to the value
671
  @param val: the value to be converted
672
  @return: The converted value if the conversion was successful,
673
      otherwise the original value.
674

675
  """
676
  try:
677
    nv = fn(val)
678
  except (ValueError, TypeError), err:
679
    nv = val
680
  return nv
681

    
682

    
683
def IsValidIP(ip):
684
  """Verifies the syntax of an IPv4 address.
685

686
  This function checks if the IPv4 address passes is valid or not based
687
  on syntax (not IP range, class calculations, etc.).
688

689
  @type ip: str
690
  @param ip: the address to be checked
691
  @rtype: a regular expression match object
692
  @return: a regular epression match object, or None if the
693
      address is not valid
694

695
  """
696
  unit = "(0|[1-9]\d{0,2})"
697
  #TODO: convert and return only boolean
698
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
699

    
700

    
701
def IsValidShellParam(word):
702
  """Verifies is the given word is safe from the shell's p.o.v.
703

704
  This means that we can pass this to a command via the shell and be
705
  sure that it doesn't alter the command line and is passed as such to
706
  the actual command.
707

708
  Note that we are overly restrictive here, in order to be on the safe
709
  side.
710

711
  @type word: str
712
  @param word: the word to check
713
  @rtype: boolean
714
  @return: True if the word is 'safe'
715

716
  """
717
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
718

    
719

    
720
def BuildShellCmd(template, *args):
721
  """Build a safe shell command line from the given arguments.
722

723
  This function will check all arguments in the args list so that they
724
  are valid shell parameters (i.e. they don't contain shell
725
  metacharaters). If everything is ok, it will return the result of
726
  template % args.
727

728
  @type template: str
729
  @param template: the string holding the template for the
730
      string formatting
731
  @rtype: str
732
  @return: the expanded command line
733

734
  """
735
  for word in args:
736
    if not IsValidShellParam(word):
737
      raise errors.ProgrammerError("Shell argument '%s' contains"
738
                                   " invalid characters" % word)
739
  return template % args
740

    
741

    
742
def FormatUnit(value, units):
743
  """Formats an incoming number of MiB with the appropriate unit.
744

745
  @type value: int
746
  @param value: integer representing the value in MiB (1048576)
747
  @type units: char
748
  @param units: the type of formatting we should do:
749
      - 'h' for automatic scaling
750
      - 'm' for MiBs
751
      - 'g' for GiBs
752
      - 't' for TiBs
753
  @rtype: str
754
  @return: the formatted value (with suffix)
755

756
  """
757
  if units not in ('m', 'g', 't', 'h'):
758
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
759

    
760
  suffix = ''
761

    
762
  if units == 'm' or (units == 'h' and value < 1024):
763
    if units == 'h':
764
      suffix = 'M'
765
    return "%d%s" % (round(value, 0), suffix)
766

    
767
  elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
768
    if units == 'h':
769
      suffix = 'G'
770
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
771

    
772
  else:
773
    if units == 'h':
774
      suffix = 'T'
775
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
776

    
777

    
778
def ParseUnit(input_string):
779
  """Tries to extract number and scale from the given string.
780

781
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
782
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
783
  is always an int in MiB.
784

785
  """
786
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
787
  if not m:
788
    raise errors.UnitParseError("Invalid format")
789

    
790
  value = float(m.groups()[0])
791

    
792
  unit = m.groups()[1]
793
  if unit:
794
    lcunit = unit.lower()
795
  else:
796
    lcunit = 'm'
797

    
798
  if lcunit in ('m', 'mb', 'mib'):
799
    # Value already in MiB
800
    pass
801

    
802
  elif lcunit in ('g', 'gb', 'gib'):
803
    value *= 1024
804

    
805
  elif lcunit in ('t', 'tb', 'tib'):
806
    value *= 1024 * 1024
807

    
808
  else:
809
    raise errors.UnitParseError("Unknown unit: %s" % unit)
810

    
811
  # Make sure we round up
812
  if int(value) < value:
813
    value += 1
814

    
815
  # Round up to the next multiple of 4
816
  value = int(value)
817
  if value % 4:
818
    value += 4 - value % 4
819

    
820
  return value
821

    
822

    
823
def AddAuthorizedKey(file_name, key):
824
  """Adds an SSH public key to an authorized_keys file.
825

826
  @type file_name: str
827
  @param file_name: path to authorized_keys file
828
  @type key: str
829
  @param key: string containing key
830

831
  """
832
  key_fields = key.split()
833

    
834
  f = open(file_name, 'a+')
835
  try:
836
    nl = True
837
    for line in f:
838
      # Ignore whitespace changes
839
      if line.split() == key_fields:
840
        break
841
      nl = line.endswith('\n')
842
    else:
843
      if not nl:
844
        f.write("\n")
845
      f.write(key.rstrip('\r\n'))
846
      f.write("\n")
847
      f.flush()
848
  finally:
849
    f.close()
850

    
851

    
852
def RemoveAuthorizedKey(file_name, key):
853
  """Removes an SSH public key from an authorized_keys file.
854

855
  @type file_name: str
856
  @param file_name: path to authorized_keys file
857
  @type key: str
858
  @param key: string containing key
859

860
  """
861
  key_fields = key.split()
862

    
863
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
864
  try:
865
    out = os.fdopen(fd, 'w')
866
    try:
867
      f = open(file_name, 'r')
868
      try:
869
        for line in f:
870
          # Ignore whitespace changes while comparing lines
871
          if line.split() != key_fields:
872
            out.write(line)
873

    
874
        out.flush()
875
        os.rename(tmpname, file_name)
876
      finally:
877
        f.close()
878
    finally:
879
      out.close()
880
  except:
881
    RemoveFile(tmpname)
882
    raise
883

    
884

    
885
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
886
  """Sets the name of an IP address and hostname in /etc/hosts.
887

888
  @type file_name: str
889
  @param file_name: path to the file to modify (usually C{/etc/hosts})
890
  @type ip: str
891
  @param ip: the IP address
892
  @type hostname: str
893
  @param hostname: the hostname to be added
894
  @type aliases: list
895
  @param aliases: the list of aliases to add for the hostname
896

897
  """
898
  # FIXME: use WriteFile + fn rather than duplicating its efforts
899
  # Ensure aliases are unique
900
  aliases = UniqueSequence([hostname] + aliases)[1:]
901

    
902
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
903
  try:
904
    out = os.fdopen(fd, 'w')
905
    try:
906
      f = open(file_name, 'r')
907
      try:
908
        for line in f:
909
          fields = line.split()
910
          if fields and not fields[0].startswith('#') and ip == fields[0]:
911
            continue
912
          out.write(line)
913

    
914
        out.write("%s\t%s" % (ip, hostname))
915
        if aliases:
916
          out.write(" %s" % ' '.join(aliases))
917
        out.write('\n')
918

    
919
        out.flush()
920
        os.fsync(out)
921
        os.chmod(tmpname, 0644)
922
        os.rename(tmpname, file_name)
923
      finally:
924
        f.close()
925
    finally:
926
      out.close()
927
  except:
928
    RemoveFile(tmpname)
929
    raise
930

    
931

    
932
def AddHostToEtcHosts(hostname):
933
  """Wrapper around SetEtcHostsEntry.
934

935
  @type hostname: str
936
  @param hostname: a hostname that will be resolved and added to
937
      L{constants.ETC_HOSTS}
938

939
  """
940
  hi = HostInfo(name=hostname)
941
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
942

    
943

    
944
def RemoveEtcHostsEntry(file_name, hostname):
945
  """Removes a hostname from /etc/hosts.
946

947
  IP addresses without names are removed from the file.
948

949
  @type file_name: str
950
  @param file_name: path to the file to modify (usually C{/etc/hosts})
951
  @type hostname: str
952
  @param hostname: the hostname to be removed
953

954
  """
955
  # FIXME: use WriteFile + fn rather than duplicating its efforts
956
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
957
  try:
958
    out = os.fdopen(fd, 'w')
959
    try:
960
      f = open(file_name, 'r')
961
      try:
962
        for line in f:
963
          fields = line.split()
964
          if len(fields) > 1 and not fields[0].startswith('#'):
965
            names = fields[1:]
966
            if hostname in names:
967
              while hostname in names:
968
                names.remove(hostname)
969
              if names:
970
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
971
              continue
972

    
973
          out.write(line)
974

    
975
        out.flush()
976
        os.fsync(out)
977
        os.chmod(tmpname, 0644)
978
        os.rename(tmpname, file_name)
979
      finally:
980
        f.close()
981
    finally:
982
      out.close()
983
  except:
984
    RemoveFile(tmpname)
985
    raise
986

    
987

    
988
def RemoveHostFromEtcHosts(hostname):
989
  """Wrapper around RemoveEtcHostsEntry.
990

991
  @type hostname: str
992
  @param hostname: hostname that will be resolved and its
993
      full and shot name will be removed from
994
      L{constants.ETC_HOSTS}
995

996
  """
997
  hi = HostInfo(name=hostname)
998
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
999
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
1000

    
1001

    
1002
def CreateBackup(file_name):
1003
  """Creates a backup of a file.
1004

1005
  @type file_name: str
1006
  @param file_name: file to be backed up
1007
  @rtype: str
1008
  @return: the path to the newly created backup
1009
  @raise errors.ProgrammerError: for invalid file names
1010

1011
  """
1012
  if not os.path.isfile(file_name):
1013
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1014
                                file_name)
1015

    
1016
  prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1017
  dir_name = os.path.dirname(file_name)
1018

    
1019
  fsrc = open(file_name, 'rb')
1020
  try:
1021
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1022
    fdst = os.fdopen(fd, 'wb')
1023
    try:
1024
      shutil.copyfileobj(fsrc, fdst)
1025
    finally:
1026
      fdst.close()
1027
  finally:
1028
    fsrc.close()
1029

    
1030
  return backup_name
1031

    
1032

    
1033
def ShellQuote(value):
1034
  """Quotes shell argument according to POSIX.
1035

1036
  @type value: str
1037
  @param value: the argument to be quoted
1038
  @rtype: str
1039
  @return: the quoted value
1040

1041
  """
1042
  if _re_shell_unquoted.match(value):
1043
    return value
1044
  else:
1045
    return "'%s'" % value.replace("'", "'\\''")
1046

    
1047

    
1048
def ShellQuoteArgs(args):
1049
  """Quotes a list of shell arguments.
1050

1051
  @type args: list
1052
  @param args: list of arguments to be quoted
1053
  @rtype: str
1054
  @return: the quoted arguments concatenaned with spaces
1055

1056
  """
1057
  return ' '.join([ShellQuote(i) for i in args])
1058

    
1059

    
1060
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1061
  """Simple ping implementation using TCP connect(2).
1062

1063
  Check if the given IP is reachable by doing attempting a TCP connect
1064
  to it.
1065

1066
  @type target: str
1067
  @param target: the IP or hostname to ping
1068
  @type port: int
1069
  @param port: the port to connect to
1070
  @type timeout: int
1071
  @param timeout: the timeout on the connection attemp
1072
  @type live_port_needed: boolean
1073
  @param live_port_needed: whether a closed port will cause the
1074
      function to return failure, as if there was a timeout
1075
  @type source: str or None
1076
  @param source: if specified, will cause the connect to be made
1077
      from this specific source address; failures to bind other
1078
      than C{EADDRNOTAVAIL} will be ignored
1079

1080
  """
1081
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1082

    
1083
  success = False
1084

    
1085
  if source is not None:
1086
    try:
1087
      sock.bind((source, 0))
1088
    except socket.error, (errcode, errstring):
1089
      if errcode == errno.EADDRNOTAVAIL:
1090
        success = False
1091

    
1092
  sock.settimeout(timeout)
1093

    
1094
  try:
1095
    sock.connect((target, port))
1096
    sock.close()
1097
    success = True
1098
  except socket.timeout:
1099
    success = False
1100
  except socket.error, (errcode, errstring):
1101
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1102

    
1103
  return success
1104

    
1105

    
1106
def OwnIpAddress(address):
1107
  """Check if the current host has the the given IP address.
1108

1109
  Currently this is done by TCP-pinging the address from the loopback
1110
  address.
1111

1112
  @type address: string
1113
  @param address: the addres to check
1114
  @rtype: bool
1115
  @return: True if we own the address
1116

1117
  """
1118
  return TcpPing(address, constants.DEFAULT_NODED_PORT,
1119
                 source=constants.LOCALHOST_IP_ADDRESS)
1120

    
1121

    
1122
def ListVisibleFiles(path):
1123
  """Returns a list of visible files in a directory.
1124

1125
  @type path: str
1126
  @param path: the directory to enumerate
1127
  @rtype: list
1128
  @return: the list of all files not starting with a dot
1129

1130
  """
1131
  files = [i for i in os.listdir(path) if not i.startswith(".")]
1132
  files.sort()
1133
  return files
1134

    
1135

    
1136
def GetHomeDir(user, default=None):
1137
  """Try to get the homedir of the given user.
1138

1139
  The user can be passed either as a string (denoting the name) or as
1140
  an integer (denoting the user id). If the user is not found, the
1141
  'default' argument is returned, which defaults to None.
1142

1143
  """
1144
  try:
1145
    if isinstance(user, basestring):
1146
      result = pwd.getpwnam(user)
1147
    elif isinstance(user, (int, long)):
1148
      result = pwd.getpwuid(user)
1149
    else:
1150
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1151
                                   type(user))
1152
  except KeyError:
1153
    return default
1154
  return result.pw_dir
1155

    
1156

    
1157
def NewUUID():
1158
  """Returns a random UUID.
1159

1160
  @note: This is a Linux-specific method as it uses the /proc
1161
      filesystem.
1162
  @rtype: str
1163

1164
  """
1165
  f = open("/proc/sys/kernel/random/uuid", "r")
1166
  try:
1167
    return f.read(128).rstrip("\n")
1168
  finally:
1169
    f.close()
1170

    
1171

    
1172
def GenerateSecret():
1173
  """Generates a random secret.
1174

1175
  This will generate a pseudo-random secret, and return its sha digest
1176
  (so that it can be used where an ASCII string is needed).
1177

1178
  @rtype: str
1179
  @return: a sha1 hexdigest of a block of 64 random bytes
1180

1181
  """
1182
  return sha.new(os.urandom(64)).hexdigest()
1183

    
1184

    
1185
def ReadFile(file_name, size=None):
1186
  """Reads a file.
1187

1188
  @type size: None or int
1189
  @param size: Read at most size bytes
1190
  @rtype: str
1191
  @return: the (possibly partial) conent of the file
1192

1193
  """
1194
  f = open(file_name, "r")
1195
  try:
1196
    if size is None:
1197
      return f.read()
1198
    else:
1199
      return f.read(size)
1200
  finally:
1201
    f.close()
1202

    
1203

    
1204
def WriteFile(file_name, fn=None, data=None,
1205
              mode=None, uid=-1, gid=-1,
1206
              atime=None, mtime=None, close=True,
1207
              dry_run=False, backup=False,
1208
              prewrite=None, postwrite=None):
1209
  """(Over)write a file atomically.
1210

1211
  The file_name and either fn (a function taking one argument, the
1212
  file descriptor, and which should write the data to it) or data (the
1213
  contents of the file) must be passed. The other arguments are
1214
  optional and allow setting the file mode, owner and group, and the
1215
  mtime/atime of the file.
1216

1217
  If the function doesn't raise an exception, it has succeeded and the
1218
  target file has the new contents. If the function has raised an
1219
  exception, an existing target file should be unmodified and the
1220
  temporary file should be removed.
1221

1222
  @type file_name: str
1223
  @param file_name: the target filename
1224
  @type fn: callable
1225
  @param fn: content writing function, called with
1226
      file descriptor as parameter
1227
  @type data: str
1228
  @param data: contents of the file
1229
  @type mode: int
1230
  @param mode: file mode
1231
  @type uid: int
1232
  @param uid: the owner of the file
1233
  @type gid: int
1234
  @param gid: the group of the file
1235
  @type atime: int
1236
  @param atime: a custom access time to be set on the file
1237
  @type mtime: int
1238
  @param mtime: a custom modification time to be set on the file
1239
  @type close: boolean
1240
  @param close: whether to close file after writing it
1241
  @type prewrite: callable
1242
  @param prewrite: function to be called before writing content
1243
  @type postwrite: callable
1244
  @param postwrite: function to be called after writing content
1245

1246
  @rtype: None or int
1247
  @return: None if the 'close' parameter evaluates to True,
1248
      otherwise the file descriptor
1249

1250
  @raise errors.ProgrammerError: if any of the arguments are not valid
1251

1252
  """
1253
  if not os.path.isabs(file_name):
1254
    raise errors.ProgrammerError("Path passed to WriteFile is not"
1255
                                 " absolute: '%s'" % file_name)
1256

    
1257
  if [fn, data].count(None) != 1:
1258
    raise errors.ProgrammerError("fn or data required")
1259

    
1260
  if [atime, mtime].count(None) == 1:
1261
    raise errors.ProgrammerError("Both atime and mtime must be either"
1262
                                 " set or None")
1263

    
1264
  if backup and not dry_run and os.path.isfile(file_name):
1265
    CreateBackup(file_name)
1266

    
1267
  dir_name, base_name = os.path.split(file_name)
1268
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1269
  # here we need to make sure we remove the temp file, if any error
1270
  # leaves it in place
1271
  try:
1272
    if uid != -1 or gid != -1:
1273
      os.chown(new_name, uid, gid)
1274
    if mode:
1275
      os.chmod(new_name, mode)
1276
    if callable(prewrite):
1277
      prewrite(fd)
1278
    if data is not None:
1279
      os.write(fd, data)
1280
    else:
1281
      fn(fd)
1282
    if callable(postwrite):
1283
      postwrite(fd)
1284
    os.fsync(fd)
1285
    if atime is not None and mtime is not None:
1286
      os.utime(new_name, (atime, mtime))
1287
    if not dry_run:
1288
      os.rename(new_name, file_name)
1289
  finally:
1290
    if close:
1291
      os.close(fd)
1292
      result = None
1293
    else:
1294
      result = fd
1295
    RemoveFile(new_name)
1296

    
1297
  return result
1298

    
1299

    
1300
def FirstFree(seq, base=0):
1301
  """Returns the first non-existing integer from seq.
1302

1303
  The seq argument should be a sorted list of positive integers. The
1304
  first time the index of an element is smaller than the element
1305
  value, the index will be returned.
1306

1307
  The base argument is used to start at a different offset,
1308
  i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1309

1310
  Example: C{[0, 1, 3]} will return I{2}.
1311

1312
  @type seq: sequence
1313
  @param seq: the sequence to be analyzed.
1314
  @type base: int
1315
  @param base: use this value as the base index of the sequence
1316
  @rtype: int
1317
  @return: the first non-used index in the sequence
1318

1319
  """
1320
  for idx, elem in enumerate(seq):
1321
    assert elem >= base, "Passed element is higher than base offset"
1322
    if elem > idx + base:
1323
      # idx is not used
1324
      return idx + base
1325
  return None
1326

    
1327

    
1328
def all(seq, pred=bool):
1329
  "Returns True if pred(x) is True for every element in the iterable"
1330
  for elem in itertools.ifilterfalse(pred, seq):
1331
    return False
1332
  return True
1333

    
1334

    
1335
def any(seq, pred=bool):
1336
  "Returns True if pred(x) is True for at least one element in the iterable"
1337
  for elem in itertools.ifilter(pred, seq):
1338
    return True
1339
  return False
1340

    
1341

    
1342
def UniqueSequence(seq):
1343
  """Returns a list with unique elements.
1344

1345
  Element order is preserved.
1346

1347
  @type seq: sequence
1348
  @param seq: the sequence with the source elementes
1349
  @rtype: list
1350
  @return: list of unique elements from seq
1351

1352
  """
1353
  seen = set()
1354
  return [i for i in seq if i not in seen and not seen.add(i)]
1355

    
1356

    
1357
def IsValidMac(mac):
1358
  """Predicate to check if a MAC address is valid.
1359

1360
  Checks wether the supplied MAC address is formally correct, only
1361
  accepts colon separated format.
1362

1363
  @type mac: str
1364
  @param mac: the MAC to be validated
1365
  @rtype: boolean
1366
  @return: True is the MAC seems valid
1367

1368
  """
1369
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1370
  return mac_check.match(mac) is not None
1371

    
1372

    
1373
def TestDelay(duration):
1374
  """Sleep for a fixed amount of time.
1375

1376
  @type duration: float
1377
  @param duration: the sleep duration
1378
  @rtype: boolean
1379
  @return: False for negative value, True otherwise
1380

1381
  """
1382
  if duration < 0:
1383
    return False
1384
  time.sleep(duration)
1385
  return True
1386

    
1387

    
1388
def _CloseFDNoErr(fd, retries=5):
1389
  """Close a file descriptor ignoring errors.
1390

1391
  @type fd: int
1392
  @param fd: the file descriptor
1393
  @type retries: int
1394
  @param retries: how many retries to make, in case we get any
1395
      other error than EBADF
1396

1397
  """
1398
  try:
1399
    os.close(fd)
1400
  except OSError, err:
1401
    if err.errno != errno.EBADF:
1402
      if retries > 0:
1403
        _CloseFDNoErr(fd, retries - 1)
1404
    # else either it's closed already or we're out of retries, so we
1405
    # ignore this and go on
1406

    
1407

    
1408
def CloseFDs(noclose_fds=None):
1409
  """Close file descriptors.
1410

1411
  This closes all file descriptors above 2 (i.e. except
1412
  stdin/out/err).
1413

1414
  @type noclose_fds: list or None
1415
  @param noclose_fds: if given, it denotes a list of file descriptor
1416
      that should not be closed
1417

1418
  """
1419
  # Default maximum for the number of available file descriptors.
1420
  if 'SC_OPEN_MAX' in os.sysconf_names:
1421
    try:
1422
      MAXFD = os.sysconf('SC_OPEN_MAX')
1423
      if MAXFD < 0:
1424
        MAXFD = 1024
1425
    except OSError:
1426
      MAXFD = 1024
1427
  else:
1428
    MAXFD = 1024
1429
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1430
  if (maxfd == resource.RLIM_INFINITY):
1431
    maxfd = MAXFD
1432

    
1433
  # Iterate through and close all file descriptors (except the standard ones)
1434
  for fd in range(3, maxfd):
1435
    if noclose_fds and fd in noclose_fds:
1436
      continue
1437
    _CloseFDNoErr(fd)
1438

    
1439

    
1440
def Daemonize(logfile):
1441
  """Daemonize the current process.
1442

1443
  This detaches the current process from the controlling terminal and
1444
  runs it in the background as a daemon.
1445

1446
  @type logfile: str
1447
  @param logfile: the logfile to which we should redirect stdout/stderr
1448
  @rtype: int
1449
  @returns: the value zero
1450

1451
  """
1452
  UMASK = 077
1453
  WORKDIR = "/"
1454

    
1455
  # this might fail
1456
  pid = os.fork()
1457
  if (pid == 0):  # The first child.
1458
    os.setsid()
1459
    # this might fail
1460
    pid = os.fork() # Fork a second child.
1461
    if (pid == 0):  # The second child.
1462
      os.chdir(WORKDIR)
1463
      os.umask(UMASK)
1464
    else:
1465
      # exit() or _exit()?  See below.
1466
      os._exit(0) # Exit parent (the first child) of the second child.
1467
  else:
1468
    os._exit(0) # Exit parent of the first child.
1469

    
1470
  for fd in range(3):
1471
    _CloseFDNoErr(fd)
1472
  i = os.open("/dev/null", os.O_RDONLY) # stdin
1473
  assert i == 0, "Can't close/reopen stdin"
1474
  i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1475
  assert i == 1, "Can't close/reopen stdout"
1476
  # Duplicate standard output to standard error.
1477
  os.dup2(1, 2)
1478
  return 0
1479

    
1480

    
1481
def DaemonPidFileName(name):
1482
  """Compute a ganeti pid file absolute path
1483

1484
  @type name: str
1485
  @param name: the daemon name
1486
  @rtype: str
1487
  @return: the full path to the pidfile corresponding to the given
1488
      daemon name
1489

1490
  """
1491
  return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1492

    
1493

    
1494
def WritePidFile(name):
1495
  """Write the current process pidfile.
1496

1497
  The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1498

1499
  @type name: str
1500
  @param name: the daemon name to use
1501
  @raise errors.GenericError: if the pid file already exists and
1502
      points to a live process
1503

1504
  """
1505
  pid = os.getpid()
1506
  pidfilename = DaemonPidFileName(name)
1507
  if IsProcessAlive(ReadPidFile(pidfilename)):
1508
    raise errors.GenericError("%s contains a live process" % pidfilename)
1509

    
1510
  WriteFile(pidfilename, data="%d\n" % pid)
1511

    
1512

    
1513
def RemovePidFile(name):
1514
  """Remove the current process pidfile.
1515

1516
  Any errors are ignored.
1517

1518
  @type name: str
1519
  @param name: the daemon name used to derive the pidfile name
1520

1521
  """
1522
  pid = os.getpid()
1523
  pidfilename = DaemonPidFileName(name)
1524
  # TODO: we could check here that the file contains our pid
1525
  try:
1526
    RemoveFile(pidfilename)
1527
  except:
1528
    pass
1529

    
1530

    
1531
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1532
                waitpid=False):
1533
  """Kill a process given by its pid.
1534

1535
  @type pid: int
1536
  @param pid: The PID to terminate.
1537
  @type signal_: int
1538
  @param signal_: The signal to send, by default SIGTERM
1539
  @type timeout: int
1540
  @param timeout: The timeout after which, if the process is still alive,
1541
                  a SIGKILL will be sent. If not positive, no such checking
1542
                  will be done
1543
  @type waitpid: boolean
1544
  @param waitpid: If true, we should waitpid on this process after
1545
      sending signals, since it's our own child and otherwise it
1546
      would remain as zombie
1547

1548
  """
1549
  def _helper(pid, signal_, wait):
1550
    """Simple helper to encapsulate the kill/waitpid sequence"""
1551
    os.kill(pid, signal_)
1552
    if wait:
1553
      try:
1554
        os.waitpid(pid, os.WNOHANG)
1555
      except OSError:
1556
        pass
1557

    
1558
  if pid <= 0:
1559
    # kill with pid=0 == suicide
1560
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1561

    
1562
  if not IsProcessAlive(pid):
1563
    return
1564
  _helper(pid, signal_, waitpid)
1565
  if timeout <= 0:
1566
    return
1567

    
1568
  # Wait up to $timeout seconds
1569
  end = time.time() + timeout
1570
  wait = 0.01
1571
  while time.time() < end and IsProcessAlive(pid):
1572
    try:
1573
      (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1574
      if result_pid > 0:
1575
        break
1576
    except OSError:
1577
      pass
1578
    time.sleep(wait)
1579
    # Make wait time longer for next try
1580
    if wait < 0.1:
1581
      wait *= 1.5
1582

    
1583
  if IsProcessAlive(pid):
1584
    # Kill process if it's still alive
1585
    _helper(pid, signal.SIGKILL, waitpid)
1586

    
1587

    
1588
def FindFile(name, search_path, test=os.path.exists):
1589
  """Look for a filesystem object in a given path.
1590

1591
  This is an abstract method to search for filesystem object (files,
1592
  dirs) under a given search path.
1593

1594
  @type name: str
1595
  @param name: the name to look for
1596
  @type search_path: str
1597
  @param search_path: location to start at
1598
  @type test: callable
1599
  @param test: a function taking one argument that should return True
1600
      if the a given object is valid; the default value is
1601
      os.path.exists, causing only existing files to be returned
1602
  @rtype: str or None
1603
  @return: full path to the object if found, None otherwise
1604

1605
  """
1606
  for dir_name in search_path:
1607
    item_name = os.path.sep.join([dir_name, name])
1608
    if test(item_name):
1609
      return item_name
1610
  return None
1611

    
1612

    
1613
def CheckVolumeGroupSize(vglist, vgname, minsize):
1614
  """Checks if the volume group list is valid.
1615

1616
  The function will check if a given volume group is in the list of
1617
  volume groups and has a minimum size.
1618

1619
  @type vglist: dict
1620
  @param vglist: dictionary of volume group names and their size
1621
  @type vgname: str
1622
  @param vgname: the volume group we should check
1623
  @type minsize: int
1624
  @param minsize: the minimum size we accept
1625
  @rtype: None or str
1626
  @return: None for success, otherwise the error message
1627

1628
  """
1629
  vgsize = vglist.get(vgname, None)
1630
  if vgsize is None:
1631
    return "volume group '%s' missing" % vgname
1632
  elif vgsize < minsize:
1633
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1634
            (vgname, minsize, vgsize))
1635
  return None
1636

    
1637

    
1638
def SplitTime(value):
1639
  """Splits time as floating point number into a tuple.
1640

1641
  @param value: Time in seconds
1642
  @type value: int or float
1643
  @return: Tuple containing (seconds, microseconds)
1644

1645
  """
1646
  (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1647

    
1648
  assert 0 <= seconds, \
1649
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1650
  assert 0 <= microseconds <= 999999, \
1651
    "Microseconds must be 0-999999, but are %s" % microseconds
1652

    
1653
  return (int(seconds), int(microseconds))
1654

    
1655

    
1656
def MergeTime(timetuple):
1657
  """Merges a tuple into time as a floating point number.
1658

1659
  @param timetuple: Time as tuple, (seconds, microseconds)
1660
  @type timetuple: tuple
1661
  @return: Time as a floating point number expressed in seconds
1662

1663
  """
1664
  (seconds, microseconds) = timetuple
1665

    
1666
  assert 0 <= seconds, \
1667
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1668
  assert 0 <= microseconds <= 999999, \
1669
    "Microseconds must be 0-999999, but are %s" % microseconds
1670

    
1671
  return float(seconds) + (float(microseconds) * 0.000001)
1672

    
1673

    
1674
def GetNodeDaemonPort():
1675
  """Get the node daemon port for this cluster.
1676

1677
  Note that this routine does not read a ganeti-specific file, but
1678
  instead uses C{socket.getservbyname} to allow pre-customization of
1679
  this parameter outside of Ganeti.
1680

1681
  @rtype: int
1682

1683
  """
1684
  try:
1685
    port = socket.getservbyname("ganeti-noded", "tcp")
1686
  except socket.error:
1687
    port = constants.DEFAULT_NODED_PORT
1688

    
1689
  return port
1690

    
1691

    
1692
def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1693
                 multithreaded=False):
1694
  """Configures the logging module.
1695

1696
  @type logfile: str
1697
  @param logfile: the filename to which we should log
1698
  @type debug: boolean
1699
  @param debug: whether to enable debug messages too or
1700
      only those at C{INFO} and above level
1701
  @type stderr_logging: boolean
1702
  @param stderr_logging: whether we should also log to the standard error
1703
  @type program: str
1704
  @param program: the name under which we should log messages
1705
  @type multithreaded: boolean
1706
  @param multithreaded: if True, will add the thread name to the log file
1707
  @raise EnvironmentError: if we can't open the log file and
1708
      stderr logging is disabled
1709

1710
  """
1711
  fmt = "%(asctime)s: " + program + " pid=%(process)d"
1712
  if multithreaded:
1713
    fmt += "/%(threadName)s"
1714
  if debug:
1715
    fmt += " %(module)s:%(lineno)s"
1716
  fmt += " %(levelname)s %(message)s"
1717
  formatter = logging.Formatter(fmt)
1718

    
1719
  root_logger = logging.getLogger("")
1720
  root_logger.setLevel(logging.NOTSET)
1721

    
1722
  # Remove all previously setup handlers
1723
  for handler in root_logger.handlers:
1724
    handler.close()
1725
    root_logger.removeHandler(handler)
1726

    
1727
  if stderr_logging:
1728
    stderr_handler = logging.StreamHandler()
1729
    stderr_handler.setFormatter(formatter)
1730
    if debug:
1731
      stderr_handler.setLevel(logging.NOTSET)
1732
    else:
1733
      stderr_handler.setLevel(logging.CRITICAL)
1734
    root_logger.addHandler(stderr_handler)
1735

    
1736
  # this can fail, if the logging directories are not setup or we have
1737
  # a permisssion problem; in this case, it's best to log but ignore
1738
  # the error if stderr_logging is True, and if false we re-raise the
1739
  # exception since otherwise we could run but without any logs at all
1740
  try:
1741
    logfile_handler = logging.FileHandler(logfile)
1742
    logfile_handler.setFormatter(formatter)
1743
    if debug:
1744
      logfile_handler.setLevel(logging.DEBUG)
1745
    else:
1746
      logfile_handler.setLevel(logging.INFO)
1747
    root_logger.addHandler(logfile_handler)
1748
  except EnvironmentError:
1749
    if stderr_logging:
1750
      logging.exception("Failed to enable logging to file '%s'", logfile)
1751
    else:
1752
      # we need to re-raise the exception
1753
      raise
1754

    
1755

    
1756
def TailFile(fname, lines=20):
1757
  """Return the last lines from a file.
1758

1759
  @note: this function will only read and parse the last 4KB of
1760
      the file; if the lines are very long, it could be that less
1761
      than the requested number of lines are returned
1762

1763
  @param fname: the file name
1764
  @type lines: int
1765
  @param lines: the (maximum) number of lines to return
1766

1767
  """
1768
  fd = open(fname, "r")
1769
  try:
1770
    fd.seek(0, 2)
1771
    pos = fd.tell()
1772
    pos = max(0, pos-4096)
1773
    fd.seek(pos, 0)
1774
    raw_data = fd.read()
1775
  finally:
1776
    fd.close()
1777

    
1778
  rows = raw_data.splitlines()
1779
  return rows[-lines:]
1780

    
1781

    
1782
def SafeEncode(text):
1783
  """Return a 'safe' version of a source string.
1784

1785
  This function mangles the input string and returns a version that
1786
  should be safe to disply/encode as ASCII. To this end, we first
1787
  convert it to ASCII using the 'backslashreplace' encoding which
1788
  should get rid of any non-ASCII chars, and then we again encode it
1789
  via 'string_escape' which converts '\n' into '\\n' so that log
1790
  messages remain one-line.
1791

1792
  @type text: str or unicode
1793
  @param text: input data
1794
  @rtype: str
1795
  @return: a safe version of text
1796

1797
  """
1798
  text = text.encode('ascii', 'backslashreplace')
1799
  text = text.encode('string_escape')
1800
  return text
1801

    
1802

    
1803
def LockedMethod(fn):
1804
  """Synchronized object access decorator.
1805

1806
  This decorator is intended to protect access to an object using the
1807
  object's own lock which is hardcoded to '_lock'.
1808

1809
  """
1810
  def _LockDebug(*args, **kwargs):
1811
    if debug_locks:
1812
      logging.debug(*args, **kwargs)
1813

    
1814
  def wrapper(self, *args, **kwargs):
1815
    assert hasattr(self, '_lock')
1816
    lock = self._lock
1817
    _LockDebug("Waiting for %s", lock)
1818
    lock.acquire()
1819
    try:
1820
      _LockDebug("Acquired %s", lock)
1821
      result = fn(self, *args, **kwargs)
1822
    finally:
1823
      _LockDebug("Releasing %s", lock)
1824
      lock.release()
1825
      _LockDebug("Released %s", lock)
1826
    return result
1827
  return wrapper
1828

    
1829

    
1830
def LockFile(fd):
1831
  """Locks a file using POSIX locks.
1832

1833
  @type fd: int
1834
  @param fd: the file descriptor we need to lock
1835

1836
  """
1837
  try:
1838
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1839
  except IOError, err:
1840
    if err.errno == errno.EAGAIN:
1841
      raise errors.LockError("File already locked")
1842
    raise
1843

    
1844

    
1845
class FileLock(object):
1846
  """Utility class for file locks.
1847

1848
  """
1849
  def __init__(self, filename):
1850
    """Constructor for FileLock.
1851

1852
    This will open the file denoted by the I{filename} argument.
1853

1854
    @type filename: str
1855
    @param filename: path to the file to be locked
1856

1857
    """
1858
    self.filename = filename
1859
    self.fd = open(self.filename, "w")
1860

    
1861
  def __del__(self):
1862
    self.Close()
1863

    
1864
  def Close(self):
1865
    """Close the file and release the lock.
1866

1867
    """
1868
    if self.fd:
1869
      self.fd.close()
1870
      self.fd = None
1871

    
1872
  def _flock(self, flag, blocking, timeout, errmsg):
1873
    """Wrapper for fcntl.flock.
1874

1875
    @type flag: int
1876
    @param flag: operation flag
1877
    @type blocking: bool
1878
    @param blocking: whether the operation should be done in blocking mode.
1879
    @type timeout: None or float
1880
    @param timeout: for how long the operation should be retried (implies
1881
                    non-blocking mode).
1882
    @type errmsg: string
1883
    @param errmsg: error message in case operation fails.
1884

1885
    """
1886
    assert self.fd, "Lock was closed"
1887
    assert timeout is None or timeout >= 0, \
1888
      "If specified, timeout must be positive"
1889

    
1890
    if timeout is not None:
1891
      flag |= fcntl.LOCK_NB
1892
      timeout_end = time.time() + timeout
1893

    
1894
    # Blocking doesn't have effect with timeout
1895
    elif not blocking:
1896
      flag |= fcntl.LOCK_NB
1897
      timeout_end = None
1898

    
1899
    retry = True
1900
    while retry:
1901
      try:
1902
        fcntl.flock(self.fd, flag)
1903
        retry = False
1904
      except IOError, err:
1905
        if err.errno in (errno.EAGAIN, ):
1906
          if timeout_end is not None and time.time() < timeout_end:
1907
            # Wait before trying again
1908
            time.sleep(max(0.1, min(1.0, timeout)))
1909
          else:
1910
            raise errors.LockError(errmsg)
1911
        else:
1912
          logging.exception("fcntl.flock failed")
1913
          raise
1914

    
1915
  def Exclusive(self, blocking=False, timeout=None):
1916
    """Locks the file in exclusive mode.
1917

1918
    @type blocking: boolean
1919
    @param blocking: whether to block and wait until we
1920
        can lock the file or return immediately
1921
    @type timeout: int or None
1922
    @param timeout: if not None, the duration to wait for the lock
1923
        (in blocking mode)
1924

1925
    """
1926
    self._flock(fcntl.LOCK_EX, blocking, timeout,
1927
                "Failed to lock %s in exclusive mode" % self.filename)
1928

    
1929
  def Shared(self, blocking=False, timeout=None):
1930
    """Locks the file in shared mode.
1931

1932
    @type blocking: boolean
1933
    @param blocking: whether to block and wait until we
1934
        can lock the file or return immediately
1935
    @type timeout: int or None
1936
    @param timeout: if not None, the duration to wait for the lock
1937
        (in blocking mode)
1938

1939
    """
1940
    self._flock(fcntl.LOCK_SH, blocking, timeout,
1941
                "Failed to lock %s in shared mode" % self.filename)
1942

    
1943
  def Unlock(self, blocking=True, timeout=None):
1944
    """Unlocks the file.
1945

1946
    According to C{flock(2)}, unlocking can also be a nonblocking
1947
    operation::
1948

1949
      To make a non-blocking request, include LOCK_NB with any of the above
1950
      operations.
1951

1952
    @type blocking: boolean
1953
    @param blocking: whether to block and wait until we
1954
        can lock the file or return immediately
1955
    @type timeout: int or None
1956
    @param timeout: if not None, the duration to wait for the lock
1957
        (in blocking mode)
1958

1959
    """
1960
    self._flock(fcntl.LOCK_UN, blocking, timeout,
1961
                "Failed to unlock %s" % self.filename)
1962

    
1963

    
1964
class SignalHandler(object):
1965
  """Generic signal handler class.
1966

1967
  It automatically restores the original handler when deconstructed or
1968
  when L{Reset} is called. You can either pass your own handler
1969
  function in or query the L{called} attribute to detect whether the
1970
  signal was sent.
1971

1972
  @type signum: list
1973
  @ivar signum: the signals we handle
1974
  @type called: boolean
1975
  @ivar called: tracks whether any of the signals have been raised
1976

1977
  """
1978
  def __init__(self, signum):
1979
    """Constructs a new SignalHandler instance.
1980

1981
    @type signum: int or list of ints
1982
    @param signum: Single signal number or set of signal numbers
1983

1984
    """
1985
    if isinstance(signum, (int, long)):
1986
      self.signum = set([signum])
1987
    else:
1988
      self.signum = set(signum)
1989

    
1990
    self.called = False
1991

    
1992
    self._previous = {}
1993
    try:
1994
      for signum in self.signum:
1995
        # Setup handler
1996
        prev_handler = signal.signal(signum, self._HandleSignal)
1997
        try:
1998
          self._previous[signum] = prev_handler
1999
        except:
2000
          # Restore previous handler
2001
          signal.signal(signum, prev_handler)
2002
          raise
2003
    except:
2004
      # Reset all handlers
2005
      self.Reset()
2006
      # Here we have a race condition: a handler may have already been called,
2007
      # but there's not much we can do about it at this point.
2008
      raise
2009

    
2010
  def __del__(self):
2011
    self.Reset()
2012

    
2013
  def Reset(self):
2014
    """Restore previous handler.
2015

2016
    This will reset all the signals to their previous handlers.
2017

2018
    """
2019
    for signum, prev_handler in self._previous.items():
2020
      signal.signal(signum, prev_handler)
2021
      # If successful, remove from dict
2022
      del self._previous[signum]
2023

    
2024
  def Clear(self):
2025
    """Unsets the L{called} flag.
2026

2027
    This function can be used in case a signal may arrive several times.
2028

2029
    """
2030
    self.called = False
2031

    
2032
  def _HandleSignal(self, signum, frame):
2033
    """Actual signal handling function.
2034

2035
    """
2036
    # This is not nice and not absolutely atomic, but it appears to be the only
2037
    # solution in Python -- there are no atomic types.
2038
    self.called = True
2039

    
2040

    
2041
class FieldSet(object):
2042
  """A simple field set.
2043

2044
  Among the features are:
2045
    - checking if a string is among a list of static string or regex objects
2046
    - checking if a whole list of string matches
2047
    - returning the matching groups from a regex match
2048

2049
  Internally, all fields are held as regular expression objects.
2050

2051
  """
2052
  def __init__(self, *items):
2053
    self.items = [re.compile("^%s$" % value) for value in items]
2054

    
2055
  def Extend(self, other_set):
2056
    """Extend the field set with the items from another one"""
2057
    self.items.extend(other_set.items)
2058

    
2059
  def Matches(self, field):
2060
    """Checks if a field matches the current set
2061

2062
    @type field: str
2063
    @param field: the string to match
2064
    @return: either False or a regular expression match object
2065

2066
    """
2067
    for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2068
      return m
2069
    return False
2070

    
2071
  def NonMatching(self, items):
2072
    """Returns the list of fields not matching the current set
2073

2074
    @type items: list
2075
    @param items: the list of fields to check
2076
    @rtype: list
2077
    @return: list of non-matching fields
2078

2079
    """
2080
    return [val for val in items if not self.Matches(val)]