Statistics
| Branch: | Tag: | Revision:

root / lib / utils.py @ 451575de

History | View | Annotate | Download (58.4 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Ganeti utility module.
23

24
This module holds functions that can be used in both daemons (all) and
25
the command line scripts.
26

27
"""
28

    
29

    
30
import os
31
import time
32
import subprocess
33
import re
34
import socket
35
import tempfile
36
import shutil
37
import errno
38
import pwd
39
import itertools
40
import select
41
import fcntl
42
import resource
43
import logging
44
import signal
45

    
46
from cStringIO import StringIO
47

    
48
try:
49
  from hashlib import sha1
50
except ImportError:
51
  import sha
52
  sha1 = sha.new
53

    
54
from ganeti import errors
55
from ganeti import constants
56

    
57

    
58
_locksheld = []
59
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
60

    
61
debug_locks = False
62

    
63
#: when set to True, L{RunCmd} is disabled
64
no_fork = False
65

    
66

    
67
class RunResult(object):
68
  """Holds the result of running external programs.
69

70
  @type exit_code: int
71
  @ivar exit_code: the exit code of the program, or None (if the program
72
      didn't exit())
73
  @type signal: int or None
74
  @ivar signal: the signal that caused the program to finish, or None
75
      (if the program wasn't terminated by a signal)
76
  @type stdout: str
77
  @ivar stdout: the standard output of the program
78
  @type stderr: str
79
  @ivar stderr: the standard error of the program
80
  @type failed: boolean
81
  @ivar failed: True in case the program was
82
      terminated by a signal or exited with a non-zero exit code
83
  @ivar fail_reason: a string detailing the termination reason
84

85
  """
86
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
87
               "failed", "fail_reason", "cmd"]
88

    
89

    
90
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
91
    self.cmd = cmd
92
    self.exit_code = exit_code
93
    self.signal = signal_
94
    self.stdout = stdout
95
    self.stderr = stderr
96
    self.failed = (signal_ is not None or exit_code != 0)
97

    
98
    if self.signal is not None:
99
      self.fail_reason = "terminated by signal %s" % self.signal
100
    elif self.exit_code is not None:
101
      self.fail_reason = "exited with exit code %s" % self.exit_code
102
    else:
103
      self.fail_reason = "unable to determine termination reason"
104

    
105
    if self.failed:
106
      logging.debug("Command '%s' failed (%s); output: %s",
107
                    self.cmd, self.fail_reason, self.output)
108

    
109
  def _GetOutput(self):
110
    """Returns the combined stdout and stderr for easier usage.
111

112
    """
113
    return self.stdout + self.stderr
114

    
115
  output = property(_GetOutput, None, None, "Return full output")
116

    
117

    
118
def RunCmd(cmd, env=None, output=None, cwd='/'):
119
  """Execute a (shell) command.
120

121
  The command should not read from its standard input, as it will be
122
  closed.
123

124
  @type  cmd: string or list
125
  @param cmd: Command to run
126
  @type env: dict
127
  @param env: Additional environment
128
  @type output: str
129
  @param output: if desired, the output of the command can be
130
      saved in a file instead of the RunResult instance; this
131
      parameter denotes the file name (if not None)
132
  @type cwd: string
133
  @param cwd: if specified, will be used as the working
134
      directory for the command; the default will be /
135
  @rtype: L{RunResult}
136
  @return: RunResult instance
137
  @raise errors.ProgrammerError: if we call this when forks are disabled
138

139
  """
140
  if no_fork:
141
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
142

    
143
  if isinstance(cmd, list):
144
    cmd = [str(val) for val in cmd]
145
    strcmd = " ".join(cmd)
146
    shell = False
147
  else:
148
    strcmd = cmd
149
    shell = True
150
  logging.debug("RunCmd '%s'", strcmd)
151

    
152
  cmd_env = os.environ.copy()
153
  cmd_env["LC_ALL"] = "C"
154
  if env is not None:
155
    cmd_env.update(env)
156

    
157
  try:
158
    if output is None:
159
      out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd)
160
    else:
161
      status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
162
      out = err = ""
163
  except OSError, err:
164
    if err.errno == errno.ENOENT:
165
      raise errors.OpExecError("Can't execute '%s': not found (%s)" %
166
                               (strcmd, err))
167
    else:
168
      raise
169

    
170
  if status >= 0:
171
    exitcode = status
172
    signal_ = None
173
  else:
174
    exitcode = None
175
    signal_ = -status
176

    
177
  return RunResult(exitcode, signal_, out, err, strcmd)
178

    
179

    
180
def _RunCmdPipe(cmd, env, via_shell, cwd):
181
  """Run a command and return its output.
182

183
  @type  cmd: string or list
184
  @param cmd: Command to run
185
  @type env: dict
186
  @param env: The environment to use
187
  @type via_shell: bool
188
  @param via_shell: if we should run via the shell
189
  @type cwd: string
190
  @param cwd: the working directory for the program
191
  @rtype: tuple
192
  @return: (out, err, status)
193

194
  """
195
  poller = select.poll()
196
  child = subprocess.Popen(cmd, shell=via_shell,
197
                           stderr=subprocess.PIPE,
198
                           stdout=subprocess.PIPE,
199
                           stdin=subprocess.PIPE,
200
                           close_fds=True, env=env,
201
                           cwd=cwd)
202

    
203
  child.stdin.close()
204
  poller.register(child.stdout, select.POLLIN)
205
  poller.register(child.stderr, select.POLLIN)
206
  out = StringIO()
207
  err = StringIO()
208
  fdmap = {
209
    child.stdout.fileno(): (out, child.stdout),
210
    child.stderr.fileno(): (err, child.stderr),
211
    }
212
  for fd in fdmap:
213
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
214
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
215

    
216
  while fdmap:
217
    try:
218
      pollresult = poller.poll()
219
    except EnvironmentError, eerr:
220
      if eerr.errno == errno.EINTR:
221
        continue
222
      raise
223
    except select.error, serr:
224
      if serr[0] == errno.EINTR:
225
        continue
226
      raise
227

    
228
    for fd, event in pollresult:
229
      if event & select.POLLIN or event & select.POLLPRI:
230
        data = fdmap[fd][1].read()
231
        # no data from read signifies EOF (the same as POLLHUP)
232
        if not data:
233
          poller.unregister(fd)
234
          del fdmap[fd]
235
          continue
236
        fdmap[fd][0].write(data)
237
      if (event & select.POLLNVAL or event & select.POLLHUP or
238
          event & select.POLLERR):
239
        poller.unregister(fd)
240
        del fdmap[fd]
241

    
242
  out = out.getvalue()
243
  err = err.getvalue()
244

    
245
  status = child.wait()
246
  return out, err, status
247

    
248

    
249
def _RunCmdFile(cmd, env, via_shell, output, cwd):
250
  """Run a command and save its output to a file.
251

252
  @type  cmd: string or list
253
  @param cmd: Command to run
254
  @type env: dict
255
  @param env: The environment to use
256
  @type via_shell: bool
257
  @param via_shell: if we should run via the shell
258
  @type output: str
259
  @param output: the filename in which to save the output
260
  @type cwd: string
261
  @param cwd: the working directory for the program
262
  @rtype: int
263
  @return: the exit status
264

265
  """
266
  fh = open(output, "a")
267
  try:
268
    child = subprocess.Popen(cmd, shell=via_shell,
269
                             stderr=subprocess.STDOUT,
270
                             stdout=fh,
271
                             stdin=subprocess.PIPE,
272
                             close_fds=True, env=env,
273
                             cwd=cwd)
274

    
275
    child.stdin.close()
276
    status = child.wait()
277
  finally:
278
    fh.close()
279
  return status
280

    
281

    
282
def RemoveFile(filename):
283
  """Remove a file ignoring some errors.
284

285
  Remove a file, ignoring non-existing ones or directories. Other
286
  errors are passed.
287

288
  @type filename: str
289
  @param filename: the file to be removed
290

291
  """
292
  try:
293
    os.unlink(filename)
294
  except OSError, err:
295
    if err.errno not in (errno.ENOENT, errno.EISDIR):
296
      raise
297

    
298

    
299
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
300
  """Renames a file.
301

302
  @type old: string
303
  @param old: Original path
304
  @type new: string
305
  @param new: New path
306
  @type mkdir: bool
307
  @param mkdir: Whether to create target directory if it doesn't exist
308
  @type mkdir_mode: int
309
  @param mkdir_mode: Mode for newly created directories
310

311
  """
312
  try:
313
    return os.rename(old, new)
314
  except OSError, err:
315
    # In at least one use case of this function, the job queue, directory
316
    # creation is very rare. Checking for the directory before renaming is not
317
    # as efficient.
318
    if mkdir and err.errno == errno.ENOENT:
319
      # Create directory and try again
320
      os.makedirs(os.path.dirname(new), mkdir_mode)
321
      return os.rename(old, new)
322
    raise
323

    
324

    
325
def _FingerprintFile(filename):
326
  """Compute the fingerprint of a file.
327

328
  If the file does not exist, a None will be returned
329
  instead.
330

331
  @type filename: str
332
  @param filename: the filename to checksum
333
  @rtype: str
334
  @return: the hex digest of the sha checksum of the contents
335
      of the file
336

337
  """
338
  if not (os.path.exists(filename) and os.path.isfile(filename)):
339
    return None
340

    
341
  f = open(filename)
342

    
343
  fp = sha1()
344
  while True:
345
    data = f.read(4096)
346
    if not data:
347
      break
348

    
349
    fp.update(data)
350

    
351
  return fp.hexdigest()
352

    
353

    
354
def FingerprintFiles(files):
355
  """Compute fingerprints for a list of files.
356

357
  @type files: list
358
  @param files: the list of filename to fingerprint
359
  @rtype: dict
360
  @return: a dictionary filename: fingerprint, holding only
361
      existing files
362

363
  """
364
  ret = {}
365

    
366
  for filename in files:
367
    cksum = _FingerprintFile(filename)
368
    if cksum:
369
      ret[filename] = cksum
370

    
371
  return ret
372

    
373

    
374
def ForceDictType(target, key_types, allowed_values=None):
375
  """Force the values of a dict to have certain types.
376

377
  @type target: dict
378
  @param target: the dict to update
379
  @type key_types: dict
380
  @param key_types: dict mapping target dict keys to types
381
                    in constants.ENFORCEABLE_TYPES
382
  @type allowed_values: list
383
  @keyword allowed_values: list of specially allowed values
384

385
  """
386
  if allowed_values is None:
387
    allowed_values = []
388

    
389
  if not isinstance(target, dict):
390
    msg = "Expected dictionary, got '%s'" % target
391
    raise errors.TypeEnforcementError(msg)
392

    
393
  for key in target:
394
    if key not in key_types:
395
      msg = "Unknown key '%s'" % key
396
      raise errors.TypeEnforcementError(msg)
397

    
398
    if target[key] in allowed_values:
399
      continue
400

    
401
    ktype = key_types[key]
402
    if ktype not in constants.ENFORCEABLE_TYPES:
403
      msg = "'%s' has non-enforceable type %s" % (key, ktype)
404
      raise errors.ProgrammerError(msg)
405

    
406
    if ktype == constants.VTYPE_STRING:
407
      if not isinstance(target[key], basestring):
408
        if isinstance(target[key], bool) and not target[key]:
409
          target[key] = ''
410
        else:
411
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
412
          raise errors.TypeEnforcementError(msg)
413
    elif ktype == constants.VTYPE_BOOL:
414
      if isinstance(target[key], basestring) and target[key]:
415
        if target[key].lower() == constants.VALUE_FALSE:
416
          target[key] = False
417
        elif target[key].lower() == constants.VALUE_TRUE:
418
          target[key] = True
419
        else:
420
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
421
          raise errors.TypeEnforcementError(msg)
422
      elif target[key]:
423
        target[key] = True
424
      else:
425
        target[key] = False
426
    elif ktype == constants.VTYPE_SIZE:
427
      try:
428
        target[key] = ParseUnit(target[key])
429
      except errors.UnitParseError, err:
430
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
431
              (key, target[key], err)
432
        raise errors.TypeEnforcementError(msg)
433
    elif ktype == constants.VTYPE_INT:
434
      try:
435
        target[key] = int(target[key])
436
      except (ValueError, TypeError):
437
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
438
        raise errors.TypeEnforcementError(msg)
439

    
440

    
441
def IsProcessAlive(pid):
442
  """Check if a given pid exists on the system.
443

444
  @note: zombie status is not handled, so zombie processes
445
      will be returned as alive
446
  @type pid: int
447
  @param pid: the process ID to check
448
  @rtype: boolean
449
  @return: True if the process exists
450

451
  """
452
  if pid <= 0:
453
    return False
454

    
455
  try:
456
    os.stat("/proc/%d/status" % pid)
457
    return True
458
  except EnvironmentError, err:
459
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
460
      return False
461
    raise
462

    
463

    
464
def ReadPidFile(pidfile):
465
  """Read a pid from a file.
466

467
  @type  pidfile: string
468
  @param pidfile: path to the file containing the pid
469
  @rtype: int
470
  @return: The process id, if the file exists and contains a valid PID,
471
           otherwise 0
472

473
  """
474
  try:
475
    pf = open(pidfile, 'r')
476
  except EnvironmentError, err:
477
    if err.errno != errno.ENOENT:
478
      logging.exception("Can't read pid file?!")
479
    return 0
480

    
481
  try:
482
    pid = int(pf.read())
483
  except ValueError, err:
484
    logging.info("Can't parse pid file contents", exc_info=True)
485
    return 0
486

    
487
  return pid
488

    
489

    
490
def MatchNameComponent(key, name_list):
491
  """Try to match a name against a list.
492

493
  This function will try to match a name like test1 against a list
494
  like C{['test1.example.com', 'test2.example.com', ...]}. Against
495
  this list, I{'test1'} as well as I{'test1.example'} will match, but
496
  not I{'test1.ex'}. A multiple match will be considered as no match
497
  at all (e.g. I{'test1'} against C{['test1.example.com',
498
  'test1.example.org']}).
499

500
  @type key: str
501
  @param key: the name to be searched
502
  @type name_list: list
503
  @param name_list: the list of strings against which to search the key
504

505
  @rtype: None or str
506
  @return: None if there is no match I{or} if there are multiple matches,
507
      otherwise the element from the list which matches
508

509
  """
510
  mo = re.compile("^%s(\..*)?$" % re.escape(key))
511
  names_filtered = [name for name in name_list if mo.match(name) is not None]
512
  if len(names_filtered) != 1:
513
    return None
514
  return names_filtered[0]
515

    
516

    
517
class HostInfo:
518
  """Class implementing resolver and hostname functionality
519

520
  """
521
  def __init__(self, name=None):
522
    """Initialize the host name object.
523

524
    If the name argument is not passed, it will use this system's
525
    name.
526

527
    """
528
    if name is None:
529
      name = self.SysName()
530

    
531
    self.query = name
532
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
533
    self.ip = self.ipaddrs[0]
534

    
535
  def ShortName(self):
536
    """Returns the hostname without domain.
537

538
    """
539
    return self.name.split('.')[0]
540

    
541
  @staticmethod
542
  def SysName():
543
    """Return the current system's name.
544

545
    This is simply a wrapper over C{socket.gethostname()}.
546

547
    """
548
    return socket.gethostname()
549

    
550
  @staticmethod
551
  def LookupHostname(hostname):
552
    """Look up hostname
553

554
    @type hostname: str
555
    @param hostname: hostname to look up
556

557
    @rtype: tuple
558
    @return: a tuple (name, aliases, ipaddrs) as returned by
559
        C{socket.gethostbyname_ex}
560
    @raise errors.ResolverError: in case of errors in resolving
561

562
    """
563
    try:
564
      result = socket.gethostbyname_ex(hostname)
565
    except socket.gaierror, err:
566
      # hostname not found in DNS
567
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
568

    
569
    return result
570

    
571

    
572
def ListVolumeGroups():
573
  """List volume groups and their size
574

575
  @rtype: dict
576
  @return:
577
       Dictionary with keys volume name and values
578
       the size of the volume
579

580
  """
581
  command = "vgs --noheadings --units m --nosuffix -o name,size"
582
  result = RunCmd(command)
583
  retval = {}
584
  if result.failed:
585
    return retval
586

    
587
  for line in result.stdout.splitlines():
588
    try:
589
      name, size = line.split()
590
      size = int(float(size))
591
    except (IndexError, ValueError), err:
592
      logging.error("Invalid output from vgs (%s): %s", err, line)
593
      continue
594

    
595
    retval[name] = size
596

    
597
  return retval
598

    
599

    
600
def BridgeExists(bridge):
601
  """Check whether the given bridge exists in the system
602

603
  @type bridge: str
604
  @param bridge: the bridge name to check
605
  @rtype: boolean
606
  @return: True if it does
607

608
  """
609
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
610

    
611

    
612
def NiceSort(name_list):
613
  """Sort a list of strings based on digit and non-digit groupings.
614

615
  Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
616
  will sort the list in the logical order C{['a1', 'a2', 'a10',
617
  'a11']}.
618

619
  The sort algorithm breaks each name in groups of either only-digits
620
  or no-digits. Only the first eight such groups are considered, and
621
  after that we just use what's left of the string.
622

623
  @type name_list: list
624
  @param name_list: the names to be sorted
625
  @rtype: list
626
  @return: a copy of the name list sorted with our algorithm
627

628
  """
629
  _SORTER_BASE = "(\D+|\d+)"
630
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
631
                                                  _SORTER_BASE, _SORTER_BASE,
632
                                                  _SORTER_BASE, _SORTER_BASE,
633
                                                  _SORTER_BASE, _SORTER_BASE)
634
  _SORTER_RE = re.compile(_SORTER_FULL)
635
  _SORTER_NODIGIT = re.compile("^\D*$")
636
  def _TryInt(val):
637
    """Attempts to convert a variable to integer."""
638
    if val is None or _SORTER_NODIGIT.match(val):
639
      return val
640
    rval = int(val)
641
    return rval
642

    
643
  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
644
             for name in name_list]
645
  to_sort.sort()
646
  return [tup[1] for tup in to_sort]
647

    
648

    
649
def TryConvert(fn, val):
650
  """Try to convert a value ignoring errors.
651

652
  This function tries to apply function I{fn} to I{val}. If no
653
  C{ValueError} or C{TypeError} exceptions are raised, it will return
654
  the result, else it will return the original value. Any other
655
  exceptions are propagated to the caller.
656

657
  @type fn: callable
658
  @param fn: function to apply to the value
659
  @param val: the value to be converted
660
  @return: The converted value if the conversion was successful,
661
      otherwise the original value.
662

663
  """
664
  try:
665
    nv = fn(val)
666
  except (ValueError, TypeError):
667
    nv = val
668
  return nv
669

    
670

    
671
def IsValidIP(ip):
672
  """Verifies the syntax of an IPv4 address.
673

674
  This function checks if the IPv4 address passes is valid or not based
675
  on syntax (not IP range, class calculations, etc.).
676

677
  @type ip: str
678
  @param ip: the address to be checked
679
  @rtype: a regular expression match object
680
  @return: a regular expression match object, or None if the
681
      address is not valid
682

683
  """
684
  unit = "(0|[1-9]\d{0,2})"
685
  #TODO: convert and return only boolean
686
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
687

    
688

    
689
def IsValidShellParam(word):
690
  """Verifies is the given word is safe from the shell's p.o.v.
691

692
  This means that we can pass this to a command via the shell and be
693
  sure that it doesn't alter the command line and is passed as such to
694
  the actual command.
695

696
  Note that we are overly restrictive here, in order to be on the safe
697
  side.
698

699
  @type word: str
700
  @param word: the word to check
701
  @rtype: boolean
702
  @return: True if the word is 'safe'
703

704
  """
705
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
706

    
707

    
708
def BuildShellCmd(template, *args):
709
  """Build a safe shell command line from the given arguments.
710

711
  This function will check all arguments in the args list so that they
712
  are valid shell parameters (i.e. they don't contain shell
713
  metacharacters). If everything is ok, it will return the result of
714
  template % args.
715

716
  @type template: str
717
  @param template: the string holding the template for the
718
      string formatting
719
  @rtype: str
720
  @return: the expanded command line
721

722
  """
723
  for word in args:
724
    if not IsValidShellParam(word):
725
      raise errors.ProgrammerError("Shell argument '%s' contains"
726
                                   " invalid characters" % word)
727
  return template % args
728

    
729

    
730
def FormatUnit(value, units):
731
  """Formats an incoming number of MiB with the appropriate unit.
732

733
  @type value: int
734
  @param value: integer representing the value in MiB (1048576)
735
  @type units: char
736
  @param units: the type of formatting we should do:
737
      - 'h' for automatic scaling
738
      - 'm' for MiBs
739
      - 'g' for GiBs
740
      - 't' for TiBs
741
  @rtype: str
742
  @return: the formatted value (with suffix)
743

744
  """
745
  if units not in ('m', 'g', 't', 'h'):
746
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
747

    
748
  suffix = ''
749

    
750
  if units == 'm' or (units == 'h' and value < 1024):
751
    if units == 'h':
752
      suffix = 'M'
753
    return "%d%s" % (round(value, 0), suffix)
754

    
755
  elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
756
    if units == 'h':
757
      suffix = 'G'
758
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
759

    
760
  else:
761
    if units == 'h':
762
      suffix = 'T'
763
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
764

    
765

    
766
def ParseUnit(input_string):
767
  """Tries to extract number and scale from the given string.
768

769
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
770
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
771
  is always an int in MiB.
772

773
  """
774
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
775
  if not m:
776
    raise errors.UnitParseError("Invalid format")
777

    
778
  value = float(m.groups()[0])
779

    
780
  unit = m.groups()[1]
781
  if unit:
782
    lcunit = unit.lower()
783
  else:
784
    lcunit = 'm'
785

    
786
  if lcunit in ('m', 'mb', 'mib'):
787
    # Value already in MiB
788
    pass
789

    
790
  elif lcunit in ('g', 'gb', 'gib'):
791
    value *= 1024
792

    
793
  elif lcunit in ('t', 'tb', 'tib'):
794
    value *= 1024 * 1024
795

    
796
  else:
797
    raise errors.UnitParseError("Unknown unit: %s" % unit)
798

    
799
  # Make sure we round up
800
  if int(value) < value:
801
    value += 1
802

    
803
  # Round up to the next multiple of 4
804
  value = int(value)
805
  if value % 4:
806
    value += 4 - value % 4
807

    
808
  return value
809

    
810

    
811
def AddAuthorizedKey(file_name, key):
812
  """Adds an SSH public key to an authorized_keys file.
813

814
  @type file_name: str
815
  @param file_name: path to authorized_keys file
816
  @type key: str
817
  @param key: string containing key
818

819
  """
820
  key_fields = key.split()
821

    
822
  f = open(file_name, 'a+')
823
  try:
824
    nl = True
825
    for line in f:
826
      # Ignore whitespace changes
827
      if line.split() == key_fields:
828
        break
829
      nl = line.endswith('\n')
830
    else:
831
      if not nl:
832
        f.write("\n")
833
      f.write(key.rstrip('\r\n'))
834
      f.write("\n")
835
      f.flush()
836
  finally:
837
    f.close()
838

    
839

    
840
def RemoveAuthorizedKey(file_name, key):
841
  """Removes an SSH public key from an authorized_keys file.
842

843
  @type file_name: str
844
  @param file_name: path to authorized_keys file
845
  @type key: str
846
  @param key: string containing key
847

848
  """
849
  key_fields = key.split()
850

    
851
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
852
  try:
853
    out = os.fdopen(fd, 'w')
854
    try:
855
      f = open(file_name, 'r')
856
      try:
857
        for line in f:
858
          # Ignore whitespace changes while comparing lines
859
          if line.split() != key_fields:
860
            out.write(line)
861

    
862
        out.flush()
863
        os.rename(tmpname, file_name)
864
      finally:
865
        f.close()
866
    finally:
867
      out.close()
868
  except:
869
    RemoveFile(tmpname)
870
    raise
871

    
872

    
873
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
874
  """Sets the name of an IP address and hostname in /etc/hosts.
875

876
  @type file_name: str
877
  @param file_name: path to the file to modify (usually C{/etc/hosts})
878
  @type ip: str
879
  @param ip: the IP address
880
  @type hostname: str
881
  @param hostname: the hostname to be added
882
  @type aliases: list
883
  @param aliases: the list of aliases to add for the hostname
884

885
  """
886
  # FIXME: use WriteFile + fn rather than duplicating its efforts
887
  # Ensure aliases are unique
888
  aliases = UniqueSequence([hostname] + aliases)[1:]
889

    
890
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
891
  try:
892
    out = os.fdopen(fd, 'w')
893
    try:
894
      f = open(file_name, 'r')
895
      try:
896
        for line in f:
897
          fields = line.split()
898
          if fields and not fields[0].startswith('#') and ip == fields[0]:
899
            continue
900
          out.write(line)
901

    
902
        out.write("%s\t%s" % (ip, hostname))
903
        if aliases:
904
          out.write(" %s" % ' '.join(aliases))
905
        out.write('\n')
906

    
907
        out.flush()
908
        os.fsync(out)
909
        os.chmod(tmpname, 0644)
910
        os.rename(tmpname, file_name)
911
      finally:
912
        f.close()
913
    finally:
914
      out.close()
915
  except:
916
    RemoveFile(tmpname)
917
    raise
918

    
919

    
920
def AddHostToEtcHosts(hostname):
921
  """Wrapper around SetEtcHostsEntry.
922

923
  @type hostname: str
924
  @param hostname: a hostname that will be resolved and added to
925
      L{constants.ETC_HOSTS}
926

927
  """
928
  hi = HostInfo(name=hostname)
929
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
930

    
931

    
932
def RemoveEtcHostsEntry(file_name, hostname):
933
  """Removes a hostname from /etc/hosts.
934

935
  IP addresses without names are removed from the file.
936

937
  @type file_name: str
938
  @param file_name: path to the file to modify (usually C{/etc/hosts})
939
  @type hostname: str
940
  @param hostname: the hostname to be removed
941

942
  """
943
  # FIXME: use WriteFile + fn rather than duplicating its efforts
944
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
945
  try:
946
    out = os.fdopen(fd, 'w')
947
    try:
948
      f = open(file_name, 'r')
949
      try:
950
        for line in f:
951
          fields = line.split()
952
          if len(fields) > 1 and not fields[0].startswith('#'):
953
            names = fields[1:]
954
            if hostname in names:
955
              while hostname in names:
956
                names.remove(hostname)
957
              if names:
958
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
959
              continue
960

    
961
          out.write(line)
962

    
963
        out.flush()
964
        os.fsync(out)
965
        os.chmod(tmpname, 0644)
966
        os.rename(tmpname, file_name)
967
      finally:
968
        f.close()
969
    finally:
970
      out.close()
971
  except:
972
    RemoveFile(tmpname)
973
    raise
974

    
975

    
976
def RemoveHostFromEtcHosts(hostname):
977
  """Wrapper around RemoveEtcHostsEntry.
978

979
  @type hostname: str
980
  @param hostname: hostname that will be resolved and its
981
      full and shot name will be removed from
982
      L{constants.ETC_HOSTS}
983

984
  """
985
  hi = HostInfo(name=hostname)
986
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
987
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
988

    
989

    
990
def CreateBackup(file_name):
991
  """Creates a backup of a file.
992

993
  @type file_name: str
994
  @param file_name: file to be backed up
995
  @rtype: str
996
  @return: the path to the newly created backup
997
  @raise errors.ProgrammerError: for invalid file names
998

999
  """
1000
  if not os.path.isfile(file_name):
1001
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1002
                                file_name)
1003

    
1004
  prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
1005
  dir_name = os.path.dirname(file_name)
1006

    
1007
  fsrc = open(file_name, 'rb')
1008
  try:
1009
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1010
    fdst = os.fdopen(fd, 'wb')
1011
    try:
1012
      shutil.copyfileobj(fsrc, fdst)
1013
    finally:
1014
      fdst.close()
1015
  finally:
1016
    fsrc.close()
1017

    
1018
  return backup_name
1019

    
1020

    
1021
def ShellQuote(value):
1022
  """Quotes shell argument according to POSIX.
1023

1024
  @type value: str
1025
  @param value: the argument to be quoted
1026
  @rtype: str
1027
  @return: the quoted value
1028

1029
  """
1030
  if _re_shell_unquoted.match(value):
1031
    return value
1032
  else:
1033
    return "'%s'" % value.replace("'", "'\\''")
1034

    
1035

    
1036
def ShellQuoteArgs(args):
1037
  """Quotes a list of shell arguments.
1038

1039
  @type args: list
1040
  @param args: list of arguments to be quoted
1041
  @rtype: str
1042
  @return: the quoted arguments concatenated with spaces
1043

1044
  """
1045
  return ' '.join([ShellQuote(i) for i in args])
1046

    
1047

    
1048
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
1049
  """Simple ping implementation using TCP connect(2).
1050

1051
  Check if the given IP is reachable by doing attempting a TCP connect
1052
  to it.
1053

1054
  @type target: str
1055
  @param target: the IP or hostname to ping
1056
  @type port: int
1057
  @param port: the port to connect to
1058
  @type timeout: int
1059
  @param timeout: the timeout on the connection attempt
1060
  @type live_port_needed: boolean
1061
  @param live_port_needed: whether a closed port will cause the
1062
      function to return failure, as if there was a timeout
1063
  @type source: str or None
1064
  @param source: if specified, will cause the connect to be made
1065
      from this specific source address; failures to bind other
1066
      than C{EADDRNOTAVAIL} will be ignored
1067

1068
  """
1069
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1070

    
1071
  success = False
1072

    
1073
  if source is not None:
1074
    try:
1075
      sock.bind((source, 0))
1076
    except socket.error, (errcode, _):
1077
      if errcode == errno.EADDRNOTAVAIL:
1078
        success = False
1079

    
1080
  sock.settimeout(timeout)
1081

    
1082
  try:
1083
    sock.connect((target, port))
1084
    sock.close()
1085
    success = True
1086
  except socket.timeout:
1087
    success = False
1088
  except socket.error, (errcode, errstring):
1089
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
1090

    
1091
  return success
1092

    
1093

    
1094
def OwnIpAddress(address):
1095
  """Check if the current host has the the given IP address.
1096

1097
  Currently this is done by TCP-pinging the address from the loopback
1098
  address.
1099

1100
  @type address: string
1101
  @param address: the address to check
1102
  @rtype: bool
1103
  @return: True if we own the address
1104

1105
  """
1106
  return TcpPing(address, constants.DEFAULT_NODED_PORT,
1107
                 source=constants.LOCALHOST_IP_ADDRESS)
1108

    
1109

    
1110
def ListVisibleFiles(path):
1111
  """Returns a list of visible files in a directory.
1112

1113
  @type path: str
1114
  @param path: the directory to enumerate
1115
  @rtype: list
1116
  @return: the list of all files not starting with a dot
1117

1118
  """
1119
  files = [i for i in os.listdir(path) if not i.startswith(".")]
1120
  files.sort()
1121
  return files
1122

    
1123

    
1124
def GetHomeDir(user, default=None):
1125
  """Try to get the homedir of the given user.
1126

1127
  The user can be passed either as a string (denoting the name) or as
1128
  an integer (denoting the user id). If the user is not found, the
1129
  'default' argument is returned, which defaults to None.
1130

1131
  """
1132
  try:
1133
    if isinstance(user, basestring):
1134
      result = pwd.getpwnam(user)
1135
    elif isinstance(user, (int, long)):
1136
      result = pwd.getpwuid(user)
1137
    else:
1138
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1139
                                   type(user))
1140
  except KeyError:
1141
    return default
1142
  return result.pw_dir
1143

    
1144

    
1145
def NewUUID():
1146
  """Returns a random UUID.
1147

1148
  @note: This is a Linux-specific method as it uses the /proc
1149
      filesystem.
1150
  @rtype: str
1151

1152
  """
1153
  f = open("/proc/sys/kernel/random/uuid", "r")
1154
  try:
1155
    return f.read(128).rstrip("\n")
1156
  finally:
1157
    f.close()
1158

    
1159

    
1160
def GenerateSecret():
1161
  """Generates a random secret.
1162

1163
  This will generate a pseudo-random secret, and return its sha digest
1164
  (so that it can be used where an ASCII string is needed).
1165

1166
  @rtype: str
1167
  @return: a sha1 hexdigest of a block of 64 random bytes
1168

1169
  """
1170
  return sha1(os.urandom(64)).hexdigest()
1171

    
1172

    
1173
def EnsureDirs(dirs):
1174
  """Make required directories, if they don't exist.
1175

1176
  @param dirs: list of tuples (dir_name, dir_mode)
1177
  @type dirs: list of (string, integer)
1178

1179
  """
1180
  for dir_name, dir_mode in dirs:
1181
    try:
1182
      os.mkdir(dir_name, dir_mode)
1183
    except EnvironmentError, err:
1184
      if err.errno != errno.EEXIST:
1185
        raise errors.GenericError("Cannot create needed directory"
1186
                                  " '%s': %s" % (dir_name, err))
1187
    if not os.path.isdir(dir_name):
1188
      raise errors.GenericError("%s is not a directory" % dir_name)
1189

    
1190

    
1191
def ReadFile(file_name, size=None):
1192
  """Reads a file.
1193

1194
  @type size: None or int
1195
  @param size: Read at most size bytes
1196
  @rtype: str
1197
  @return: the (possibly partial) content of the file
1198

1199
  """
1200
  f = open(file_name, "r")
1201
  try:
1202
    if size is None:
1203
      return f.read()
1204
    else:
1205
      return f.read(size)
1206
  finally:
1207
    f.close()
1208

    
1209

    
1210
def WriteFile(file_name, fn=None, data=None,
1211
              mode=None, uid=-1, gid=-1,
1212
              atime=None, mtime=None, close=True,
1213
              dry_run=False, backup=False,
1214
              prewrite=None, postwrite=None):
1215
  """(Over)write a file atomically.
1216

1217
  The file_name and either fn (a function taking one argument, the
1218
  file descriptor, and which should write the data to it) or data (the
1219
  contents of the file) must be passed. The other arguments are
1220
  optional and allow setting the file mode, owner and group, and the
1221
  mtime/atime of the file.
1222

1223
  If the function doesn't raise an exception, it has succeeded and the
1224
  target file has the new contents. If the function has raised an
1225
  exception, an existing target file should be unmodified and the
1226
  temporary file should be removed.
1227

1228
  @type file_name: str
1229
  @param file_name: the target filename
1230
  @type fn: callable
1231
  @param fn: content writing function, called with
1232
      file descriptor as parameter
1233
  @type data: str
1234
  @param data: contents of the file
1235
  @type mode: int
1236
  @param mode: file mode
1237
  @type uid: int
1238
  @param uid: the owner of the file
1239
  @type gid: int
1240
  @param gid: the group of the file
1241
  @type atime: int
1242
  @param atime: a custom access time to be set on the file
1243
  @type mtime: int
1244
  @param mtime: a custom modification time to be set on the file
1245
  @type close: boolean
1246
  @param close: whether to close file after writing it
1247
  @type prewrite: callable
1248
  @param prewrite: function to be called before writing content
1249
  @type postwrite: callable
1250
  @param postwrite: function to be called after writing content
1251

1252
  @rtype: None or int
1253
  @return: None if the 'close' parameter evaluates to True,
1254
      otherwise the file descriptor
1255

1256
  @raise errors.ProgrammerError: if any of the arguments are not valid
1257

1258
  """
1259
  if not os.path.isabs(file_name):
1260
    raise errors.ProgrammerError("Path passed to WriteFile is not"
1261
                                 " absolute: '%s'" % file_name)
1262

    
1263
  if [fn, data].count(None) != 1:
1264
    raise errors.ProgrammerError("fn or data required")
1265

    
1266
  if [atime, mtime].count(None) == 1:
1267
    raise errors.ProgrammerError("Both atime and mtime must be either"
1268
                                 " set or None")
1269

    
1270
  if backup and not dry_run and os.path.isfile(file_name):
1271
    CreateBackup(file_name)
1272

    
1273
  dir_name, base_name = os.path.split(file_name)
1274
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1275
  do_remove = True
1276
  # here we need to make sure we remove the temp file, if any error
1277
  # leaves it in place
1278
  try:
1279
    if uid != -1 or gid != -1:
1280
      os.chown(new_name, uid, gid)
1281
    if mode:
1282
      os.chmod(new_name, mode)
1283
    if callable(prewrite):
1284
      prewrite(fd)
1285
    if data is not None:
1286
      os.write(fd, data)
1287
    else:
1288
      fn(fd)
1289
    if callable(postwrite):
1290
      postwrite(fd)
1291
    os.fsync(fd)
1292
    if atime is not None and mtime is not None:
1293
      os.utime(new_name, (atime, mtime))
1294
    if not dry_run:
1295
      os.rename(new_name, file_name)
1296
      do_remove = False
1297
  finally:
1298
    if close:
1299
      os.close(fd)
1300
      result = None
1301
    else:
1302
      result = fd
1303
    if do_remove:
1304
      RemoveFile(new_name)
1305

    
1306
  return result
1307

    
1308

    
1309
def FirstFree(seq, base=0):
1310
  """Returns the first non-existing integer from seq.
1311

1312
  The seq argument should be a sorted list of positive integers. The
1313
  first time the index of an element is smaller than the element
1314
  value, the index will be returned.
1315

1316
  The base argument is used to start at a different offset,
1317
  i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1318

1319
  Example: C{[0, 1, 3]} will return I{2}.
1320

1321
  @type seq: sequence
1322
  @param seq: the sequence to be analyzed.
1323
  @type base: int
1324
  @param base: use this value as the base index of the sequence
1325
  @rtype: int
1326
  @return: the first non-used index in the sequence
1327

1328
  """
1329
  for idx, elem in enumerate(seq):
1330
    assert elem >= base, "Passed element is higher than base offset"
1331
    if elem > idx + base:
1332
      # idx is not used
1333
      return idx + base
1334
  return None
1335

    
1336

    
1337
def all(seq, pred=bool):
1338
  "Returns True if pred(x) is True for every element in the iterable"
1339
  for _ in itertools.ifilterfalse(pred, seq):
1340
    return False
1341
  return True
1342

    
1343

    
1344
def any(seq, pred=bool):
1345
  "Returns True if pred(x) is True for at least one element in the iterable"
1346
  for _ in itertools.ifilter(pred, seq):
1347
    return True
1348
  return False
1349

    
1350

    
1351
def UniqueSequence(seq):
1352
  """Returns a list with unique elements.
1353

1354
  Element order is preserved.
1355

1356
  @type seq: sequence
1357
  @param seq: the sequence with the source elements
1358
  @rtype: list
1359
  @return: list of unique elements from seq
1360

1361
  """
1362
  seen = set()
1363
  return [i for i in seq if i not in seen and not seen.add(i)]
1364

    
1365

    
1366
def IsValidMac(mac):
1367
  """Predicate to check if a MAC address is valid.
1368

1369
  Checks whether the supplied MAC address is formally correct, only
1370
  accepts colon separated format.
1371

1372
  @type mac: str
1373
  @param mac: the MAC to be validated
1374
  @rtype: boolean
1375
  @return: True is the MAC seems valid
1376

1377
  """
1378
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
1379
  return mac_check.match(mac) is not None
1380

    
1381

    
1382
def TestDelay(duration):
1383
  """Sleep for a fixed amount of time.
1384

1385
  @type duration: float
1386
  @param duration: the sleep duration
1387
  @rtype: boolean
1388
  @return: False for negative value, True otherwise
1389

1390
  """
1391
  if duration < 0:
1392
    return False, "Invalid sleep duration"
1393
  time.sleep(duration)
1394
  return True, None
1395

    
1396

    
1397
def _CloseFDNoErr(fd, retries=5):
1398
  """Close a file descriptor ignoring errors.
1399

1400
  @type fd: int
1401
  @param fd: the file descriptor
1402
  @type retries: int
1403
  @param retries: how many retries to make, in case we get any
1404
      other error than EBADF
1405

1406
  """
1407
  try:
1408
    os.close(fd)
1409
  except OSError, err:
1410
    if err.errno != errno.EBADF:
1411
      if retries > 0:
1412
        _CloseFDNoErr(fd, retries - 1)
1413
    # else either it's closed already or we're out of retries, so we
1414
    # ignore this and go on
1415

    
1416

    
1417
def CloseFDs(noclose_fds=None):
1418
  """Close file descriptors.
1419

1420
  This closes all file descriptors above 2 (i.e. except
1421
  stdin/out/err).
1422

1423
  @type noclose_fds: list or None
1424
  @param noclose_fds: if given, it denotes a list of file descriptor
1425
      that should not be closed
1426

1427
  """
1428
  # Default maximum for the number of available file descriptors.
1429
  if 'SC_OPEN_MAX' in os.sysconf_names:
1430
    try:
1431
      MAXFD = os.sysconf('SC_OPEN_MAX')
1432
      if MAXFD < 0:
1433
        MAXFD = 1024
1434
    except OSError:
1435
      MAXFD = 1024
1436
  else:
1437
    MAXFD = 1024
1438
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1439
  if (maxfd == resource.RLIM_INFINITY):
1440
    maxfd = MAXFD
1441

    
1442
  # Iterate through and close all file descriptors (except the standard ones)
1443
  for fd in range(3, maxfd):
1444
    if noclose_fds and fd in noclose_fds:
1445
      continue
1446
    _CloseFDNoErr(fd)
1447

    
1448

    
1449
def Daemonize(logfile):
1450
  """Daemonize the current process.
1451

1452
  This detaches the current process from the controlling terminal and
1453
  runs it in the background as a daemon.
1454

1455
  @type logfile: str
1456
  @param logfile: the logfile to which we should redirect stdout/stderr
1457
  @rtype: int
1458
  @return: the value zero
1459

1460
  """
1461
  UMASK = 077
1462
  WORKDIR = "/"
1463

    
1464
  # this might fail
1465
  pid = os.fork()
1466
  if (pid == 0):  # The first child.
1467
    os.setsid()
1468
    # this might fail
1469
    pid = os.fork() # Fork a second child.
1470
    if (pid == 0):  # The second child.
1471
      os.chdir(WORKDIR)
1472
      os.umask(UMASK)
1473
    else:
1474
      # exit() or _exit()?  See below.
1475
      os._exit(0) # Exit parent (the first child) of the second child.
1476
  else:
1477
    os._exit(0) # Exit parent of the first child.
1478

    
1479
  for fd in range(3):
1480
    _CloseFDNoErr(fd)
1481
  i = os.open("/dev/null", os.O_RDONLY) # stdin
1482
  assert i == 0, "Can't close/reopen stdin"
1483
  i = os.open(logfile, os.O_WRONLY|os.O_CREAT|os.O_APPEND, 0600) # stdout
1484
  assert i == 1, "Can't close/reopen stdout"
1485
  # Duplicate standard output to standard error.
1486
  os.dup2(1, 2)
1487
  return 0
1488

    
1489

    
1490
def DaemonPidFileName(name):
1491
  """Compute a ganeti pid file absolute path
1492

1493
  @type name: str
1494
  @param name: the daemon name
1495
  @rtype: str
1496
  @return: the full path to the pidfile corresponding to the given
1497
      daemon name
1498

1499
  """
1500
  return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1501

    
1502

    
1503
def WritePidFile(name):
1504
  """Write the current process pidfile.
1505

1506
  The file will be written to L{constants.RUN_GANETI_DIR}I{/name.pid}
1507

1508
  @type name: str
1509
  @param name: the daemon name to use
1510
  @raise errors.GenericError: if the pid file already exists and
1511
      points to a live process
1512

1513
  """
1514
  pid = os.getpid()
1515
  pidfilename = DaemonPidFileName(name)
1516
  if IsProcessAlive(ReadPidFile(pidfilename)):
1517
    raise errors.GenericError("%s contains a live process" % pidfilename)
1518

    
1519
  WriteFile(pidfilename, data="%d\n" % pid)
1520

    
1521

    
1522
def RemovePidFile(name):
1523
  """Remove the current process pidfile.
1524

1525
  Any errors are ignored.
1526

1527
  @type name: str
1528
  @param name: the daemon name used to derive the pidfile name
1529

1530
  """
1531
  pidfilename = DaemonPidFileName(name)
1532
  # TODO: we could check here that the file contains our pid
1533
  try:
1534
    RemoveFile(pidfilename)
1535
  except:
1536
    pass
1537

    
1538

    
1539
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30,
1540
                waitpid=False):
1541
  """Kill a process given by its pid.
1542

1543
  @type pid: int
1544
  @param pid: The PID to terminate.
1545
  @type signal_: int
1546
  @param signal_: The signal to send, by default SIGTERM
1547
  @type timeout: int
1548
  @param timeout: The timeout after which, if the process is still alive,
1549
                  a SIGKILL will be sent. If not positive, no such checking
1550
                  will be done
1551
  @type waitpid: boolean
1552
  @param waitpid: If true, we should waitpid on this process after
1553
      sending signals, since it's our own child and otherwise it
1554
      would remain as zombie
1555

1556
  """
1557
  def _helper(pid, signal_, wait):
1558
    """Simple helper to encapsulate the kill/waitpid sequence"""
1559
    os.kill(pid, signal_)
1560
    if wait:
1561
      try:
1562
        os.waitpid(pid, os.WNOHANG)
1563
      except OSError:
1564
        pass
1565

    
1566
  if pid <= 0:
1567
    # kill with pid=0 == suicide
1568
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1569

    
1570
  if not IsProcessAlive(pid):
1571
    return
1572
  _helper(pid, signal_, waitpid)
1573
  if timeout <= 0:
1574
    return
1575

    
1576
  # Wait up to $timeout seconds
1577
  end = time.time() + timeout
1578
  wait = 0.01
1579
  while time.time() < end and IsProcessAlive(pid):
1580
    try:
1581
      (result_pid, _) = os.waitpid(pid, os.WNOHANG)
1582
      if result_pid > 0:
1583
        break
1584
    except OSError:
1585
      pass
1586
    time.sleep(wait)
1587
    # Make wait time longer for next try
1588
    if wait < 0.1:
1589
      wait *= 1.5
1590

    
1591
  if IsProcessAlive(pid):
1592
    # Kill process if it's still alive
1593
    _helper(pid, signal.SIGKILL, waitpid)
1594

    
1595

    
1596
def FindFile(name, search_path, test=os.path.exists):
1597
  """Look for a filesystem object in a given path.
1598

1599
  This is an abstract method to search for filesystem object (files,
1600
  dirs) under a given search path.
1601

1602
  @type name: str
1603
  @param name: the name to look for
1604
  @type search_path: str
1605
  @param search_path: location to start at
1606
  @type test: callable
1607
  @param test: a function taking one argument that should return True
1608
      if the a given object is valid; the default value is
1609
      os.path.exists, causing only existing files to be returned
1610
  @rtype: str or None
1611
  @return: full path to the object if found, None otherwise
1612

1613
  """
1614
  for dir_name in search_path:
1615
    item_name = os.path.sep.join([dir_name, name])
1616
    if test(item_name):
1617
      return item_name
1618
  return None
1619

    
1620

    
1621
def CheckVolumeGroupSize(vglist, vgname, minsize):
1622
  """Checks if the volume group list is valid.
1623

1624
  The function will check if a given volume group is in the list of
1625
  volume groups and has a minimum size.
1626

1627
  @type vglist: dict
1628
  @param vglist: dictionary of volume group names and their size
1629
  @type vgname: str
1630
  @param vgname: the volume group we should check
1631
  @type minsize: int
1632
  @param minsize: the minimum size we accept
1633
  @rtype: None or str
1634
  @return: None for success, otherwise the error message
1635

1636
  """
1637
  vgsize = vglist.get(vgname, None)
1638
  if vgsize is None:
1639
    return "volume group '%s' missing" % vgname
1640
  elif vgsize < minsize:
1641
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1642
            (vgname, minsize, vgsize))
1643
  return None
1644

    
1645

    
1646
def SplitTime(value):
1647
  """Splits time as floating point number into a tuple.
1648

1649
  @param value: Time in seconds
1650
  @type value: int or float
1651
  @return: Tuple containing (seconds, microseconds)
1652

1653
  """
1654
  (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
1655

    
1656
  assert 0 <= seconds, \
1657
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1658
  assert 0 <= microseconds <= 999999, \
1659
    "Microseconds must be 0-999999, but are %s" % microseconds
1660

    
1661
  return (int(seconds), int(microseconds))
1662

    
1663

    
1664
def MergeTime(timetuple):
1665
  """Merges a tuple into time as a floating point number.
1666

1667
  @param timetuple: Time as tuple, (seconds, microseconds)
1668
  @type timetuple: tuple
1669
  @return: Time as a floating point number expressed in seconds
1670

1671
  """
1672
  (seconds, microseconds) = timetuple
1673

    
1674
  assert 0 <= seconds, \
1675
    "Seconds must be larger than or equal to 0, but are %s" % seconds
1676
  assert 0 <= microseconds <= 999999, \
1677
    "Microseconds must be 0-999999, but are %s" % microseconds
1678

    
1679
  return float(seconds) + (float(microseconds) * 0.000001)
1680

    
1681

    
1682
def GetDaemonPort(daemon_name):
1683
  """Get the daemon port for this cluster.
1684

1685
  Note that this routine does not read a ganeti-specific file, but
1686
  instead uses C{socket.getservbyname} to allow pre-customization of
1687
  this parameter outside of Ganeti.
1688

1689
  @type daemon_name: string
1690
  @param daemon_name: daemon name (in constants.DAEMONS_PORTS)
1691
  @rtype: int
1692

1693
  """
1694
  if daemon_name not in constants.DAEMONS_PORTS:
1695
    raise errors.ProgrammerError("Unknown daemon: %s" % daemon_name)
1696

    
1697
  (proto, default_port) = constants.DAEMONS_PORTS[daemon_name]
1698
  try:
1699
    port = socket.getservbyname(daemon_name, proto)
1700
  except socket.error:
1701
    port = default_port
1702

    
1703
  return port
1704

    
1705

    
1706
def SetupLogging(logfile, debug=False, stderr_logging=False, program="",
1707
                 multithreaded=False):
1708
  """Configures the logging module.
1709

1710
  @type logfile: str
1711
  @param logfile: the filename to which we should log
1712
  @type debug: boolean
1713
  @param debug: whether to enable debug messages too or
1714
      only those at C{INFO} and above level
1715
  @type stderr_logging: boolean
1716
  @param stderr_logging: whether we should also log to the standard error
1717
  @type program: str
1718
  @param program: the name under which we should log messages
1719
  @type multithreaded: boolean
1720
  @param multithreaded: if True, will add the thread name to the log file
1721
  @raise EnvironmentError: if we can't open the log file and
1722
      stderr logging is disabled
1723

1724
  """
1725
  fmt = "%(asctime)s: " + program + " pid=%(process)d"
1726
  if multithreaded:
1727
    fmt += "/%(threadName)s"
1728
  if debug:
1729
    fmt += " %(module)s:%(lineno)s"
1730
  fmt += " %(levelname)s %(message)s"
1731
  formatter = logging.Formatter(fmt)
1732

    
1733
  root_logger = logging.getLogger("")
1734
  root_logger.setLevel(logging.NOTSET)
1735

    
1736
  # Remove all previously setup handlers
1737
  for handler in root_logger.handlers:
1738
    handler.close()
1739
    root_logger.removeHandler(handler)
1740

    
1741
  if stderr_logging:
1742
    stderr_handler = logging.StreamHandler()
1743
    stderr_handler.setFormatter(formatter)
1744
    if debug:
1745
      stderr_handler.setLevel(logging.NOTSET)
1746
    else:
1747
      stderr_handler.setLevel(logging.CRITICAL)
1748
    root_logger.addHandler(stderr_handler)
1749

    
1750
  # this can fail, if the logging directories are not setup or we have
1751
  # a permisssion problem; in this case, it's best to log but ignore
1752
  # the error if stderr_logging is True, and if false we re-raise the
1753
  # exception since otherwise we could run but without any logs at all
1754
  try:
1755
    logfile_handler = logging.FileHandler(logfile)
1756
    logfile_handler.setFormatter(formatter)
1757
    if debug:
1758
      logfile_handler.setLevel(logging.DEBUG)
1759
    else:
1760
      logfile_handler.setLevel(logging.INFO)
1761
    root_logger.addHandler(logfile_handler)
1762
  except EnvironmentError:
1763
    if stderr_logging:
1764
      logging.exception("Failed to enable logging to file '%s'", logfile)
1765
    else:
1766
      # we need to re-raise the exception
1767
      raise
1768

    
1769
def IsNormAbsPath(path):
1770
  """Check whether a path is absolute and also normalized
1771

1772
  This avoids things like /dir/../../other/path to be valid.
1773

1774
  """
1775
  return os.path.normpath(path) == path and os.path.isabs(path)
1776

    
1777
def TailFile(fname, lines=20):
1778
  """Return the last lines from a file.
1779

1780
  @note: this function will only read and parse the last 4KB of
1781
      the file; if the lines are very long, it could be that less
1782
      than the requested number of lines are returned
1783

1784
  @param fname: the file name
1785
  @type lines: int
1786
  @param lines: the (maximum) number of lines to return
1787

1788
  """
1789
  fd = open(fname, "r")
1790
  try:
1791
    fd.seek(0, 2)
1792
    pos = fd.tell()
1793
    pos = max(0, pos-4096)
1794
    fd.seek(pos, 0)
1795
    raw_data = fd.read()
1796
  finally:
1797
    fd.close()
1798

    
1799
  rows = raw_data.splitlines()
1800
  return rows[-lines:]
1801

    
1802

    
1803
def SafeEncode(text):
1804
  """Return a 'safe' version of a source string.
1805

1806
  This function mangles the input string and returns a version that
1807
  should be safe to display/encode as ASCII. To this end, we first
1808
  convert it to ASCII using the 'backslashreplace' encoding which
1809
  should get rid of any non-ASCII chars, and then we process it
1810
  through a loop copied from the string repr sources in the python; we
1811
  don't use string_escape anymore since that escape single quotes and
1812
  backslashes too, and that is too much; and that escaping is not
1813
  stable, i.e. string_escape(string_escape(x)) != string_escape(x).
1814

1815
  @type text: str or unicode
1816
  @param text: input data
1817
  @rtype: str
1818
  @return: a safe version of text
1819

1820
  """
1821
  if isinstance(text, unicode):
1822
    # only if unicode; if str already, we handle it below
1823
    text = text.encode('ascii', 'backslashreplace')
1824
  resu = ""
1825
  for char in text:
1826
    c = ord(char)
1827
    if char  == '\t':
1828
      resu += r'\t'
1829
    elif char == '\n':
1830
      resu += r'\n'
1831
    elif char == '\r':
1832
      resu += r'\'r'
1833
    elif c < 32 or c >= 127: # non-printable
1834
      resu += "\\x%02x" % (c & 0xff)
1835
    else:
1836
      resu += char
1837
  return resu
1838

    
1839

    
1840
def CommaJoin(names):
1841
  """Nicely join a set of identifiers.
1842

1843
  @param names: set, list or tuple
1844
  @return: a string with the formatted results
1845

1846
  """
1847
  return ", ".join(["'%s'" % val for val in names])
1848

    
1849

    
1850
def BytesToMebibyte(value):
1851
  """Converts bytes to mebibytes.
1852

1853
  @type value: int
1854
  @param value: Value in bytes
1855
  @rtype: int
1856
  @return: Value in mebibytes
1857

1858
  """
1859
  return int(round(value / (1024.0 * 1024.0), 0))
1860

    
1861

    
1862
def CalculateDirectorySize(path):
1863
  """Calculates the size of a directory recursively.
1864

1865
  @type path: string
1866
  @param path: Path to directory
1867
  @rtype: int
1868
  @return: Size in mebibytes
1869

1870
  """
1871
  size = 0
1872

    
1873
  for (curpath, _, files) in os.walk(path):
1874
    for file in files:
1875
      st = os.lstat(os.path.join(curpath, file))
1876
      size += st.st_size
1877

    
1878
  return BytesToMebibyte(size)
1879

    
1880

    
1881
def GetFreeFilesystemSpace(path):
1882
  """Returns the free space on a filesystem.
1883

1884
  @type path: string
1885
  @param path: Path on filesystem to be examined
1886
  @rtype: int
1887
  @return: Free space in mebibytes
1888

1889
  """
1890
  st = os.statvfs(path)
1891

    
1892
  return BytesToMebibyte(st.f_bavail * st.f_frsize)
1893

    
1894

    
1895
def LockedMethod(fn):
1896
  """Synchronized object access decorator.
1897

1898
  This decorator is intended to protect access to an object using the
1899
  object's own lock which is hardcoded to '_lock'.
1900

1901
  """
1902
  def _LockDebug(*args, **kwargs):
1903
    if debug_locks:
1904
      logging.debug(*args, **kwargs)
1905

    
1906
  def wrapper(self, *args, **kwargs):
1907
    assert hasattr(self, '_lock')
1908
    lock = self._lock
1909
    _LockDebug("Waiting for %s", lock)
1910
    lock.acquire()
1911
    try:
1912
      _LockDebug("Acquired %s", lock)
1913
      result = fn(self, *args, **kwargs)
1914
    finally:
1915
      _LockDebug("Releasing %s", lock)
1916
      lock.release()
1917
      _LockDebug("Released %s", lock)
1918
    return result
1919
  return wrapper
1920

    
1921

    
1922
def LockFile(fd):
1923
  """Locks a file using POSIX locks.
1924

1925
  @type fd: int
1926
  @param fd: the file descriptor we need to lock
1927

1928
  """
1929
  try:
1930
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1931
  except IOError, err:
1932
    if err.errno == errno.EAGAIN:
1933
      raise errors.LockError("File already locked")
1934
    raise
1935

    
1936

    
1937
def FormatTime(val):
1938
  """Formats a time value.
1939

1940
  @type val: float or None
1941
  @param val: the timestamp as returned by time.time()
1942
  @return: a string value or N/A if we don't have a valid timestamp
1943

1944
  """
1945
  if val is None or not isinstance(val, (int, float)):
1946
    return "N/A"
1947
  # these two codes works on Linux, but they are not guaranteed on all
1948
  # platforms
1949
  return time.strftime("%F %T", time.localtime(val))
1950

    
1951

    
1952
class FileLock(object):
1953
  """Utility class for file locks.
1954

1955
  """
1956
  def __init__(self, filename):
1957
    """Constructor for FileLock.
1958

1959
    This will open the file denoted by the I{filename} argument.
1960

1961
    @type filename: str
1962
    @param filename: path to the file to be locked
1963

1964
    """
1965
    self.filename = filename
1966
    self.fd = open(self.filename, "w")
1967

    
1968
  def __del__(self):
1969
    self.Close()
1970

    
1971
  def Close(self):
1972
    """Close the file and release the lock.
1973

1974
    """
1975
    if self.fd:
1976
      self.fd.close()
1977
      self.fd = None
1978

    
1979
  def _flock(self, flag, blocking, timeout, errmsg):
1980
    """Wrapper for fcntl.flock.
1981

1982
    @type flag: int
1983
    @param flag: operation flag
1984
    @type blocking: bool
1985
    @param blocking: whether the operation should be done in blocking mode.
1986
    @type timeout: None or float
1987
    @param timeout: for how long the operation should be retried (implies
1988
                    non-blocking mode).
1989
    @type errmsg: string
1990
    @param errmsg: error message in case operation fails.
1991

1992
    """
1993
    assert self.fd, "Lock was closed"
1994
    assert timeout is None or timeout >= 0, \
1995
      "If specified, timeout must be positive"
1996

    
1997
    if timeout is not None:
1998
      flag |= fcntl.LOCK_NB
1999
      timeout_end = time.time() + timeout
2000

    
2001
    # Blocking doesn't have effect with timeout
2002
    elif not blocking:
2003
      flag |= fcntl.LOCK_NB
2004
      timeout_end = None
2005

    
2006
    retry = True
2007
    while retry:
2008
      try:
2009
        fcntl.flock(self.fd, flag)
2010
        retry = False
2011
      except IOError, err:
2012
        if err.errno in (errno.EAGAIN, ):
2013
          if timeout_end is not None and time.time() < timeout_end:
2014
            # Wait before trying again
2015
            time.sleep(max(0.1, min(1.0, timeout)))
2016
          else:
2017
            raise errors.LockError(errmsg)
2018
        else:
2019
          logging.exception("fcntl.flock failed")
2020
          raise
2021

    
2022
  def Exclusive(self, blocking=False, timeout=None):
2023
    """Locks the file in exclusive mode.
2024

2025
    @type blocking: boolean
2026
    @param blocking: whether to block and wait until we
2027
        can lock the file or return immediately
2028
    @type timeout: int or None
2029
    @param timeout: if not None, the duration to wait for the lock
2030
        (in blocking mode)
2031

2032
    """
2033
    self._flock(fcntl.LOCK_EX, blocking, timeout,
2034
                "Failed to lock %s in exclusive mode" % self.filename)
2035

    
2036
  def Shared(self, blocking=False, timeout=None):
2037
    """Locks the file in shared mode.
2038

2039
    @type blocking: boolean
2040
    @param blocking: whether to block and wait until we
2041
        can lock the file or return immediately
2042
    @type timeout: int or None
2043
    @param timeout: if not None, the duration to wait for the lock
2044
        (in blocking mode)
2045

2046
    """
2047
    self._flock(fcntl.LOCK_SH, blocking, timeout,
2048
                "Failed to lock %s in shared mode" % self.filename)
2049

    
2050
  def Unlock(self, blocking=True, timeout=None):
2051
    """Unlocks the file.
2052

2053
    According to C{flock(2)}, unlocking can also be a nonblocking
2054
    operation::
2055

2056
      To make a non-blocking request, include LOCK_NB with any of the above
2057
      operations.
2058

2059
    @type blocking: boolean
2060
    @param blocking: whether to block and wait until we
2061
        can lock the file or return immediately
2062
    @type timeout: int or None
2063
    @param timeout: if not None, the duration to wait for the lock
2064
        (in blocking mode)
2065

2066
    """
2067
    self._flock(fcntl.LOCK_UN, blocking, timeout,
2068
                "Failed to unlock %s" % self.filename)
2069

    
2070

    
2071
def SignalHandled(signums):
2072
  """Signal Handled decoration.
2073

2074
  This special decorator installs a signal handler and then calls the target
2075
  function. The function must accept a 'signal_handlers' keyword argument,
2076
  which will contain a dict indexed by signal number, with SignalHandler
2077
  objects as values.
2078

2079
  The decorator can be safely stacked with iself, to handle multiple signals
2080
  with different handlers.
2081

2082
  @type signums: list
2083
  @param signums: signals to intercept
2084

2085
  """
2086
  def wrap(fn):
2087
    def sig_function(*args, **kwargs):
2088
      assert 'signal_handlers' not in kwargs or \
2089
             kwargs['signal_handlers'] is None or \
2090
             isinstance(kwargs['signal_handlers'], dict), \
2091
             "Wrong signal_handlers parameter in original function call"
2092
      if 'signal_handlers' in kwargs and kwargs['signal_handlers'] is not None:
2093
        signal_handlers = kwargs['signal_handlers']
2094
      else:
2095
        signal_handlers = {}
2096
        kwargs['signal_handlers'] = signal_handlers
2097
      sighandler = SignalHandler(signums)
2098
      try:
2099
        for sig in signums:
2100
          signal_handlers[sig] = sighandler
2101
        return fn(*args, **kwargs)
2102
      finally:
2103
        sighandler.Reset()
2104
    return sig_function
2105
  return wrap
2106

    
2107

    
2108
class SignalHandler(object):
2109
  """Generic signal handler class.
2110

2111
  It automatically restores the original handler when deconstructed or
2112
  when L{Reset} is called. You can either pass your own handler
2113
  function in or query the L{called} attribute to detect whether the
2114
  signal was sent.
2115

2116
  @type signum: list
2117
  @ivar signum: the signals we handle
2118
  @type called: boolean
2119
  @ivar called: tracks whether any of the signals have been raised
2120

2121
  """
2122
  def __init__(self, signum):
2123
    """Constructs a new SignalHandler instance.
2124

2125
    @type signum: int or list of ints
2126
    @param signum: Single signal number or set of signal numbers
2127

2128
    """
2129
    self.signum = set(signum)
2130
    self.called = False
2131

    
2132
    self._previous = {}
2133
    try:
2134
      for signum in self.signum:
2135
        # Setup handler
2136
        prev_handler = signal.signal(signum, self._HandleSignal)
2137
        try:
2138
          self._previous[signum] = prev_handler
2139
        except:
2140
          # Restore previous handler
2141
          signal.signal(signum, prev_handler)
2142
          raise
2143
    except:
2144
      # Reset all handlers
2145
      self.Reset()
2146
      # Here we have a race condition: a handler may have already been called,
2147
      # but there's not much we can do about it at this point.
2148
      raise
2149

    
2150
  def __del__(self):
2151
    self.Reset()
2152

    
2153
  def Reset(self):
2154
    """Restore previous handler.
2155

2156
    This will reset all the signals to their previous handlers.
2157

2158
    """
2159
    for signum, prev_handler in self._previous.items():
2160
      signal.signal(signum, prev_handler)
2161
      # If successful, remove from dict
2162
      del self._previous[signum]
2163

    
2164
  def Clear(self):
2165
    """Unsets the L{called} flag.
2166

2167
    This function can be used in case a signal may arrive several times.
2168

2169
    """
2170
    self.called = False
2171

    
2172
  def _HandleSignal(self, signum, frame):
2173
    """Actual signal handling function.
2174

2175
    """
2176
    # This is not nice and not absolutely atomic, but it appears to be the only
2177
    # solution in Python -- there are no atomic types.
2178
    self.called = True
2179

    
2180

    
2181
class FieldSet(object):
2182
  """A simple field set.
2183

2184
  Among the features are:
2185
    - checking if a string is among a list of static string or regex objects
2186
    - checking if a whole list of string matches
2187
    - returning the matching groups from a regex match
2188

2189
  Internally, all fields are held as regular expression objects.
2190

2191
  """
2192
  def __init__(self, *items):
2193
    self.items = [re.compile("^%s$" % value) for value in items]
2194

    
2195
  def Extend(self, other_set):
2196
    """Extend the field set with the items from another one"""
2197
    self.items.extend(other_set.items)
2198

    
2199
  def Matches(self, field):
2200
    """Checks if a field matches the current set
2201

2202
    @type field: str
2203
    @param field: the string to match
2204
    @return: either False or a regular expression match object
2205

2206
    """
2207
    for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
2208
      return m
2209
    return False
2210

    
2211
  def NonMatching(self, items):
2212
    """Returns the list of fields not matching the current set
2213

2214
    @type items: list
2215
    @param items: the list of fields to check
2216
    @rtype: list
2217
    @return: list of non-matching fields
2218

2219
    """
2220
    return [val for val in items if not self.Matches(val)]