Statistics
| Branch: | Tag: | Revision:

root / lib / utils.py @ aa65ed72

History | View | Annotate | Download (32.9 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Ganeti small utilities
23

24
"""
25

    
26

    
27
import sys
28
import os
29
import sha
30
import time
31
import subprocess
32
import re
33
import socket
34
import tempfile
35
import shutil
36
import errno
37
import pwd
38
import itertools
39
import select
40
import fcntl
41
import resource
42
import logging
43
import signal
44

    
45
from cStringIO import StringIO
46

    
47
from ganeti import errors
48
from ganeti import constants
49

    
50

    
51
_locksheld = []
52
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
53

    
54
debug = False
55
no_fork = False
56

    
57

    
58
class RunResult(object):
59
  """Simple class for holding the result of running external programs.
60

61
  Instance variables:
62
    exit_code: the exit code of the program, or None (if the program
63
               didn't exit())
64
    signal: numeric signal that caused the program to finish, or None
65
            (if the program wasn't terminated by a signal)
66
    stdout: the standard output of the program
67
    stderr: the standard error of the program
68
    failed: a Boolean value which is True in case the program was
69
            terminated by a signal or exited with a non-zero exit code
70
    fail_reason: a string detailing the termination reason
71

72
  """
73
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
74
               "failed", "fail_reason", "cmd"]
75

    
76

    
77
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
78
    self.cmd = cmd
79
    self.exit_code = exit_code
80
    self.signal = signal_
81
    self.stdout = stdout
82
    self.stderr = stderr
83
    self.failed = (signal_ is not None or exit_code != 0)
84

    
85
    if self.signal is not None:
86
      self.fail_reason = "terminated by signal %s" % self.signal
87
    elif self.exit_code is not None:
88
      self.fail_reason = "exited with exit code %s" % self.exit_code
89
    else:
90
      self.fail_reason = "unable to determine termination reason"
91

    
92
    if self.failed:
93
      logging.debug("Command '%s' failed (%s); output: %s",
94
                    self.cmd, self.fail_reason, self.output)
95

    
96
  def _GetOutput(self):
97
    """Returns the combined stdout and stderr for easier usage.
98

99
    """
100
    return self.stdout + self.stderr
101

    
102
  output = property(_GetOutput, None, None, "Return full output")
103

    
104

    
105
def RunCmd(cmd):
106
  """Execute a (shell) command.
107

108
  The command should not read from its standard input, as it will be
109
  closed.
110

111
  Args:
112
    cmd: command to run. (str)
113

114
  Returns: `RunResult` instance
115

116
  """
117
  if no_fork:
118
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
119

    
120
  if isinstance(cmd, list):
121
    cmd = [str(val) for val in cmd]
122
    strcmd = " ".join(cmd)
123
    shell = False
124
  else:
125
    strcmd = cmd
126
    shell = True
127
  logging.debug("RunCmd '%s'", strcmd)
128
  env = os.environ.copy()
129
  env["LC_ALL"] = "C"
130
  poller = select.poll()
131
  child = subprocess.Popen(cmd, shell=shell,
132
                           stderr=subprocess.PIPE,
133
                           stdout=subprocess.PIPE,
134
                           stdin=subprocess.PIPE,
135
                           close_fds=True, env=env)
136

    
137
  child.stdin.close()
138
  poller.register(child.stdout, select.POLLIN)
139
  poller.register(child.stderr, select.POLLIN)
140
  out = StringIO()
141
  err = StringIO()
142
  fdmap = {
143
    child.stdout.fileno(): (out, child.stdout),
144
    child.stderr.fileno(): (err, child.stderr),
145
    }
146
  for fd in fdmap:
147
    status = fcntl.fcntl(fd, fcntl.F_GETFL)
148
    fcntl.fcntl(fd, fcntl.F_SETFL, status | os.O_NONBLOCK)
149

    
150
  while fdmap:
151
    for fd, event in poller.poll():
152
      if event & select.POLLIN or event & select.POLLPRI:
153
        data = fdmap[fd][1].read()
154
        # no data from read signifies EOF (the same as POLLHUP)
155
        if not data:
156
          poller.unregister(fd)
157
          del fdmap[fd]
158
          continue
159
        fdmap[fd][0].write(data)
160
      if (event & select.POLLNVAL or event & select.POLLHUP or
161
          event & select.POLLERR):
162
        poller.unregister(fd)
163
        del fdmap[fd]
164

    
165
  out = out.getvalue()
166
  err = err.getvalue()
167

    
168
  status = child.wait()
169
  if status >= 0:
170
    exitcode = status
171
    signal_ = None
172
  else:
173
    exitcode = None
174
    signal_ = -status
175

    
176
  return RunResult(exitcode, signal_, out, err, strcmd)
177

    
178

    
179
def RemoveFile(filename):
180
  """Remove a file ignoring some errors.
181

182
  Remove a file, ignoring non-existing ones or directories. Other
183
  errors are passed.
184

185
  """
186
  try:
187
    os.unlink(filename)
188
  except OSError, err:
189
    if err.errno not in (errno.ENOENT, errno.EISDIR):
190
      raise
191

    
192

    
193
def _FingerprintFile(filename):
194
  """Compute the fingerprint of a file.
195

196
  If the file does not exist, a None will be returned
197
  instead.
198

199
  Args:
200
    filename - Filename (str)
201

202
  """
203
  if not (os.path.exists(filename) and os.path.isfile(filename)):
204
    return None
205

    
206
  f = open(filename)
207

    
208
  fp = sha.sha()
209
  while True:
210
    data = f.read(4096)
211
    if not data:
212
      break
213

    
214
    fp.update(data)
215

    
216
  return fp.hexdigest()
217

    
218

    
219
def FingerprintFiles(files):
220
  """Compute fingerprints for a list of files.
221

222
  Args:
223
    files - array of filenames.  ( [str, ...] )
224

225
  Return value:
226
    dictionary of filename: fingerprint for the files that exist
227

228
  """
229
  ret = {}
230

    
231
  for filename in files:
232
    cksum = _FingerprintFile(filename)
233
    if cksum:
234
      ret[filename] = cksum
235

    
236
  return ret
237

    
238

    
239
def CheckDict(target, template, logname=None):
240
  """Ensure a dictionary has a required set of keys.
241

242
  For the given dictionaries `target` and `template`, ensure target
243
  has all the keys from template. Missing keys are added with values
244
  from template.
245

246
  Args:
247
    target   - the dictionary to check
248
    template - template dictionary
249
    logname  - a caller-chosen string to identify the debug log
250
               entry; if None, no logging will be done
251

252
  Returns value:
253
    None
254

255
  """
256
  missing = []
257
  for k in template:
258
    if k not in target:
259
      missing.append(k)
260
      target[k] = template[k]
261

    
262
  if missing and logname:
263
    logging.warning('%s missing keys %s', logname, ', '.join(missing))
264

    
265

    
266
def IsProcessAlive(pid):
267
  """Check if a given pid exists on the system.
268

269
  Returns: true or false, depending on if the pid exists or not
270

271
  Remarks: zombie processes treated as not alive, and giving a pid <=
272
  0 makes the function to return False.
273

274
  """
275
  if pid <= 0:
276
    return False
277

    
278
  try:
279
    f = open("/proc/%d/status" % pid)
280
  except IOError, err:
281
    if err.errno in (errno.ENOENT, errno.ENOTDIR):
282
      return False
283

    
284
  alive = True
285
  try:
286
    data = f.readlines()
287
    if len(data) > 1:
288
      state = data[1].split()
289
      if len(state) > 1 and state[1] == "Z":
290
        alive = False
291
  finally:
292
    f.close()
293

    
294
  return alive
295

    
296

    
297
def ReadPidFile(pidfile):
298
  """Read the pid from a file.
299

300
  @param pidfile: Path to a file containing the pid to be checked
301
  @type  pidfile: string (filename)
302
  @return: The process id, if the file exista and contains a valid PID,
303
           otherwise 0
304
  @rtype: int
305

306
  """
307
  try:
308
    pf = open(pidfile, 'r')
309
  except EnvironmentError, err:
310
    if err.errno != errno.ENOENT:
311
      logging.exception("Can't read pid file?!")
312
    return 0
313

    
314
  try:
315
    pid = int(pf.read())
316
  except ValueError, err:
317
    logging.info("Can't parse pid file contents", exc_info=True)
318
    return 0
319

    
320
  return pid
321

    
322

    
323
def MatchNameComponent(key, name_list):
324
  """Try to match a name against a list.
325

326
  This function will try to match a name like test1 against a list
327
  like ['test1.example.com', 'test2.example.com', ...]. Against this
328
  list, 'test1' as well as 'test1.example' will match, but not
329
  'test1.ex'. A multiple match will be considered as no match at all
330
  (e.g. 'test1' against ['test1.example.com', 'test1.example.org']).
331

332
  Args:
333
    key: the name to be searched
334
    name_list: the list of strings against which to search the key
335

336
  Returns:
337
    None if there is no match *or* if there are multiple matches
338
    otherwise the element from the list which matches
339

340
  """
341
  mo = re.compile("^%s(\..*)?$" % re.escape(key))
342
  names_filtered = [name for name in name_list if mo.match(name) is not None]
343
  if len(names_filtered) != 1:
344
    return None
345
  return names_filtered[0]
346

    
347

    
348
class HostInfo:
349
  """Class implementing resolver and hostname functionality
350

351
  """
352
  def __init__(self, name=None):
353
    """Initialize the host name object.
354

355
    If the name argument is not passed, it will use this system's
356
    name.
357

358
    """
359
    if name is None:
360
      name = self.SysName()
361

    
362
    self.query = name
363
    self.name, self.aliases, self.ipaddrs = self.LookupHostname(name)
364
    self.ip = self.ipaddrs[0]
365

    
366
  def ShortName(self):
367
    """Returns the hostname without domain.
368

369
    """
370
    return self.name.split('.')[0]
371

    
372
  @staticmethod
373
  def SysName():
374
    """Return the current system's name.
375

376
    This is simply a wrapper over socket.gethostname()
377

378
    """
379
    return socket.gethostname()
380

    
381
  @staticmethod
382
  def LookupHostname(hostname):
383
    """Look up hostname
384

385
    Args:
386
      hostname: hostname to look up
387

388
    Returns:
389
      a tuple (name, aliases, ipaddrs) as returned by socket.gethostbyname_ex
390
      in case of errors in resolving, we raise a ResolverError
391

392
    """
393
    try:
394
      result = socket.gethostbyname_ex(hostname)
395
    except socket.gaierror, err:
396
      # hostname not found in DNS
397
      raise errors.ResolverError(hostname, err.args[0], err.args[1])
398

    
399
    return result
400

    
401

    
402
def ListVolumeGroups():
403
  """List volume groups and their size
404

405
  Returns:
406
     Dictionary with keys volume name and values the size of the volume
407

408
  """
409
  command = "vgs --noheadings --units m --nosuffix -o name,size"
410
  result = RunCmd(command)
411
  retval = {}
412
  if result.failed:
413
    return retval
414

    
415
  for line in result.stdout.splitlines():
416
    try:
417
      name, size = line.split()
418
      size = int(float(size))
419
    except (IndexError, ValueError), err:
420
      logging.error("Invalid output from vgs (%s): %s", err, line)
421
      continue
422

    
423
    retval[name] = size
424

    
425
  return retval
426

    
427

    
428
def BridgeExists(bridge):
429
  """Check whether the given bridge exists in the system
430

431
  Returns:
432
     True if it does, false otherwise.
433

434
  """
435
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
436

    
437

    
438
def NiceSort(name_list):
439
  """Sort a list of strings based on digit and non-digit groupings.
440

441
  Given a list of names ['a1', 'a10', 'a11', 'a2'] this function will
442
  sort the list in the logical order ['a1', 'a2', 'a10', 'a11'].
443

444
  The sort algorithm breaks each name in groups of either only-digits
445
  or no-digits. Only the first eight such groups are considered, and
446
  after that we just use what's left of the string.
447

448
  Return value
449
    - a copy of the list sorted according to our algorithm
450

451
  """
452
  _SORTER_BASE = "(\D+|\d+)"
453
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
454
                                                  _SORTER_BASE, _SORTER_BASE,
455
                                                  _SORTER_BASE, _SORTER_BASE,
456
                                                  _SORTER_BASE, _SORTER_BASE)
457
  _SORTER_RE = re.compile(_SORTER_FULL)
458
  _SORTER_NODIGIT = re.compile("^\D*$")
459
  def _TryInt(val):
460
    """Attempts to convert a variable to integer."""
461
    if val is None or _SORTER_NODIGIT.match(val):
462
      return val
463
    rval = int(val)
464
    return rval
465

    
466
  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
467
             for name in name_list]
468
  to_sort.sort()
469
  return [tup[1] for tup in to_sort]
470

    
471

    
472
def TryConvert(fn, val):
473
  """Try to convert a value ignoring errors.
474

475
  This function tries to apply function `fn` to `val`. If no
476
  ValueError or TypeError exceptions are raised, it will return the
477
  result, else it will return the original value. Any other exceptions
478
  are propagated to the caller.
479

480
  """
481
  try:
482
    nv = fn(val)
483
  except (ValueError, TypeError), err:
484
    nv = val
485
  return nv
486

    
487

    
488
def IsValidIP(ip):
489
  """Verifies the syntax of an IP address.
490

491
  This function checks if the ip address passes is valid or not based
492
  on syntax (not ip range, class calculations or anything).
493

494
  """
495
  unit = "(0|[1-9]\d{0,2})"
496
  return re.match("^%s\.%s\.%s\.%s$" % (unit, unit, unit, unit), ip)
497

    
498

    
499
def IsValidShellParam(word):
500
  """Verifies is the given word is safe from the shell's p.o.v.
501

502
  This means that we can pass this to a command via the shell and be
503
  sure that it doesn't alter the command line and is passed as such to
504
  the actual command.
505

506
  Note that we are overly restrictive here, in order to be on the safe
507
  side.
508

509
  """
510
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
511

    
512

    
513
def BuildShellCmd(template, *args):
514
  """Build a safe shell command line from the given arguments.
515

516
  This function will check all arguments in the args list so that they
517
  are valid shell parameters (i.e. they don't contain shell
518
  metacharaters). If everything is ok, it will return the result of
519
  template % args.
520

521
  """
522
  for word in args:
523
    if not IsValidShellParam(word):
524
      raise errors.ProgrammerError("Shell argument '%s' contains"
525
                                   " invalid characters" % word)
526
  return template % args
527

    
528

    
529
def FormatUnit(value):
530
  """Formats an incoming number of MiB with the appropriate unit.
531

532
  Value needs to be passed as a numeric type. Return value is always a string.
533

534
  """
535
  if value < 1024:
536
    return "%dM" % round(value, 0)
537

    
538
  elif value < (1024 * 1024):
539
    return "%0.1fG" % round(float(value) / 1024, 1)
540

    
541
  else:
542
    return "%0.1fT" % round(float(value) / 1024 / 1024, 1)
543

    
544

    
545
def ParseUnit(input_string):
546
  """Tries to extract number and scale from the given string.
547

548
  Input must be in the format NUMBER+ [DOT NUMBER+] SPACE* [UNIT]. If no unit
549
  is specified, it defaults to MiB. Return value is always an int in MiB.
550

551
  """
552
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', input_string)
553
  if not m:
554
    raise errors.UnitParseError("Invalid format")
555

    
556
  value = float(m.groups()[0])
557

    
558
  unit = m.groups()[1]
559
  if unit:
560
    lcunit = unit.lower()
561
  else:
562
    lcunit = 'm'
563

    
564
  if lcunit in ('m', 'mb', 'mib'):
565
    # Value already in MiB
566
    pass
567

    
568
  elif lcunit in ('g', 'gb', 'gib'):
569
    value *= 1024
570

    
571
  elif lcunit in ('t', 'tb', 'tib'):
572
    value *= 1024 * 1024
573

    
574
  else:
575
    raise errors.UnitParseError("Unknown unit: %s" % unit)
576

    
577
  # Make sure we round up
578
  if int(value) < value:
579
    value += 1
580

    
581
  # Round up to the next multiple of 4
582
  value = int(value)
583
  if value % 4:
584
    value += 4 - value % 4
585

    
586
  return value
587

    
588

    
589
def AddAuthorizedKey(file_name, key):
590
  """Adds an SSH public key to an authorized_keys file.
591

592
  Args:
593
    file_name: Path to authorized_keys file
594
    key: String containing key
595
  """
596
  key_fields = key.split()
597

    
598
  f = open(file_name, 'a+')
599
  try:
600
    nl = True
601
    for line in f:
602
      # Ignore whitespace changes
603
      if line.split() == key_fields:
604
        break
605
      nl = line.endswith('\n')
606
    else:
607
      if not nl:
608
        f.write("\n")
609
      f.write(key.rstrip('\r\n'))
610
      f.write("\n")
611
      f.flush()
612
  finally:
613
    f.close()
614

    
615

    
616
def RemoveAuthorizedKey(file_name, key):
617
  """Removes an SSH public key from an authorized_keys file.
618

619
  Args:
620
    file_name: Path to authorized_keys file
621
    key: String containing key
622
  """
623
  key_fields = key.split()
624

    
625
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
626
  try:
627
    out = os.fdopen(fd, 'w')
628
    try:
629
      f = open(file_name, 'r')
630
      try:
631
        for line in f:
632
          # Ignore whitespace changes while comparing lines
633
          if line.split() != key_fields:
634
            out.write(line)
635

    
636
        out.flush()
637
        os.rename(tmpname, file_name)
638
      finally:
639
        f.close()
640
    finally:
641
      out.close()
642
  except:
643
    RemoveFile(tmpname)
644
    raise
645

    
646

    
647
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
648
  """Sets the name of an IP address and hostname in /etc/hosts.
649

650
  """
651
  # Ensure aliases are unique
652
  aliases = UniqueSequence([hostname] + aliases)[1:]
653

    
654
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
655
  try:
656
    out = os.fdopen(fd, 'w')
657
    try:
658
      f = open(file_name, 'r')
659
      try:
660
        written = False
661
        for line in f:
662
          fields = line.split()
663
          if fields and not fields[0].startswith('#') and ip == fields[0]:
664
            continue
665
          out.write(line)
666

    
667
        out.write("%s\t%s" % (ip, hostname))
668
        if aliases:
669
          out.write(" %s" % ' '.join(aliases))
670
        out.write('\n')
671

    
672
        out.flush()
673
        os.fsync(out)
674
        os.rename(tmpname, file_name)
675
      finally:
676
        f.close()
677
    finally:
678
      out.close()
679
  except:
680
    RemoveFile(tmpname)
681
    raise
682

    
683

    
684
def AddHostToEtcHosts(hostname):
685
  """Wrapper around SetEtcHostsEntry.
686

687
  """
688
  hi = HostInfo(name=hostname)
689
  SetEtcHostsEntry(constants.ETC_HOSTS, hi.ip, hi.name, [hi.ShortName()])
690

    
691

    
692
def RemoveEtcHostsEntry(file_name, hostname):
693
  """Removes a hostname from /etc/hosts.
694

695
  IP addresses without names are removed from the file.
696
  """
697
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
698
  try:
699
    out = os.fdopen(fd, 'w')
700
    try:
701
      f = open(file_name, 'r')
702
      try:
703
        for line in f:
704
          fields = line.split()
705
          if len(fields) > 1 and not fields[0].startswith('#'):
706
            names = fields[1:]
707
            if hostname in names:
708
              while hostname in names:
709
                names.remove(hostname)
710
              if names:
711
                out.write("%s %s\n" % (fields[0], ' '.join(names)))
712
              continue
713

    
714
          out.write(line)
715

    
716
        out.flush()
717
        os.fsync(out)
718
        os.rename(tmpname, file_name)
719
      finally:
720
        f.close()
721
    finally:
722
      out.close()
723
  except:
724
    RemoveFile(tmpname)
725
    raise
726

    
727

    
728
def RemoveHostFromEtcHosts(hostname):
729
  """Wrapper around RemoveEtcHostsEntry.
730

731
  """
732
  hi = HostInfo(name=hostname)
733
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.name)
734
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hi.ShortName())
735

    
736

    
737
def CreateBackup(file_name):
738
  """Creates a backup of a file.
739

740
  Returns: the path to the newly created backup file.
741

742
  """
743
  if not os.path.isfile(file_name):
744
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
745
                                file_name)
746

    
747
  prefix = '%s.backup-%d.' % (os.path.basename(file_name), int(time.time()))
748
  dir_name = os.path.dirname(file_name)
749

    
750
  fsrc = open(file_name, 'rb')
751
  try:
752
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
753
    fdst = os.fdopen(fd, 'wb')
754
    try:
755
      shutil.copyfileobj(fsrc, fdst)
756
    finally:
757
      fdst.close()
758
  finally:
759
    fsrc.close()
760

    
761
  return backup_name
762

    
763

    
764
def ShellQuote(value):
765
  """Quotes shell argument according to POSIX.
766

767
  """
768
  if _re_shell_unquoted.match(value):
769
    return value
770
  else:
771
    return "'%s'" % value.replace("'", "'\\''")
772

    
773

    
774
def ShellQuoteArgs(args):
775
  """Quotes all given shell arguments and concatenates using spaces.
776

777
  """
778
  return ' '.join([ShellQuote(i) for i in args])
779

    
780

    
781
def TcpPing(target, port, timeout=10, live_port_needed=False, source=None):
782
  """Simple ping implementation using TCP connect(2).
783

784
  Try to do a TCP connect(2) from an optional source IP to the
785
  specified target IP and the specified target port. If the optional
786
  parameter live_port_needed is set to true, requires the remote end
787
  to accept the connection. The timeout is specified in seconds and
788
  defaults to 10 seconds. If the source optional argument is not
789
  passed, the source address selection is left to the kernel,
790
  otherwise we try to connect using the passed address (failures to
791
  bind other than EADDRNOTAVAIL will be ignored).
792

793
  """
794
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
795

    
796
  sucess = False
797

    
798
  if source is not None:
799
    try:
800
      sock.bind((source, 0))
801
    except socket.error, (errcode, errstring):
802
      if errcode == errno.EADDRNOTAVAIL:
803
        success = False
804

    
805
  sock.settimeout(timeout)
806

    
807
  try:
808
    sock.connect((target, port))
809
    sock.close()
810
    success = True
811
  except socket.timeout:
812
    success = False
813
  except socket.error, (errcode, errstring):
814
    success = (not live_port_needed) and (errcode == errno.ECONNREFUSED)
815

    
816
  return success
817

    
818

    
819
def ListVisibleFiles(path):
820
  """Returns a list of all visible files in a directory.
821

822
  """
823
  files = [i for i in os.listdir(path) if not i.startswith(".")]
824
  files.sort()
825
  return files
826

    
827

    
828
def GetHomeDir(user, default=None):
829
  """Try to get the homedir of the given user.
830

831
  The user can be passed either as a string (denoting the name) or as
832
  an integer (denoting the user id). If the user is not found, the
833
  'default' argument is returned, which defaults to None.
834

835
  """
836
  try:
837
    if isinstance(user, basestring):
838
      result = pwd.getpwnam(user)
839
    elif isinstance(user, (int, long)):
840
      result = pwd.getpwuid(user)
841
    else:
842
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
843
                                   type(user))
844
  except KeyError:
845
    return default
846
  return result.pw_dir
847

    
848

    
849
def NewUUID():
850
  """Returns a random UUID.
851

852
  """
853
  f = open("/proc/sys/kernel/random/uuid", "r")
854
  try:
855
    return f.read(128).rstrip("\n")
856
  finally:
857
    f.close()
858

    
859

    
860
def WriteFile(file_name, fn=None, data=None,
861
              mode=None, uid=-1, gid=-1,
862
              atime=None, mtime=None, close=True,
863
              dry_run=False, backup=False,
864
              prewrite=None, postwrite=None):
865
  """(Over)write a file atomically.
866

867
  The file_name and either fn (a function taking one argument, the
868
  file descriptor, and which should write the data to it) or data (the
869
  contents of the file) must be passed. The other arguments are
870
  optional and allow setting the file mode, owner and group, and the
871
  mtime/atime of the file.
872

873
  If the function doesn't raise an exception, it has succeeded and the
874
  target file has the new contents. If the file has raised an
875
  exception, an existing target file should be unmodified and the
876
  temporary file should be removed.
877

878
  Args:
879
    file_name: New filename
880
    fn: Content writing function, called with file descriptor as parameter
881
    data: Content as string
882
    mode: File mode
883
    uid: Owner
884
    gid: Group
885
    atime: Access time
886
    mtime: Modification time
887
    close: Whether to close file after writing it
888
    prewrite: Function object called before writing content
889
    postwrite: Function object called after writing content
890

891
  Returns:
892
    None if "close" parameter evaluates to True, otherwise file descriptor.
893

894
  """
895
  if not os.path.isabs(file_name):
896
    raise errors.ProgrammerError("Path passed to WriteFile is not"
897
                                 " absolute: '%s'" % file_name)
898

    
899
  if [fn, data].count(None) != 1:
900
    raise errors.ProgrammerError("fn or data required")
901

    
902
  if [atime, mtime].count(None) == 1:
903
    raise errors.ProgrammerError("Both atime and mtime must be either"
904
                                 " set or None")
905

    
906
  if backup and not dry_run and os.path.isfile(file_name):
907
    CreateBackup(file_name)
908

    
909
  dir_name, base_name = os.path.split(file_name)
910
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
911
  # here we need to make sure we remove the temp file, if any error
912
  # leaves it in place
913
  try:
914
    if uid != -1 or gid != -1:
915
      os.chown(new_name, uid, gid)
916
    if mode:
917
      os.chmod(new_name, mode)
918
    if callable(prewrite):
919
      prewrite(fd)
920
    if data is not None:
921
      os.write(fd, data)
922
    else:
923
      fn(fd)
924
    if callable(postwrite):
925
      postwrite(fd)
926
    os.fsync(fd)
927
    if atime is not None and mtime is not None:
928
      os.utime(new_name, (atime, mtime))
929
    if not dry_run:
930
      os.rename(new_name, file_name)
931
  finally:
932
    if close:
933
      os.close(fd)
934
      result = None
935
    else:
936
      result = fd
937
    RemoveFile(new_name)
938

    
939
  return result
940

    
941

    
942
def FirstFree(seq, base=0):
943
  """Returns the first non-existing integer from seq.
944

945
  The seq argument should be a sorted list of positive integers. The
946
  first time the index of an element is smaller than the element
947
  value, the index will be returned.
948

949
  The base argument is used to start at a different offset,
950
  i.e. [3, 4, 6] with offset=3 will return 5.
951

952
  Example: [0, 1, 3] will return 2.
953

954
  """
955
  for idx, elem in enumerate(seq):
956
    assert elem >= base, "Passed element is higher than base offset"
957
    if elem > idx + base:
958
      # idx is not used
959
      return idx + base
960
  return None
961

    
962

    
963
def all(seq, pred=bool):
964
  "Returns True if pred(x) is True for every element in the iterable"
965
  for elem in itertools.ifilterfalse(pred, seq):
966
    return False
967
  return True
968

    
969

    
970
def any(seq, pred=bool):
971
  "Returns True if pred(x) is True for at least one element in the iterable"
972
  for elem in itertools.ifilter(pred, seq):
973
    return True
974
  return False
975

    
976

    
977
def UniqueSequence(seq):
978
  """Returns a list with unique elements.
979

980
  Element order is preserved.
981
  """
982
  seen = set()
983
  return [i for i in seq if i not in seen and not seen.add(i)]
984

    
985

    
986
def IsValidMac(mac):
987
  """Predicate to check if a MAC address is valid.
988

989
  Checks wether the supplied MAC address is formally correct, only
990
  accepts colon separated format.
991
  """
992
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$")
993
  return mac_check.match(mac) is not None
994

    
995

    
996
def TestDelay(duration):
997
  """Sleep for a fixed amount of time.
998

999
  """
1000
  if duration < 0:
1001
    return False
1002
  time.sleep(duration)
1003
  return True
1004

    
1005

    
1006
def Daemonize(logfile, noclose_fds=None):
1007
  """Daemonize the current process.
1008

1009
  This detaches the current process from the controlling terminal and
1010
  runs it in the background as a daemon.
1011

1012
  """
1013
  UMASK = 077
1014
  WORKDIR = "/"
1015
  # Default maximum for the number of available file descriptors.
1016
  if 'SC_OPEN_MAX' in os.sysconf_names:
1017
    try:
1018
      MAXFD = os.sysconf('SC_OPEN_MAX')
1019
      if MAXFD < 0:
1020
        MAXFD = 1024
1021
    except OSError:
1022
      MAXFD = 1024
1023
  else:
1024
    MAXFD = 1024
1025

    
1026
  # this might fail
1027
  pid = os.fork()
1028
  if (pid == 0):  # The first child.
1029
    os.setsid()
1030
    # this might fail
1031
    pid = os.fork() # Fork a second child.
1032
    if (pid == 0):  # The second child.
1033
      os.chdir(WORKDIR)
1034
      os.umask(UMASK)
1035
    else:
1036
      # exit() or _exit()?  See below.
1037
      os._exit(0) # Exit parent (the first child) of the second child.
1038
  else:
1039
    os._exit(0) # Exit parent of the first child.
1040
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1041
  if (maxfd == resource.RLIM_INFINITY):
1042
    maxfd = MAXFD
1043

    
1044
  # Iterate through and close all file descriptors.
1045
  for fd in range(0, maxfd):
1046
    if noclose_fds and fd in noclose_fds:
1047
      continue
1048
    try:
1049
      os.close(fd)
1050
    except OSError: # ERROR, fd wasn't open to begin with (ignored)
1051
      pass
1052
  os.open(logfile, os.O_RDWR|os.O_CREAT|os.O_APPEND, 0600)
1053
  # Duplicate standard input to standard output and standard error.
1054
  os.dup2(0, 1)     # standard output (1)
1055
  os.dup2(0, 2)     # standard error (2)
1056
  return 0
1057

    
1058

    
1059
def DaemonPidFileName(name):
1060
  """Compute a ganeti pid file absolute path, given the daemon name.
1061

1062
  """
1063
  return os.path.join(constants.RUN_GANETI_DIR, "%s.pid" % name)
1064

    
1065

    
1066
def WritePidFile(name):
1067
  """Write the current process pidfile.
1068

1069
  The file will be written to constants.RUN_GANETI_DIR/name.pid
1070

1071
  """
1072
  pid = os.getpid()
1073
  pidfilename = DaemonPidFileName(name)
1074
  if IsProcessAlive(ReadPidFile(pidfilename)):
1075
    raise errors.GenericError("%s contains a live process" % pidfilename)
1076

    
1077
  WriteFile(pidfilename, data="%d\n" % pid)
1078

    
1079

    
1080
def RemovePidFile(name):
1081
  """Remove the current process pidfile.
1082

1083
  Any errors are ignored.
1084

1085
  """
1086
  pid = os.getpid()
1087
  pidfilename = DaemonPidFileName(name)
1088
  # TODO: we could check here that the file contains our pid
1089
  try:
1090
    RemoveFile(pidfilename)
1091
  except:
1092
    pass
1093

    
1094

    
1095
def KillProcess(pid, signal_=signal.SIGTERM, timeout=30):
1096
  """Kill a process given by its pid.
1097

1098
  @type pid: int
1099
  @param pid: The PID to terminate.
1100
  @type signal_: int
1101
  @param signal_: The signal to send, by default SIGTERM
1102
  @type timeout: int
1103
  @param timeout: The timeout after which, if the process is still alive,
1104
                  a SIGKILL will be sent. If not positive, no such checking
1105
                  will be done
1106

1107
  """
1108
  if pid <= 0:
1109
    # kill with pid=0 == suicide
1110
    raise errors.ProgrammerError("Invalid pid given '%s'" % pid)
1111

    
1112
  if not IsProcessAlive(pid):
1113
    return
1114
  os.kill(pid, signal_)
1115
  if timeout <= 0:
1116
    return
1117
  end = time.time() + timeout
1118
  while time.time() < end and IsProcessAlive(pid):
1119
    time.sleep(0.1)
1120
  if IsProcessAlive(pid):
1121
    os.kill(pid, signal.SIGKILL)
1122

    
1123

    
1124
def FindFile(name, search_path, test=os.path.exists):
1125
  """Look for a filesystem object in a given path.
1126

1127
  This is an abstract method to search for filesystem object (files,
1128
  dirs) under a given search path.
1129

1130
  Args:
1131
    - name: the name to look for
1132
    - search_path: list of directory names
1133
    - test: the test which the full path must satisfy
1134
      (defaults to os.path.exists)
1135

1136
  Returns:
1137
    - full path to the item if found
1138
    - None otherwise
1139

1140
  """
1141
  for dir_name in search_path:
1142
    item_name = os.path.sep.join([dir_name, name])
1143
    if test(item_name):
1144
      return item_name
1145
  return None
1146

    
1147

    
1148
def CheckVolumeGroupSize(vglist, vgname, minsize):
1149
  """Checks if the volume group list is valid.
1150

1151
  A non-None return value means there's an error, and the return value
1152
  is the error message.
1153

1154
  """
1155
  vgsize = vglist.get(vgname, None)
1156
  if vgsize is None:
1157
    return "volume group '%s' missing" % vgname
1158
  elif vgsize < minsize:
1159
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
1160
            (vgname, minsize, vgsize))
1161
  return None
1162

    
1163

    
1164
def LockedMethod(fn):
1165
  """Synchronized object access decorator.
1166

1167
  This decorator is intended to protect access to an object using the
1168
  object's own lock which is hardcoded to '_lock'.
1169

1170
  """
1171
  def wrapper(self, *args, **kwargs):
1172
    assert hasattr(self, '_lock')
1173
    lock = self._lock
1174
    lock.acquire()
1175
    try:
1176
      result = fn(self, *args, **kwargs)
1177
    finally:
1178
      lock.release()
1179
    return result
1180
  return wrapper
1181

    
1182

    
1183
def LockFile(fd):
1184
  """Locks a file using POSIX locks.
1185

1186
  """
1187
  try:
1188
    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1189
  except IOError, err:
1190
    if err.errno == errno.EAGAIN:
1191
      raise errors.LockError("File already locked")
1192
    raise
1193

    
1194

    
1195
class FileLock(object):
1196
  """Utility class for file locks.
1197

1198
  """
1199
  def __init__(self, filename):
1200
    self.filename = filename
1201
    self.fd = open(self.filename, "w")
1202

    
1203
  def __del__(self):
1204
    self.Close()
1205

    
1206
  def Close(self):
1207
    if self.fd:
1208
      self.fd.close()
1209
      self.fd = None
1210

    
1211
  def _flock(self, flag, blocking, errmsg):
1212
    assert self.fd, "Lock was closed"
1213

    
1214
    if not blocking:
1215
      flag |= fcntl.LOCK_NB
1216

    
1217
    try:
1218
      fcntl.flock(self.fd, flag)
1219
    except IOError, err:
1220
      if err.errno in (errno.EAGAIN, ):
1221
        raise errors.LockError(errmsg)
1222
      else:
1223
        logging.exception("fcntl.flock failed")
1224
        raise
1225

    
1226
  def Exclusive(self, blocking=False):
1227
    """Locks the file in exclusive mode.
1228

1229
    """
1230
    self._flock(fcntl.LOCK_EX, blocking,
1231
                "Failed to lock %s in exclusive mode" % self.filename)
1232

    
1233
  def Shared(self, blocking=False):
1234
    """Locks the file in shared mode.
1235

1236
    """
1237
    self._flock(fcntl.LOCK_SH, blocking,
1238
                "Failed to lock %s in shared mode" % self.filename)
1239

    
1240
  def Unlock(self, blocking=True):
1241
    """Unlocks the file.
1242

1243
    According to "man flock", unlocking can also be a nonblocking operation:
1244
    "To make a non-blocking request, include LOCK_NB with any of the above
1245
    operations"
1246

1247
    """
1248
    self._flock(fcntl.LOCK_UN, blocking,
1249
                "Failed to unlock %s" % self.filename)
1250

    
1251

    
1252
class SignalHandler(object):
1253
  """Generic signal handler class.
1254

1255
  It automatically restores the original handler when deconstructed or when
1256
  Reset() is called. You can either pass your own handler function in or query
1257
  the "called" attribute to detect whether the signal was sent.
1258

1259
  """
1260
  def __init__(self, signum):
1261
    """Constructs a new SignalHandler instance.
1262

1263
    @param signum: Single signal number or set of signal numbers
1264

1265
    """
1266
    if isinstance(signum, (int, long)):
1267
      self.signum = set([signum])
1268
    else:
1269
      self.signum = set(signum)
1270

    
1271
    self.called = False
1272

    
1273
    self._previous = {}
1274
    try:
1275
      for signum in self.signum:
1276
        # Setup handler
1277
        prev_handler = signal.signal(signum, self._HandleSignal)
1278
        try:
1279
          self._previous[signum] = prev_handler
1280
        except:
1281
          # Restore previous handler
1282
          signal.signal(signum, prev_handler)
1283
          raise
1284
    except:
1285
      # Reset all handlers
1286
      self.Reset()
1287
      # Here we have a race condition: a handler may have already been called,
1288
      # but there's not much we can do about it at this point.
1289
      raise
1290

    
1291
  def __del__(self):
1292
    self.Reset()
1293

    
1294
  def Reset(self):
1295
    """Restore previous handler.
1296

1297
    """
1298
    for signum, prev_handler in self._previous.items():
1299
      signal.signal(signum, prev_handler)
1300
      # If successful, remove from dict
1301
      del self._previous[signum]
1302

    
1303
  def Clear(self):
1304
    """Unsets "called" flag.
1305

1306
    This function can be used in case a signal may arrive several times.
1307

1308
    """
1309
    self.called = False
1310

    
1311
  def _HandleSignal(self, signum, frame):
1312
    """Actual signal handling function.
1313

1314
    """
1315
    # This is not nice and not absolutely atomic, but it appears to be the only
1316
    # solution in Python -- there are no atomic types.
1317
    self.called = True