Statistics
| Branch: | Tag: | Revision:

root / lib / utils / __init__.py @ 0aee8ee9

History | View | Annotate | Download (21 kB)

1
#
2
#
3

    
4
# Copyright (C) 2006, 2007, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Ganeti utility module.
23

24
This module holds functions that can be used in both daemons (all) and
25
the command line scripts.
26

27
"""
28

    
29
# Allow wildcard import in pylint: disable=W0401
30

    
31
import os
32
import re
33
import errno
34
import pwd
35
import itertools
36
import select
37
import logging
38
import signal
39

    
40
from ganeti import errors
41
from ganeti import constants
42
from ganeti import compat
43

    
44
from ganeti.utils.algo import *
45
from ganeti.utils.filelock import *
46
from ganeti.utils.hash import *
47
from ganeti.utils.io import *
48
from ganeti.utils.log import *
49
from ganeti.utils.mlock import *
50
from ganeti.utils.nodesetup import *
51
from ganeti.utils.process import *
52
from ganeti.utils.retry import *
53
from ganeti.utils.text import *
54
from ganeti.utils.wrapper import *
55
from ganeti.utils.x509 import *
56

    
57

    
58
_VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
59

    
60
UUID_RE = re.compile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-"
61
                     "[a-f0-9]{4}-[a-f0-9]{12}$")
62

    
63

    
64
def ForceDictType(target, key_types, allowed_values=None):
65
  """Force the values of a dict to have certain types.
66

67
  @type target: dict
68
  @param target: the dict to update
69
  @type key_types: dict
70
  @param key_types: dict mapping target dict keys to types
71
                    in constants.ENFORCEABLE_TYPES
72
  @type allowed_values: list
73
  @keyword allowed_values: list of specially allowed values
74

75
  """
76
  if allowed_values is None:
77
    allowed_values = []
78

    
79
  if not isinstance(target, dict):
80
    msg = "Expected dictionary, got '%s'" % target
81
    raise errors.TypeEnforcementError(msg)
82

    
83
  for key in target:
84
    if key not in key_types:
85
      msg = "Unknown parameter '%s'" % key
86
      raise errors.TypeEnforcementError(msg)
87

    
88
    if target[key] in allowed_values:
89
      continue
90

    
91
    ktype = key_types[key]
92
    if ktype not in constants.ENFORCEABLE_TYPES:
93
      msg = "'%s' has non-enforceable type %s" % (key, ktype)
94
      raise errors.ProgrammerError(msg)
95

    
96
    if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
97
      if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
98
        pass
99
      elif not isinstance(target[key], basestring):
100
        if isinstance(target[key], bool) and not target[key]:
101
          target[key] = ""
102
        else:
103
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
104
          raise errors.TypeEnforcementError(msg)
105
    elif ktype == constants.VTYPE_BOOL:
106
      if isinstance(target[key], basestring) and target[key]:
107
        if target[key].lower() == constants.VALUE_FALSE:
108
          target[key] = False
109
        elif target[key].lower() == constants.VALUE_TRUE:
110
          target[key] = True
111
        else:
112
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
113
          raise errors.TypeEnforcementError(msg)
114
      elif target[key]:
115
        target[key] = True
116
      else:
117
        target[key] = False
118
    elif ktype == constants.VTYPE_SIZE:
119
      try:
120
        target[key] = ParseUnit(target[key])
121
      except errors.UnitParseError, err:
122
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
123
              (key, target[key], err)
124
        raise errors.TypeEnforcementError(msg)
125
    elif ktype == constants.VTYPE_INT:
126
      try:
127
        target[key] = int(target[key])
128
      except (ValueError, TypeError):
129
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
130
        raise errors.TypeEnforcementError(msg)
131

    
132

    
133
def ValidateServiceName(name):
134
  """Validate the given service name.
135

136
  @type name: number or string
137
  @param name: Service name or port specification
138

139
  """
140
  try:
141
    numport = int(name)
142
  except (ValueError, TypeError):
143
    # Non-numeric service name
144
    valid = _VALID_SERVICE_NAME_RE.match(name)
145
  else:
146
    # Numeric port (protocols other than TCP or UDP might need adjustments
147
    # here)
148
    valid = (numport >= 0 and numport < (1 << 16))
149

    
150
  if not valid:
151
    raise errors.OpPrereqError("Invalid service name '%s'" % name,
152
                               errors.ECODE_INVAL)
153

    
154
  return name
155

    
156

    
157
def ListVolumeGroups():
158
  """List volume groups and their size
159

160
  @rtype: dict
161
  @return:
162
       Dictionary with keys volume name and values
163
       the size of the volume
164

165
  """
166
  command = "vgs --noheadings --units m --nosuffix -o name,size"
167
  result = RunCmd(command)
168
  retval = {}
169
  if result.failed:
170
    return retval
171

    
172
  for line in result.stdout.splitlines():
173
    try:
174
      name, size = line.split()
175
      size = int(float(size))
176
    except (IndexError, ValueError), err:
177
      logging.error("Invalid output from vgs (%s): %s", err, line)
178
      continue
179

    
180
    retval[name] = size
181

    
182
  return retval
183

    
184

    
185
def BridgeExists(bridge):
186
  """Check whether the given bridge exists in the system
187

188
  @type bridge: str
189
  @param bridge: the bridge name to check
190
  @rtype: boolean
191
  @return: True if it does
192

193
  """
194
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
195

    
196

    
197
def TryConvert(fn, val):
198
  """Try to convert a value ignoring errors.
199

200
  This function tries to apply function I{fn} to I{val}. If no
201
  C{ValueError} or C{TypeError} exceptions are raised, it will return
202
  the result, else it will return the original value. Any other
203
  exceptions are propagated to the caller.
204

205
  @type fn: callable
206
  @param fn: function to apply to the value
207
  @param val: the value to be converted
208
  @return: The converted value if the conversion was successful,
209
      otherwise the original value.
210

211
  """
212
  try:
213
    nv = fn(val)
214
  except (ValueError, TypeError):
215
    nv = val
216
  return nv
217

    
218

    
219
def ParseCpuMask(cpu_mask):
220
  """Parse a CPU mask definition and return the list of CPU IDs.
221

222
  CPU mask format: comma-separated list of CPU IDs
223
  or dash-separated ID ranges
224
  Example: "0-2,5" -> "0,1,2,5"
225

226
  @type cpu_mask: str
227
  @param cpu_mask: CPU mask definition
228
  @rtype: list of int
229
  @return: list of CPU IDs
230

231
  """
232
  if not cpu_mask:
233
    return []
234
  cpu_list = []
235
  for range_def in cpu_mask.split(","):
236
    boundaries = range_def.split("-")
237
    n_elements = len(boundaries)
238
    if n_elements > 2:
239
      raise errors.ParseError("Invalid CPU ID range definition"
240
                              " (only one hyphen allowed): %s" % range_def)
241
    try:
242
      lower = int(boundaries[0])
243
    except (ValueError, TypeError), err:
244
      raise errors.ParseError("Invalid CPU ID value for lower boundary of"
245
                              " CPU ID range: %s" % str(err))
246
    try:
247
      higher = int(boundaries[-1])
248
    except (ValueError, TypeError), err:
249
      raise errors.ParseError("Invalid CPU ID value for higher boundary of"
250
                              " CPU ID range: %s" % str(err))
251
    if lower > higher:
252
      raise errors.ParseError("Invalid CPU ID range definition"
253
                              " (%d > %d): %s" % (lower, higher, range_def))
254
    cpu_list.extend(range(lower, higher + 1))
255
  return cpu_list
256

    
257

    
258
def GetHomeDir(user, default=None):
259
  """Try to get the homedir of the given user.
260

261
  The user can be passed either as a string (denoting the name) or as
262
  an integer (denoting the user id). If the user is not found, the
263
  'default' argument is returned, which defaults to None.
264

265
  """
266
  try:
267
    if isinstance(user, basestring):
268
      result = pwd.getpwnam(user)
269
    elif isinstance(user, (int, long)):
270
      result = pwd.getpwuid(user)
271
    else:
272
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
273
                                   type(user))
274
  except KeyError:
275
    return default
276
  return result.pw_dir
277

    
278

    
279
def FirstFree(seq, base=0):
280
  """Returns the first non-existing integer from seq.
281

282
  The seq argument should be a sorted list of positive integers. The
283
  first time the index of an element is smaller than the element
284
  value, the index will be returned.
285

286
  The base argument is used to start at a different offset,
287
  i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
288

289
  Example: C{[0, 1, 3]} will return I{2}.
290

291
  @type seq: sequence
292
  @param seq: the sequence to be analyzed.
293
  @type base: int
294
  @param base: use this value as the base index of the sequence
295
  @rtype: int
296
  @return: the first non-used index in the sequence
297

298
  """
299
  for idx, elem in enumerate(seq):
300
    assert elem >= base, "Passed element is higher than base offset"
301
    if elem > idx + base:
302
      # idx is not used
303
      return idx + base
304
  return None
305

    
306

    
307
def SingleWaitForFdCondition(fdobj, event, timeout):
308
  """Waits for a condition to occur on the socket.
309

310
  Immediately returns at the first interruption.
311

312
  @type fdobj: integer or object supporting a fileno() method
313
  @param fdobj: entity to wait for events on
314
  @type event: integer
315
  @param event: ORed condition (see select module)
316
  @type timeout: float or None
317
  @param timeout: Timeout in seconds
318
  @rtype: int or None
319
  @return: None for timeout, otherwise occured conditions
320

321
  """
322
  check = (event | select.POLLPRI |
323
           select.POLLNVAL | select.POLLHUP | select.POLLERR)
324

    
325
  if timeout is not None:
326
    # Poller object expects milliseconds
327
    timeout *= 1000
328

    
329
  poller = select.poll()
330
  poller.register(fdobj, event)
331
  try:
332
    # TODO: If the main thread receives a signal and we have no timeout, we
333
    # could wait forever. This should check a global "quit" flag or something
334
    # every so often.
335
    io_events = poller.poll(timeout)
336
  except select.error, err:
337
    if err[0] != errno.EINTR:
338
      raise
339
    io_events = []
340
  if io_events and io_events[0][1] & check:
341
    return io_events[0][1]
342
  else:
343
    return None
344

    
345

    
346
class FdConditionWaiterHelper(object):
347
  """Retry helper for WaitForFdCondition.
348

349
  This class contains the retried and wait functions that make sure
350
  WaitForFdCondition can continue waiting until the timeout is actually
351
  expired.
352

353
  """
354

    
355
  def __init__(self, timeout):
356
    self.timeout = timeout
357

    
358
  def Poll(self, fdobj, event):
359
    result = SingleWaitForFdCondition(fdobj, event, self.timeout)
360
    if result is None:
361
      raise RetryAgain()
362
    else:
363
      return result
364

    
365
  def UpdateTimeout(self, timeout):
366
    self.timeout = timeout
367

    
368

    
369
def WaitForFdCondition(fdobj, event, timeout):
370
  """Waits for a condition to occur on the socket.
371

372
  Retries until the timeout is expired, even if interrupted.
373

374
  @type fdobj: integer or object supporting a fileno() method
375
  @param fdobj: entity to wait for events on
376
  @type event: integer
377
  @param event: ORed condition (see select module)
378
  @type timeout: float or None
379
  @param timeout: Timeout in seconds
380
  @rtype: int or None
381
  @return: None for timeout, otherwise occured conditions
382

383
  """
384
  if timeout is not None:
385
    retrywaiter = FdConditionWaiterHelper(timeout)
386
    try:
387
      result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
388
                     args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
389
    except RetryTimeout:
390
      result = None
391
  else:
392
    result = None
393
    while result is None:
394
      result = SingleWaitForFdCondition(fdobj, event, timeout)
395
  return result
396

    
397

    
398
def EnsureDaemon(name):
399
  """Check for and start daemon if not alive.
400

401
  """
402
  result = RunCmd([constants.DAEMON_UTIL, "check-and-start", name])
403
  if result.failed:
404
    logging.error("Can't start daemon '%s', failure %s, output: %s",
405
                  name, result.fail_reason, result.output)
406
    return False
407

    
408
  return True
409

    
410

    
411
def StopDaemon(name):
412
  """Stop daemon
413

414
  """
415
  result = RunCmd([constants.DAEMON_UTIL, "stop", name])
416
  if result.failed:
417
    logging.error("Can't stop daemon '%s', failure %s, output: %s",
418
                  name, result.fail_reason, result.output)
419
    return False
420

    
421
  return True
422

    
423

    
424
def CheckVolumeGroupSize(vglist, vgname, minsize):
425
  """Checks if the volume group list is valid.
426

427
  The function will check if a given volume group is in the list of
428
  volume groups and has a minimum size.
429

430
  @type vglist: dict
431
  @param vglist: dictionary of volume group names and their size
432
  @type vgname: str
433
  @param vgname: the volume group we should check
434
  @type minsize: int
435
  @param minsize: the minimum size we accept
436
  @rtype: None or str
437
  @return: None for success, otherwise the error message
438

439
  """
440
  vgsize = vglist.get(vgname, None)
441
  if vgsize is None:
442
    return "volume group '%s' missing" % vgname
443
  elif vgsize < minsize:
444
    return ("volume group '%s' too small (%s MiB required, %d MiB found)" %
445
            (vgname, minsize, vgsize))
446
  return None
447

    
448

    
449
def SplitTime(value):
450
  """Splits time as floating point number into a tuple.
451

452
  @param value: Time in seconds
453
  @type value: int or float
454
  @return: Tuple containing (seconds, microseconds)
455

456
  """
457
  (seconds, microseconds) = divmod(int(value * 1000000), 1000000)
458

    
459
  assert 0 <= seconds, \
460
    "Seconds must be larger than or equal to 0, but are %s" % seconds
461
  assert 0 <= microseconds <= 999999, \
462
    "Microseconds must be 0-999999, but are %s" % microseconds
463

    
464
  return (int(seconds), int(microseconds))
465

    
466

    
467
def MergeTime(timetuple):
468
  """Merges a tuple into time as a floating point number.
469

470
  @param timetuple: Time as tuple, (seconds, microseconds)
471
  @type timetuple: tuple
472
  @return: Time as a floating point number expressed in seconds
473

474
  """
475
  (seconds, microseconds) = timetuple
476

    
477
  assert 0 <= seconds, \
478
    "Seconds must be larger than or equal to 0, but are %s" % seconds
479
  assert 0 <= microseconds <= 999999, \
480
    "Microseconds must be 0-999999, but are %s" % microseconds
481

    
482
  return float(seconds) + (float(microseconds) * 0.000001)
483

    
484

    
485
def FindMatch(data, name):
486
  """Tries to find an item in a dictionary matching a name.
487

488
  Callers have to ensure the data names aren't contradictory (e.g. a regexp
489
  that matches a string). If the name isn't a direct key, all regular
490
  expression objects in the dictionary are matched against it.
491

492
  @type data: dict
493
  @param data: Dictionary containing data
494
  @type name: string
495
  @param name: Name to look for
496
  @rtype: tuple; (value in dictionary, matched groups as list)
497

498
  """
499
  if name in data:
500
    return (data[name], [])
501

    
502
  for key, value in data.items():
503
    # Regex objects
504
    if hasattr(key, "match"):
505
      m = key.match(name)
506
      if m:
507
        return (value, list(m.groups()))
508

    
509
  return None
510

    
511

    
512
def GetMounts(filename=constants.PROC_MOUNTS):
513
  """Returns the list of mounted filesystems.
514

515
  This function is Linux-specific.
516

517
  @param filename: path of mounts file (/proc/mounts by default)
518
  @rtype: list of tuples
519
  @return: list of mount entries (device, mountpoint, fstype, options)
520

521
  """
522
  # TODO(iustin): investigate non-Linux options (e.g. via mount output)
523
  data = []
524
  mountlines = ReadFile(filename).splitlines()
525
  for line in mountlines:
526
    device, mountpoint, fstype, options, _ = line.split(None, 4)
527
    data.append((device, mountpoint, fstype, options))
528

    
529
  return data
530

    
531

    
532
def SignalHandled(signums):
533
  """Signal Handled decoration.
534

535
  This special decorator installs a signal handler and then calls the target
536
  function. The function must accept a 'signal_handlers' keyword argument,
537
  which will contain a dict indexed by signal number, with SignalHandler
538
  objects as values.
539

540
  The decorator can be safely stacked with iself, to handle multiple signals
541
  with different handlers.
542

543
  @type signums: list
544
  @param signums: signals to intercept
545

546
  """
547
  def wrap(fn):
548
    def sig_function(*args, **kwargs):
549
      assert "signal_handlers" not in kwargs or \
550
             kwargs["signal_handlers"] is None or \
551
             isinstance(kwargs["signal_handlers"], dict), \
552
             "Wrong signal_handlers parameter in original function call"
553
      if "signal_handlers" in kwargs and kwargs["signal_handlers"] is not None:
554
        signal_handlers = kwargs["signal_handlers"]
555
      else:
556
        signal_handlers = {}
557
        kwargs["signal_handlers"] = signal_handlers
558
      sighandler = SignalHandler(signums)
559
      try:
560
        for sig in signums:
561
          signal_handlers[sig] = sighandler
562
        return fn(*args, **kwargs)
563
      finally:
564
        sighandler.Reset()
565
    return sig_function
566
  return wrap
567

    
568

    
569
class SignalWakeupFd(object):
570
  try:
571
    # This is only supported in Python 2.5 and above (some distributions
572
    # backported it to Python 2.4)
573
    _set_wakeup_fd_fn = signal.set_wakeup_fd
574
  except AttributeError:
575
    # Not supported
576
    def _SetWakeupFd(self, _): # pylint: disable=R0201
577
      return -1
578
  else:
579
    def _SetWakeupFd(self, fd):
580
      return self._set_wakeup_fd_fn(fd)
581

    
582
  def __init__(self):
583
    """Initializes this class.
584

585
    """
586
    (read_fd, write_fd) = os.pipe()
587

    
588
    # Once these succeeded, the file descriptors will be closed automatically.
589
    # Buffer size 0 is important, otherwise .read() with a specified length
590
    # might buffer data and the file descriptors won't be marked readable.
591
    self._read_fh = os.fdopen(read_fd, "r", 0)
592
    self._write_fh = os.fdopen(write_fd, "w", 0)
593

    
594
    self._previous = self._SetWakeupFd(self._write_fh.fileno())
595

    
596
    # Utility functions
597
    self.fileno = self._read_fh.fileno
598
    self.read = self._read_fh.read
599

    
600
  def Reset(self):
601
    """Restores the previous wakeup file descriptor.
602

603
    """
604
    if hasattr(self, "_previous") and self._previous is not None:
605
      self._SetWakeupFd(self._previous)
606
      self._previous = None
607

    
608
  def Notify(self):
609
    """Notifies the wakeup file descriptor.
610

611
    """
612
    self._write_fh.write("\0")
613

    
614
  def __del__(self):
615
    """Called before object deletion.
616

617
    """
618
    self.Reset()
619

    
620

    
621
class SignalHandler(object):
622
  """Generic signal handler class.
623

624
  It automatically restores the original handler when deconstructed or
625
  when L{Reset} is called. You can either pass your own handler
626
  function in or query the L{called} attribute to detect whether the
627
  signal was sent.
628

629
  @type signum: list
630
  @ivar signum: the signals we handle
631
  @type called: boolean
632
  @ivar called: tracks whether any of the signals have been raised
633

634
  """
635
  def __init__(self, signum, handler_fn=None, wakeup=None):
636
    """Constructs a new SignalHandler instance.
637

638
    @type signum: int or list of ints
639
    @param signum: Single signal number or set of signal numbers
640
    @type handler_fn: callable
641
    @param handler_fn: Signal handling function
642

643
    """
644
    assert handler_fn is None or callable(handler_fn)
645

    
646
    self.signum = set(signum)
647
    self.called = False
648

    
649
    self._handler_fn = handler_fn
650
    self._wakeup = wakeup
651

    
652
    self._previous = {}
653
    try:
654
      for signum in self.signum:
655
        # Setup handler
656
        prev_handler = signal.signal(signum, self._HandleSignal)
657
        try:
658
          self._previous[signum] = prev_handler
659
        except:
660
          # Restore previous handler
661
          signal.signal(signum, prev_handler)
662
          raise
663
    except:
664
      # Reset all handlers
665
      self.Reset()
666
      # Here we have a race condition: a handler may have already been called,
667
      # but there's not much we can do about it at this point.
668
      raise
669

    
670
  def __del__(self):
671
    self.Reset()
672

    
673
  def Reset(self):
674
    """Restore previous handler.
675

676
    This will reset all the signals to their previous handlers.
677

678
    """
679
    for signum, prev_handler in self._previous.items():
680
      signal.signal(signum, prev_handler)
681
      # If successful, remove from dict
682
      del self._previous[signum]
683

    
684
  def Clear(self):
685
    """Unsets the L{called} flag.
686

687
    This function can be used in case a signal may arrive several times.
688

689
    """
690
    self.called = False
691

    
692
  def _HandleSignal(self, signum, frame):
693
    """Actual signal handling function.
694

695
    """
696
    # This is not nice and not absolutely atomic, but it appears to be the only
697
    # solution in Python -- there are no atomic types.
698
    self.called = True
699

    
700
    if self._wakeup:
701
      # Notify whoever is interested in signals
702
      self._wakeup.Notify()
703

    
704
    if self._handler_fn:
705
      self._handler_fn(signum, frame)
706

    
707

    
708
class FieldSet(object):
709
  """A simple field set.
710

711
  Among the features are:
712
    - checking if a string is among a list of static string or regex objects
713
    - checking if a whole list of string matches
714
    - returning the matching groups from a regex match
715

716
  Internally, all fields are held as regular expression objects.
717

718
  """
719
  def __init__(self, *items):
720
    self.items = [re.compile("^%s$" % value) for value in items]
721

    
722
  def Extend(self, other_set):
723
    """Extend the field set with the items from another one"""
724
    self.items.extend(other_set.items)
725

    
726
  def Matches(self, field):
727
    """Checks if a field matches the current set
728

729
    @type field: str
730
    @param field: the string to match
731
    @return: either None or a regular expression match object
732

733
    """
734
    for m in itertools.ifilter(None, (val.match(field) for val in self.items)):
735
      return m
736
    return None
737

    
738
  def NonMatching(self, items):
739
    """Returns the list of fields not matching the current set
740

741
    @type items: list
742
    @param items: the list of fields to check
743
    @rtype: list
744
    @return: list of non-matching fields
745

746
    """
747
    return [val for val in items if not self.Matches(val)]