Statistics
| Branch: | Tag: | Revision:

root / qa / qa_config.py @ c072e788

History | View | Annotate | Download (15 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""QA configuration.
23

24
"""
25

    
26
import os
27

    
28
from ganeti import constants
29
from ganeti import utils
30
from ganeti import serializer
31
from ganeti import compat
32
from ganeti import ht
33

    
34
import qa_error
35

    
36

    
37
_INSTANCE_CHECK_KEY = "instance-check"
38
_ENABLED_HV_KEY = "enabled-hypervisors"
39
_VCLUSTER_MASTER_KEY = "vcluster-master"
40
_VCLUSTER_BASEDIR_KEY = "vcluster-basedir"
41

    
42
#: QA configuration (L{_QaConfig})
43
_config = None
44

    
45

    
46
class _QaInstance(object):
47
  __slots__ = [
48
    "name",
49
    "nicmac",
50
    "_used",
51
    "_disk_template",
52
    ]
53

    
54
  def __init__(self, name, nicmac):
55
    """Initializes instances of this class.
56

57
    """
58
    self.name = name
59
    self.nicmac = nicmac
60
    self._used = None
61
    self._disk_template = None
62

    
63
  @classmethod
64
  def FromDict(cls, data):
65
    """Creates instance object from JSON dictionary.
66

67
    """
68
    nicmac = []
69

    
70
    macaddr = data.get("nic.mac/0")
71
    if macaddr:
72
      nicmac.append(macaddr)
73

    
74
    return cls(name=data["name"], nicmac=nicmac)
75

    
76
  def __repr__(self):
77
    status = [
78
      "%s.%s" % (self.__class__.__module__, self.__class__.__name__),
79
      "name=%s" % self.name,
80
      "nicmac=%s" % self.nicmac,
81
      "used=%s" % self._used,
82
      "disk_template=%s" % self._disk_template,
83
      ]
84

    
85
    return "<%s at %#x>" % (" ".join(status), id(self))
86

    
87
  def Use(self):
88
    """Marks instance as being in use.
89

90
    """
91
    assert not self._used
92
    assert self._disk_template is None
93

    
94
    self._used = True
95

    
96
  def Release(self):
97
    """Releases instance and makes it available again.
98

99
    """
100
    assert self._used, \
101
      ("Instance '%s' was never acquired or released more than once" %
102
       self.name)
103

    
104
    self._used = False
105
    self._disk_template = None
106

    
107
  def GetNicMacAddr(self, idx, default):
108
    """Returns MAC address for NIC.
109

110
    @type idx: int
111
    @param idx: NIC index
112
    @param default: Default value
113

114
    """
115
    if len(self.nicmac) > idx:
116
      return self.nicmac[idx]
117
    else:
118
      return default
119

    
120
  def SetDiskTemplate(self, template):
121
    """Set the disk template.
122

123
    """
124
    assert template in constants.DISK_TEMPLATES
125

    
126
    self._disk_template = template
127

    
128
  @property
129
  def used(self):
130
    """Returns boolean denoting whether instance is in use.
131

132
    """
133
    return self._used
134

    
135
  @property
136
  def disk_template(self):
137
    """Returns the current disk template.
138

139
    """
140
    return self._disk_template
141

    
142

    
143
class _QaNode(object):
144
  __slots__ = [
145
    "primary",
146
    "secondary",
147
    "_added",
148
    "_use_count",
149
    ]
150

    
151
  def __init__(self, primary, secondary):
152
    """Initializes instances of this class.
153

154
    """
155
    self.primary = primary
156
    self.secondary = secondary
157
    self._added = False
158
    self._use_count = 0
159

    
160
  @classmethod
161
  def FromDict(cls, data):
162
    """Creates node object from JSON dictionary.
163

164
    """
165
    return cls(primary=data["primary"], secondary=data.get("secondary"))
166

    
167
  def __repr__(self):
168
    status = [
169
      "%s.%s" % (self.__class__.__module__, self.__class__.__name__),
170
      "primary=%s" % self.primary,
171
      "secondary=%s" % self.secondary,
172
      "added=%s" % self._added,
173
      "use_count=%s" % self._use_count,
174
      ]
175

    
176
    return "<%s at %#x>" % (" ".join(status), id(self))
177

    
178
  def Use(self):
179
    """Marks a node as being in use.
180

181
    """
182
    assert self._use_count >= 0
183

    
184
    self._use_count += 1
185

    
186
    return self
187

    
188
  def Release(self):
189
    """Release a node (opposite of L{Use}).
190

191
    """
192
    assert self.use_count > 0
193

    
194
    self._use_count -= 1
195

    
196
  def MarkAdded(self):
197
    """Marks node as having been added to a cluster.
198

199
    """
200
    assert not self._added
201
    self._added = True
202

    
203
  def MarkRemoved(self):
204
    """Marks node as having been removed from a cluster.
205

206
    """
207
    assert self._added
208
    self._added = False
209

    
210
  @property
211
  def added(self):
212
    """Returns whether a node is part of a cluster.
213

214
    """
215
    return self._added
216

    
217
  @property
218
  def use_count(self):
219
    """Returns number of current uses (controlled by L{Use} and L{Release}).
220

221
    """
222
    return self._use_count
223

    
224

    
225
_RESOURCE_CONVERTER = {
226
  "instances": _QaInstance.FromDict,
227
  "nodes": _QaNode.FromDict,
228
  }
229

    
230

    
231
def _ConvertResources((key, value)):
232
  """Converts cluster resources in configuration to Python objects.
233

234
  """
235
  fn = _RESOURCE_CONVERTER.get(key, None)
236
  if fn:
237
    return (key, map(fn, value))
238
  else:
239
    return (key, value)
240

    
241

    
242
class _QaConfig(object):
243
  def __init__(self, data):
244
    """Initializes instances of this class.
245

246
    """
247
    self._data = data
248

    
249
    #: Cluster-wide run-time value of the exclusive storage flag
250
    self._exclusive_storage = None
251

    
252
  @classmethod
253
  def Load(cls, filename):
254
    """Loads a configuration file and produces a configuration object.
255

256
    @type filename: string
257
    @param filename: Path to configuration file
258
    @rtype: L{_QaConfig}
259

260
    """
261
    data = serializer.LoadJson(utils.ReadFile(filename))
262

    
263
    result = cls(dict(map(_ConvertResources,
264
                          data.items()))) # pylint: disable=E1103
265
    result.Validate()
266

    
267
    return result
268

    
269
  def Validate(self):
270
    """Validates loaded configuration data.
271

272
    """
273
    if not self.get("name"):
274
      raise qa_error.Error("Cluster name is required")
275

    
276
    if not self.get("nodes"):
277
      raise qa_error.Error("Need at least one node")
278

    
279
    if not self.get("instances"):
280
      raise qa_error.Error("Need at least one instance")
281

    
282
    if (self.get("disk") is None or
283
        self.get("disk-growth") is None or
284
        len(self.get("disk")) != len(self.get("disk-growth"))):
285
      raise qa_error.Error("Config options 'disk' and 'disk-growth' must exist"
286
                           " and have the same number of items")
287

    
288
    check = self.GetInstanceCheckScript()
289
    if check:
290
      try:
291
        os.stat(check)
292
      except EnvironmentError, err:
293
        raise qa_error.Error("Can't find instance check script '%s': %s" %
294
                             (check, err))
295

    
296
    enabled_hv = frozenset(self.GetEnabledHypervisors())
297
    if not enabled_hv:
298
      raise qa_error.Error("No hypervisor is enabled")
299

    
300
    difference = enabled_hv - constants.HYPER_TYPES
301
    if difference:
302
      raise qa_error.Error("Unknown hypervisor(s) enabled: %s" %
303
                           utils.CommaJoin(difference))
304

    
305
    (vc_master, vc_basedir) = self.GetVclusterSettings()
306
    if bool(vc_master) != bool(vc_basedir):
307
      raise qa_error.Error("All or none of the config options '%s' and '%s'"
308
                           " must be set" %
309
                           (_VCLUSTER_MASTER_KEY, _VCLUSTER_BASEDIR_KEY))
310

    
311
    if vc_basedir and not utils.IsNormAbsPath(vc_basedir):
312
      raise qa_error.Error("Path given in option '%s' must be absolute and"
313
                           " normalized" % _VCLUSTER_BASEDIR_KEY)
314

    
315
  def __getitem__(self, name):
316
    """Returns configuration value.
317

318
    @type name: string
319
    @param name: Name of configuration entry
320

321
    """
322
    return self._data[name]
323

    
324
  def get(self, name, default=None):
325
    """Returns configuration value.
326

327
    @type name: string
328
    @param name: Name of configuration entry
329
    @param default: Default value
330

331
    """
332
    return self._data.get(name, default)
333

    
334
  def GetMasterNode(self):
335
    """Returns the default master node for the cluster.
336

337
    """
338
    return self["nodes"][0]
339

    
340
  def GetInstanceCheckScript(self):
341
    """Returns path to instance check script or C{None}.
342

343
    """
344
    return self._data.get(_INSTANCE_CHECK_KEY, None)
345

    
346
  def GetEnabledHypervisors(self):
347
    """Returns list of enabled hypervisors.
348

349
    @rtype: list
350

351
    """
352
    try:
353
      value = self._data[_ENABLED_HV_KEY]
354
    except KeyError:
355
      return [constants.DEFAULT_ENABLED_HYPERVISOR]
356
    else:
357
      if value is None:
358
        return []
359
      elif isinstance(value, basestring):
360
        # The configuration key ("enabled-hypervisors") implies there can be
361
        # multiple values. Multiple hypervisors are comma-separated on the
362
        # command line option to "gnt-cluster init", so we need to handle them
363
        # equally here.
364
        return value.split(",")
365
      else:
366
        return value
367

    
368
  def GetDefaultHypervisor(self):
369
    """Returns the default hypervisor to be used.
370

371
    """
372
    return self.GetEnabledHypervisors()[0]
373

    
374
  def SetExclusiveStorage(self, value):
375
    """Set the expected value of the C{exclusive_storage} flag for the cluster.
376

377
    """
378
    self._exclusive_storage = bool(value)
379

    
380
  def GetExclusiveStorage(self):
381
    """Get the expected value of the C{exclusive_storage} flag for the cluster.
382

383
    """
384
    value = self._exclusive_storage
385
    assert value is not None
386
    return value
387

    
388
  def IsTemplateSupported(self, templ):
389
    """Is the given disk template supported by the current configuration?
390

391
    """
392
    return (not self.GetExclusiveStorage() or
393
            templ in constants.DTS_EXCL_STORAGE)
394

    
395
  def GetVclusterSettings(self):
396
    """Returns settings for virtual cluster.
397

398
    """
399
    master = self.get(_VCLUSTER_MASTER_KEY)
400
    basedir = self.get(_VCLUSTER_BASEDIR_KEY)
401

    
402
    return (master, basedir)
403

    
404

    
405
def Load(path):
406
  """Loads the passed configuration file.
407

408
  """
409
  global _config # pylint: disable=W0603
410

    
411
  _config = _QaConfig.Load(path)
412

    
413

    
414
def GetConfig():
415
  """Returns the configuration object.
416

417
  """
418
  if _config is None:
419
    raise RuntimeError("Configuration not yet loaded")
420

    
421
  return _config
422

    
423

    
424
def get(name, default=None):
425
  """Wrapper for L{_QaConfig.get}.
426

427
  """
428
  return GetConfig().get(name, default=default)
429

    
430

    
431
class Either:
432
  def __init__(self, tests):
433
    """Initializes this class.
434

435
    @type tests: list or string
436
    @param tests: List of test names
437
    @see: L{TestEnabled} for details
438

439
    """
440
    self.tests = tests
441

    
442

    
443
def _MakeSequence(value):
444
  """Make sequence of single argument.
445

446
  If the single argument is not already a list or tuple, a list with the
447
  argument as a single item is returned.
448

449
  """
450
  if isinstance(value, (list, tuple)):
451
    return value
452
  else:
453
    return [value]
454

    
455

    
456
def _TestEnabledInner(check_fn, names, fn):
457
  """Evaluate test conditions.
458

459
  @type check_fn: callable
460
  @param check_fn: Callback to check whether a test is enabled
461
  @type names: sequence or string
462
  @param names: Test name(s)
463
  @type fn: callable
464
  @param fn: Aggregation function
465
  @rtype: bool
466
  @return: Whether test is enabled
467

468
  """
469
  names = _MakeSequence(names)
470

    
471
  result = []
472

    
473
  for name in names:
474
    if isinstance(name, Either):
475
      value = _TestEnabledInner(check_fn, name.tests, compat.any)
476
    elif isinstance(name, (list, tuple)):
477
      value = _TestEnabledInner(check_fn, name, compat.all)
478
    elif callable(name):
479
      value = name()
480
    else:
481
      value = check_fn(name)
482

    
483
    result.append(value)
484

    
485
  return fn(result)
486

    
487

    
488
def TestEnabled(tests, _cfg=None):
489
  """Returns True if the given tests are enabled.
490

491
  @param tests: A single test as a string, or a list of tests to check; can
492
    contain L{Either} for OR conditions, AND is default
493

494
  """
495
  if _cfg is None:
496
    cfg = GetConfig()
497
  else:
498
    cfg = _cfg
499

    
500
  # Get settings for all tests
501
  cfg_tests = cfg.get("tests", {})
502

    
503
  # Get default setting
504
  default = cfg_tests.get("default", True)
505

    
506
  return _TestEnabledInner(lambda name: cfg_tests.get(name, default),
507
                           tests, compat.all)
508

    
509

    
510
def GetInstanceCheckScript(*args):
511
  """Wrapper for L{_QaConfig.GetInstanceCheckScript}.
512

513
  """
514
  return GetConfig().GetInstanceCheckScript(*args)
515

    
516

    
517
def GetEnabledHypervisors(*args):
518
  """Wrapper for L{_QaConfig.GetEnabledHypervisors}.
519

520
  """
521
  return GetConfig().GetEnabledHypervisors(*args)
522

    
523

    
524
def GetDefaultHypervisor(*args):
525
  """Wrapper for L{_QaConfig.GetDefaultHypervisor}.
526

527
  """
528
  return GetConfig().GetDefaultHypervisor(*args)
529

    
530

    
531
def GetMasterNode():
532
  """Wrapper for L{_QaConfig.GetMasterNode}.
533

534
  """
535
  return GetConfig().GetMasterNode()
536

    
537

    
538
def AcquireInstance(_cfg=None):
539
  """Returns an instance which isn't in use.
540

541
  """
542
  if _cfg is None:
543
    cfg = GetConfig()
544
  else:
545
    cfg = _cfg
546

    
547
  # Filter out unwanted instances
548
  instances = filter(lambda inst: not inst.used, cfg["instances"])
549

    
550
  if not instances:
551
    raise qa_error.OutOfInstancesError("No instances left")
552

    
553
  instance = instances[0]
554
  instance.Use()
555

    
556
  return instance
557

    
558

    
559
def SetExclusiveStorage(value):
560
  """Wrapper for L{_QaConfig.SetExclusiveStorage}.
561

562
  """
563
  return GetConfig().SetExclusiveStorage(value)
564

    
565

    
566
def GetExclusiveStorage():
567
  """Wrapper for L{_QaConfig.GetExclusiveStorage}.
568

569
  """
570
  return GetConfig().GetExclusiveStorage()
571

    
572

    
573
def IsTemplateSupported(templ):
574
  """Wrapper for L{_QaConfig.GetExclusiveStorage}.
575

576
  """
577
  return GetConfig().IsTemplateSupported(templ)
578

    
579

    
580
def _NodeSortKey(node):
581
  """Returns sort key for a node.
582

583
  @type node: L{_QaNode}
584

585
  """
586
  return (node.use_count, utils.NiceSortKey(node.primary))
587

    
588

    
589
def AcquireNode(exclude=None, _cfg=None):
590
  """Returns the least used node.
591

592
  """
593
  if _cfg is None:
594
    cfg = GetConfig()
595
  else:
596
    cfg = _cfg
597

    
598
  master = cfg.GetMasterNode()
599

    
600
  # Filter out unwanted nodes
601
  # TODO: Maybe combine filters
602
  if exclude is None:
603
    nodes = cfg["nodes"][:]
604
  elif isinstance(exclude, (list, tuple)):
605
    nodes = filter(lambda node: node not in exclude, cfg["nodes"])
606
  else:
607
    nodes = filter(lambda node: node != exclude, cfg["nodes"])
608

    
609
  nodes = filter(lambda node: node.added or node == master, nodes)
610

    
611
  if not nodes:
612
    raise qa_error.OutOfNodesError("No nodes left")
613

    
614
  # Return node with least number of uses
615
  return sorted(nodes, key=_NodeSortKey)[0].Use()
616

    
617

    
618
def AcquireManyNodes(num, exclude=None):
619
  """Return the least used nodes.
620

621
  @type num: int
622
  @param num: Number of nodes; can be 0.
623
  @type exclude: list of nodes or C{None}
624
  @param exclude: nodes to be excluded from the choice
625
  @rtype: list of nodes
626
  @return: C{num} different nodes
627

628
  """
629
  nodes = []
630
  if exclude is None:
631
    exclude = []
632
  elif isinstance(exclude, (list, tuple)):
633
    # Don't modify the incoming argument
634
    exclude = list(exclude)
635
  else:
636
    exclude = [exclude]
637

    
638
  try:
639
    for _ in range(0, num):
640
      n = AcquireNode(exclude=exclude)
641
      nodes.append(n)
642
      exclude.append(n)
643
  except qa_error.OutOfNodesError:
644
    ReleaseManyNodes(nodes)
645
    raise
646
  return nodes
647

    
648

    
649
def ReleaseManyNodes(nodes):
650
  for node in nodes:
651
    node.Release()
652

    
653

    
654
def GetVclusterSettings():
655
  """Wrapper for L{_QaConfig.GetVclusterSettings}.
656

657
  """
658
  return GetConfig().GetVclusterSettings()
659

    
660

    
661
def UseVirtualCluster(_cfg=None):
662
  """Returns whether a virtual cluster is used.
663

664
  @rtype: bool
665

666
  """
667
  if _cfg is None:
668
    cfg = GetConfig()
669
  else:
670
    cfg = _cfg
671

    
672
  (master, _) = cfg.GetVclusterSettings()
673

    
674
  return bool(master)
675

    
676

    
677
@ht.WithDesc("No virtual cluster")
678
def NoVirtualCluster():
679
  """Used to disable tests for virtual clusters.
680

681
  """
682
  return not UseVirtualCluster()