Statistics
| Branch: | Tag: | Revision:

root / qa / ganeti-qa.py @ aeefb4cd

History | View | Annotate | Download (25.4 kB)

1
#!/usr/bin/python -u
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Script for doing QA on Ganeti.
23

24
"""
25

    
26
# pylint: disable=C0103
27
# due to invalid name
28

    
29
import sys
30
import datetime
31
import optparse
32

    
33
import qa_cluster
34
import qa_config
35
import qa_daemon
36
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
44
import qa_rapi
45
import qa_tags
46
import qa_utils
47

    
48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

    
52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

    
55

    
56
def _FormatHeader(line, end=72):
57
  """Fill a line up to the end column.
58

59
  """
60
  line = "---- " + line + " "
61
  line += "-" * (end - len(line))
62
  line = line.rstrip()
63
  return line
64

    
65

    
66
def _DescriptionOf(fn):
67
  """Computes the description of an item.
68

69
  """
70
  if fn.__doc__:
71
    desc = fn.__doc__.splitlines()[0].strip()
72
  else:
73
    desc = "%r" % fn
74

    
75
  return desc.rstrip(".")
76

    
77

    
78
def RunTest(fn, *args, **kwargs):
79
  """Runs a test after printing a header.
80

81
  """
82

    
83
  tstart = datetime.datetime.now()
84

    
85
  desc = _DescriptionOf(fn)
86

    
87
  print
88
  print _FormatHeader("%s start %s" % (tstart, desc))
89

    
90
  try:
91
    retval = fn(*args, **kwargs)
92
    return retval
93
  finally:
94
    tstop = datetime.datetime.now()
95
    tdelta = tstop - tstart
96
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
97

    
98

    
99
def RunTestIf(testnames, fn, *args, **kwargs):
100
  """Runs a test conditionally.
101

102
  @param testnames: either a single test name in the configuration
103
      file, or a list of testnames (which will be AND-ed together)
104

105
  """
106
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
108
  else:
109
    tstart = datetime.datetime.now()
110
    desc = _DescriptionOf(fn)
111
    # TODO: Formatting test names when non-string names are involved
112
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
113
                        (tstart, desc, testnames))
114

    
115

    
116
def RunEnvTests():
117
  """Run several environment tests.
118

119
  """
120
  RunTestIf("env", qa_env.TestSshConnection)
121
  RunTestIf("env", qa_env.TestIcmpPing)
122
  RunTestIf("env", qa_env.TestGanetiCommands)
123

    
124

    
125
def SetupCluster(rapi_user, rapi_secret):
126
  """Initializes the cluster.
127

128
  @param rapi_user: Login user for RAPI
129
  @param rapi_secret: Login secret for RAPI
130

131
  """
132
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
133
            rapi_user, rapi_secret)
134
  if not qa_config.TestEnabled("create-cluster"):
135
    # If the cluster is already in place, we assume that exclusive-storage is
136
    # already set according to the configuration
137
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
138

    
139
  # Test on empty cluster
140
  RunTestIf("node-list", qa_node.TestNodeList)
141
  RunTestIf("instance-list", qa_instance.TestInstanceList)
142
  RunTestIf("job-list", qa_job.TestJobList)
143

    
144
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
145
  if not qa_config.TestEnabled("create-cluster"):
146
    # consider the nodes are already there
147
    qa_node.MarkNodeAddedAll()
148

    
149
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
150

    
151
  # enable the watcher (unconditionally)
152
  RunTest(qa_daemon.TestResumeWatcher)
153

    
154
  RunTestIf("node-list", qa_node.TestNodeList)
155

    
156
  # Test listing fields
157
  RunTestIf("node-list", qa_node.TestNodeListFields)
158
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
159
  RunTestIf("job-list", qa_job.TestJobListFields)
160
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
161

    
162
  RunTestIf("node-info", qa_node.TestNodeInfo)
163

    
164

    
165
def RunClusterTests():
166
  """Runs tests related to gnt-cluster.
167

168
  """
169
  for test, fn in [
170
    ("create-cluster", qa_cluster.TestClusterInitDisk),
171
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
172
    ("cluster-verify", qa_cluster.TestClusterVerify),
173
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
174
    # TODO: add more cluster modify tests
175
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
176
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
177
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
178
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
179
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
180
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
181
    ("cluster-rename", qa_cluster.TestClusterRename),
182
    ("cluster-info", qa_cluster.TestClusterVersion),
183
    ("cluster-info", qa_cluster.TestClusterInfo),
184
    ("cluster-info", qa_cluster.TestClusterGetmaster),
185
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
186
    (["cluster-copyfile", qa_config.NoVirtualCluster],
187
     qa_cluster.TestClusterCopyfile),
188
    ("cluster-command", qa_cluster.TestClusterCommand),
189
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
190
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
191
    ("cluster-master-failover",
192
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
193
    (["cluster-oob", qa_config.NoVirtualCluster],
194
     qa_cluster.TestClusterOob),
195
    (qa_rapi.Enabled, qa_rapi.TestVersion),
196
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
197
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
198
    ]:
199
    RunTestIf(test, fn)
200

    
201

    
202
def RunRepairDiskSizes():
203
  """Run the repair disk-sizes test.
204

205
  """
206
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
207

    
208

    
209
def RunOsTests():
210
  """Runs all tests related to gnt-os.
211

212
  """
213
  os_enabled = ["os", qa_config.NoVirtualCluster]
214

    
215
  if qa_config.TestEnabled(qa_rapi.Enabled):
216
    rapi_getos = qa_rapi.GetOperatingSystems
217
  else:
218
    rapi_getos = None
219

    
220
  for fn in [
221
    qa_os.TestOsList,
222
    qa_os.TestOsDiagnose,
223
    ]:
224
    RunTestIf(os_enabled, fn)
225

    
226
  for fn in [
227
    qa_os.TestOsValid,
228
    qa_os.TestOsInvalid,
229
    qa_os.TestOsPartiallyValid,
230
    ]:
231
    RunTestIf(os_enabled, fn, rapi_getos)
232

    
233
  for fn in [
234
    qa_os.TestOsModifyValid,
235
    qa_os.TestOsModifyInvalid,
236
    qa_os.TestOsStatesNonExisting,
237
    ]:
238
    RunTestIf(os_enabled, fn)
239

    
240

    
241
def RunCommonInstanceTests(instance):
242
  """Runs a few tests that are common to all disk types.
243

244
  """
245
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
246
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
247
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
248
  RunTestIf(["instance-shutdown", "instance-modify"],
249
            qa_instance.TestInstanceStoppedModify, instance)
250
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
251

    
252
  # Test shutdown/start via RAPI
253
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
254
            qa_rapi.TestRapiInstanceShutdown, instance)
255
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
256
            qa_rapi.TestRapiInstanceStartup, instance)
257

    
258
  RunTestIf("instance-list", qa_instance.TestInstanceList)
259

    
260
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
261

    
262
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
263
  RunTestIf(["instance-modify", qa_rapi.Enabled],
264
            qa_rapi.TestRapiInstanceModify, instance)
265

    
266
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
267
  RunTestIf(["instance-console", qa_rapi.Enabled],
268
            qa_rapi.TestRapiInstanceConsole, instance)
269

    
270
  RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
271
            instance)
272
  DOWN_TESTS = qa_config.Either([
273
    "instance-reinstall",
274
    "instance-rename",
275
    "instance-grow-disk",
276
    ])
277

    
278
  # shutdown instance for any 'down' tests
279
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)
280

    
281
  # now run the 'down' state tests
282
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
283
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
284
            qa_rapi.TestRapiInstanceReinstall, instance)
285

    
286
  if qa_config.TestEnabled("instance-rename"):
287
    tgt_instance = qa_config.AcquireInstance()
288
    try:
289
      rename_source = instance.name
290
      rename_target = tgt_instance.name
291
      # perform instance rename to the same name
292
      RunTest(qa_instance.TestInstanceRenameAndBack,
293
              rename_source, rename_source)
294
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
295
                rename_source, rename_source)
296
      if rename_target is not None:
297
        # perform instance rename to a different name, if we have one configured
298
        RunTest(qa_instance.TestInstanceRenameAndBack,
299
                rename_source, rename_target)
300
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
301
                  rename_source, rename_target)
302
    finally:
303
      tgt_instance.Release()
304

    
305
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)
306

    
307
  # and now start the instance again
308
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)
309

    
310
  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
311

    
312
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
313

    
314
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
315

    
316
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
317

    
318
  # Lists instances, too
319
  RunTestIf("node-list", qa_node.TestNodeList)
320

    
321
  # Some jobs have been run, let's test listing them
322
  RunTestIf("job-list", qa_job.TestJobList)
323

    
324

    
325
def RunCommonNodeTests():
326
  """Run a few common node tests.
327

328
  """
329
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
330
  RunTestIf("node-storage", qa_node.TestNodeStorage)
331
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
332

    
333

    
334
def RunGroupListTests():
335
  """Run tests for listing node groups.
336

337
  """
338
  RunTestIf("group-list", qa_group.TestGroupList)
339
  RunTestIf("group-list", qa_group.TestGroupListFields)
340

    
341

    
342
def RunNetworkTests():
343
  """Run tests for network management.
344

345
  """
346
  RunTestIf("network", qa_network.TestNetworkAddRemove)
347
  RunTestIf("network", qa_network.TestNetworkConnect)
348

    
349

    
350
def RunGroupRwTests():
351
  """Run tests for adding/removing/renaming groups.
352

353
  """
354
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
355
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
356
  RunTestIf("group-rwops", qa_group.TestGroupModify)
357
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
358
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
359
            qa_group.GetDefaultGroup())
360

    
361

    
362
def RunExportImportTests(instance, inodes):
363
  """Tries to export and import the instance.
364

365
  @type inodes: list of nodes
366
  @param inodes: current nodes of the instance
367

368
  """
369
  # FIXME: export explicitly bails out on file based storage. other non-lvm
370
  # based storage types are untested, though. Also note that import could still
371
  # work, but is deeply embedded into the "export" case.
372
  if (qa_config.TestEnabled("instance-export") and
373
      instance.disk_template != constants.DT_FILE):
374
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)
375

    
376
    pnode = inodes[0]
377
    expnode = qa_config.AcquireNode(exclude=pnode)
378
    try:
379
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
380

    
381
      RunTest(qa_instance.TestBackupList, expnode)
382

    
383
      if qa_config.TestEnabled("instance-import"):
384
        newinst = qa_config.AcquireInstance()
385
        try:
386
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
387
                  expnode, name)
388
          # Check if starting the instance works
389
          RunTest(qa_instance.TestInstanceStartup, newinst)
390
          RunTest(qa_instance.TestInstanceRemove, newinst)
391
        finally:
392
          newinst.Release()
393
    finally:
394
      expnode.Release()
395

    
396
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
397
  # See Issue 414.
398
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
399
      and instance.disk_template != constants.DT_FILE):
400
    newinst = qa_config.AcquireInstance()
401
    try:
402
      tnode = qa_config.AcquireNode(exclude=inodes)
403
      try:
404
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
405
                inodes, tnode)
406
      finally:
407
        tnode.Release()
408
    finally:
409
      newinst.Release()
410

    
411

    
412
def RunDaemonTests(instance):
413
  """Test the ganeti-watcher script.
414

415
  """
416
  RunTest(qa_daemon.TestPauseWatcher)
417

    
418
  RunTestIf("instance-automatic-restart",
419
            qa_daemon.TestInstanceAutomaticRestart, instance)
420
  RunTestIf("instance-consecutive-failures",
421
            qa_daemon.TestInstanceConsecutiveFailures, instance)
422

    
423
  RunTest(qa_daemon.TestResumeWatcher)
424

    
425

    
426
def RunHardwareFailureTests(instance, inodes):
427
  """Test cluster internal hardware failure recovery.
428

429
  """
430
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
431
  RunTestIf(["instance-failover", qa_rapi.Enabled],
432
            qa_rapi.TestRapiInstanceFailover, instance)
433

    
434
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
435
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
436
            qa_rapi.TestRapiInstanceMigrate, instance)
437

    
438
  if qa_config.TestEnabled("instance-replace-disks"):
439
    # We just need alternative secondary nodes, hence "- 1"
440
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
441
    try:
442
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
443
      RunTest(qa_instance.TestReplaceDisks,
444
              instance, inodes, othernodes)
445
    finally:
446
      qa_config.ReleaseManyNodes(othernodes)
447
    del othernodes
448

    
449
  if qa_config.TestEnabled("instance-recreate-disks"):
450
    try:
451
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
452
      othernodes = acquirednodes
453
    except qa_error.OutOfNodesError:
454
      if len(inodes) > 1:
455
        # If the cluster is not big enough, let's reuse some of the nodes, but
456
        # with different roles. In this way, we can test a DRBD instance even on
457
        # a 3-node cluster.
458
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
459
        othernodes = acquirednodes + inodes[:-1]
460
      else:
461
        raise
462
    try:
463
      RunTest(qa_instance.TestRecreateDisks,
464
              instance, inodes, othernodes)
465
    finally:
466
      qa_config.ReleaseManyNodes(acquirednodes)
467

    
468
  if len(inodes) >= 2:
469
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
470
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
471

    
472

    
473
def RunExclusiveStorageTests():
474
  """Test exclusive storage."""
475
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
476
    return
477

    
478
  node = qa_config.AcquireNode()
479
  try:
480
    old_es = qa_cluster.TestSetExclStorCluster(False)
481
    qa_node.TestExclStorSingleNode(node)
482

    
483
    qa_cluster.TestSetExclStorCluster(True)
484
    qa_cluster.TestExclStorSharedPv(node)
485

    
486
    if qa_config.TestEnabled("instance-add-plain-disk"):
487
      # Make sure that the cluster doesn't have any pre-existing problem
488
      qa_cluster.AssertClusterVerify()
489

    
490
      # Create and allocate instances
491
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
492
      try:
493
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
494
        try:
495
          # cluster-verify checks that disks are allocated correctly
496
          qa_cluster.AssertClusterVerify()
497

    
498
          # Remove instances
499
          qa_instance.TestInstanceRemove(instance2)
500
          qa_instance.TestInstanceRemove(instance1)
501
        finally:
502
          instance2.Release()
503
      finally:
504
        instance1.Release()
505

    
506
    if qa_config.TestEnabled("instance-add-drbd-disk"):
507
      snode = qa_config.AcquireNode()
508
      try:
509
        qa_cluster.TestSetExclStorCluster(False)
510
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
511
        try:
512
          qa_cluster.TestSetExclStorCluster(True)
513
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
514
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
515
          qa_instance.TestInstanceRemove(instance)
516
        finally:
517
          instance.Release()
518
      finally:
519
        snode.Release()
520
    qa_cluster.TestSetExclStorCluster(old_es)
521
  finally:
522
    node.Release()
523

    
524

    
525
def _BuildSpecDict(par, mn, st, mx):
526
  return {par: {"min": mn, "std": st, "max": mx}}
527

    
528

    
529
def TestIPolicyPlainInstance():
530
  """Test instance policy interaction with instances"""
531
  params = ["mem-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
532
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
533
    print "Template %s not supported" % constants.DT_PLAIN
534
    return
535

    
536
  # This test assumes that the group policy is empty
537
  (_, old_specs) = qa_cluster.TestClusterSetISpecs({})
538
  node = qa_config.AcquireNode()
539
  try:
540
    # Log of policy changes, list of tuples: (change, policy_violated)
541
    history = []
542
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
543
    try:
544
      policyerror = [constants.CV_EINSTANCEPOLICY]
545
      for par in params:
546
        qa_cluster.AssertClusterVerify()
547
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
548
        # Some specs must be multiple of 4
549
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
550
        history.append((new_spec, True))
551
        qa_cluster.TestClusterSetISpecs(new_spec)
552
        qa_cluster.AssertClusterVerify(warnings=policyerror)
553
        if iminval > 0:
554
          # Some specs must be multiple of 4
555
          if iminval >= 4:
556
            upper = iminval - 4
557
          else:
558
            upper = iminval - 1
559
          new_spec = _BuildSpecDict(par, 0, upper, upper)
560
          history.append((new_spec, True))
561
          qa_cluster.TestClusterSetISpecs(new_spec)
562
          qa_cluster.AssertClusterVerify(warnings=policyerror)
563
        qa_cluster.TestClusterSetISpecs(old_specs)
564
        history.append((old_specs, False))
565
      qa_instance.TestInstanceRemove(instance)
566
    finally:
567
      instance.Release()
568

    
569
    # Now we replay the same policy changes, and we expect that the instance
570
    # cannot be created for the cases where we had a policy violation above
571
    for (change, failed) in history:
572
      qa_cluster.TestClusterSetISpecs(change)
573
      if failed:
574
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
575
      # Instance creation with no policy violation has been tested already
576
  finally:
577
    node.Release()
578

    
579

    
580
def RunInstanceTests():
581
  """Create and exercise instances."""
582
  instance_tests = [
583
    ("instance-add-plain-disk", constants.DT_PLAIN,
584
     qa_instance.TestInstanceAddWithPlainDisk, 1),
585
    ("instance-add-drbd-disk", constants.DT_DRBD8,
586
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
587
    ("instance-add-diskless", constants.DT_DISKLESS,
588
     qa_instance.TestInstanceAddDiskless, 1),
589
    ("instance-add-file", constants.DT_FILE,
590
     qa_instance.TestInstanceAddFile, 1)
591
    ]
592

    
593
  for (test_name, templ, create_fun, num_nodes) in instance_tests:
594
    if (qa_config.TestEnabled(test_name) and
595
        qa_config.IsTemplateSupported(templ)):
596
      inodes = qa_config.AcquireManyNodes(num_nodes)
597
      try:
598
        instance = RunTest(create_fun, inodes)
599
        try:
600
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
601
          RunDaemonTests(instance)
602
          for node in inodes:
603
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
604
          if len(inodes) > 1:
605
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
606
                      constants.INITIAL_NODE_GROUP_NAME,
607
                      inodes[0].primary, inodes[1].primary)
608
          if qa_config.TestEnabled("instance-convert-disk"):
609
            RunTest(qa_instance.TestInstanceShutdown, instance)
610
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
611
                    instance, inodes)
612
            RunTest(qa_instance.TestInstanceStartup, instance)
613
          RunCommonInstanceTests(instance)
614
          if qa_config.TestEnabled("instance-modify-primary"):
615
            othernode = qa_config.AcquireNode()
616
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
617
                    instance, inodes[0], othernode)
618
            othernode.Release()
619
          RunGroupListTests()
620
          RunExportImportTests(instance, inodes)
621
          RunHardwareFailureTests(instance, inodes)
622
          RunRepairDiskSizes()
623
          RunTest(qa_instance.TestInstanceRemove, instance)
624
        finally:
625
          instance.Release()
626
        del instance
627
      finally:
628
        qa_config.ReleaseManyNodes(inodes)
629
      qa_cluster.AssertClusterVerify()
630

    
631

    
632
def RunQa():
633
  """Main QA body.
634

635
  """
636
  rapi_user = "ganeti-qa"
637
  rapi_secret = utils.GenerateSecret()
638

    
639
  RunEnvTests()
640
  SetupCluster(rapi_user, rapi_secret)
641

    
642
  # Load RAPI certificate
643
  qa_rapi.Setup(rapi_user, rapi_secret)
644

    
645
  RunClusterTests()
646
  RunOsTests()
647

    
648
  RunTestIf("tags", qa_tags.TestClusterTags)
649

    
650
  RunCommonNodeTests()
651
  RunGroupListTests()
652
  RunGroupRwTests()
653
  RunNetworkTests()
654

    
655
  # The master shouldn't be readded or put offline; "delay" needs a non-master
656
  # node to test
657
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
658
  try:
659
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
660
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
661
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
662
  finally:
663
    pnode.Release()
664

    
665
  # Make sure the cluster is clean before running instance tests
666
  qa_cluster.AssertClusterVerify()
667

    
668
  pnode = qa_config.AcquireNode()
669
  try:
670
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
671

    
672
    if qa_rapi.Enabled():
673
      RunTest(qa_rapi.TestNode, pnode)
674

    
675
      if qa_config.TestEnabled("instance-add-plain-disk"):
676
        for use_client in [True, False]:
677
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
678
                                  use_client)
679
          try:
680
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
681
              RunCommonInstanceTests(rapi_instance)
682
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
683
          finally:
684
            rapi_instance.Release()
685
          del rapi_instance
686

    
687
  finally:
688
    pnode.Release()
689

    
690
  config_list = [
691
    ("default-instance-tests", lambda: None, lambda _: None),
692
    ("exclusive-storage-instance-tests",
693
     lambda: qa_cluster.TestSetExclStorCluster(True),
694
     qa_cluster.TestSetExclStorCluster),
695
  ]
696
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
697
    if qa_config.TestEnabled(conf_name):
698
      oldconf = setup_conf_f()
699
      RunInstanceTests()
700
      restore_conf_f(oldconf)
701

    
702
  pnode = qa_config.AcquireNode()
703
  try:
704
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
705
      for shutdown in [False, True]:
706
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
707
        try:
708
          expnode = qa_config.AcquireNode(exclude=pnode)
709
          try:
710
            if shutdown:
711
              # Stop instance before exporting and removing it
712
              RunTest(qa_instance.TestInstanceShutdown, instance)
713
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
714
            RunTest(qa_instance.TestBackupList, expnode)
715
          finally:
716
            expnode.Release()
717
        finally:
718
          instance.Release()
719
        del expnode
720
        del instance
721
      qa_cluster.AssertClusterVerify()
722

    
723
  finally:
724
    pnode.Release()
725

    
726
  RunExclusiveStorageTests()
727
  RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
728
            TestIPolicyPlainInstance)
729

    
730
  RunTestIf(
731
    "instance-add-restricted-by-disktemplates",
732
    qa_instance.TestInstanceCreationRestrictedByDiskTemplates)
733

    
734
  # Test removing instance with offline drbd secondary
735
  if qa_config.TestEnabled(["instance-remove-drbd-offline",
736
                            "instance-add-drbd-disk"]):
737
    # Make sure the master is not put offline
738
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
739
    try:
740
      pnode = qa_config.AcquireNode(exclude=snode)
741
      try:
742
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
743
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
744
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
745
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
746
                set_offline, set_online)
747
      finally:
748
        pnode.Release()
749
    finally:
750
      snode.Release()
751
    qa_cluster.AssertClusterVerify()
752

    
753
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
754

    
755
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
756

    
757

    
758
@UsesRapiClient
759
def main():
760
  """Main program.
761

762
  """
763
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
764
  parser.add_option("--yes-do-it", dest="yes_do_it",
765
                    action="store_true",
766
                    help="Really execute the tests")
767
  (opts, args) = parser.parse_args()
768

    
769
  if len(args) == 1:
770
    (config_file, ) = args
771
  else:
772
    parser.error("Wrong number of arguments.")
773

    
774
  if not opts.yes_do_it:
775
    print ("Executing this script irreversibly destroys any Ganeti\n"
776
           "configuration on all nodes involved. If you really want\n"
777
           "to start testing, supply the --yes-do-it option.")
778
    sys.exit(1)
779

    
780
  qa_config.Load(config_file)
781

    
782
  primary = qa_config.GetMasterNode().primary
783
  qa_utils.StartMultiplexer(primary)
784
  print ("SSH command for primary node: %s" %
785
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
786
  print ("SSH command for other nodes: %s" %
787
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
788
  try:
789
    RunQa()
790
  finally:
791
    qa_utils.CloseMultiplexers()
792

    
793
if __name__ == "__main__":
794
  main()