Statistics
| Branch: | Tag: | Revision:

root / qa / ganeti-qa.py @ 2ef21e6e

History | View | Annotate | Download (25.3 kB)

1
#!/usr/bin/python -u
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Script for doing QA on Ganeti.
23

24
"""
25

    
26
# pylint: disable=C0103
27
# due to invalid name
28

    
29
import sys
30
import datetime
31
import optparse
32

    
33
import qa_cluster
34
import qa_config
35
import qa_daemon
36
import qa_env
37
import qa_error
38
import qa_group
39
import qa_instance
40
import qa_network
41
import qa_node
42
import qa_os
43
import qa_job
44
import qa_rapi
45
import qa_tags
46
import qa_utils
47

    
48
from ganeti import utils
49
from ganeti import rapi # pylint: disable=W0611
50
from ganeti import constants
51

    
52
import ganeti.rapi.client # pylint: disable=W0611
53
from ganeti.rapi.client import UsesRapiClient
54

    
55

    
56
def _FormatHeader(line, end=72):
57
  """Fill a line up to the end column.
58

59
  """
60
  line = "---- " + line + " "
61
  line += "-" * (end - len(line))
62
  line = line.rstrip()
63
  return line
64

    
65

    
66
def _DescriptionOf(fn):
67
  """Computes the description of an item.
68

69
  """
70
  if fn.__doc__:
71
    desc = fn.__doc__.splitlines()[0].strip()
72
  else:
73
    desc = "%r" % fn
74

    
75
  return desc.rstrip(".")
76

    
77

    
78
def RunTest(fn, *args, **kwargs):
79
  """Runs a test after printing a header.
80

81
  """
82

    
83
  tstart = datetime.datetime.now()
84

    
85
  desc = _DescriptionOf(fn)
86

    
87
  print
88
  print _FormatHeader("%s start %s" % (tstart, desc))
89

    
90
  try:
91
    retval = fn(*args, **kwargs)
92
    return retval
93
  finally:
94
    tstop = datetime.datetime.now()
95
    tdelta = tstop - tstart
96
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
97

    
98

    
99
def RunTestIf(testnames, fn, *args, **kwargs):
100
  """Runs a test conditionally.
101

102
  @param testnames: either a single test name in the configuration
103
      file, or a list of testnames (which will be AND-ed together)
104

105
  """
106
  if qa_config.TestEnabled(testnames):
107
    RunTest(fn, *args, **kwargs)
108
  else:
109
    tstart = datetime.datetime.now()
110
    desc = _DescriptionOf(fn)
111
    # TODO: Formatting test names when non-string names are involved
112
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
113
                        (tstart, desc, testnames))
114

    
115

    
116
def RunEnvTests():
117
  """Run several environment tests.
118

119
  """
120
  RunTestIf("env", qa_env.TestSshConnection)
121
  RunTestIf("env", qa_env.TestIcmpPing)
122
  RunTestIf("env", qa_env.TestGanetiCommands)
123

    
124

    
125
def SetupCluster(rapi_user, rapi_secret):
126
  """Initializes the cluster.
127

128
  @param rapi_user: Login user for RAPI
129
  @param rapi_secret: Login secret for RAPI
130

131
  """
132
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
133
            rapi_user, rapi_secret)
134
  if not qa_config.TestEnabled("create-cluster"):
135
    # If the cluster is already in place, we assume that exclusive-storage is
136
    # already set according to the configuration
137
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
138

    
139
  # Test on empty cluster
140
  RunTestIf("node-list", qa_node.TestNodeList)
141
  RunTestIf("instance-list", qa_instance.TestInstanceList)
142
  RunTestIf("job-list", qa_job.TestJobList)
143

    
144
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
145
  if not qa_config.TestEnabled("create-cluster"):
146
    # consider the nodes are already there
147
    qa_node.MarkNodeAddedAll()
148

    
149
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
150

    
151
  # enable the watcher (unconditionally)
152
  RunTest(qa_daemon.TestResumeWatcher)
153

    
154
  RunTestIf("node-list", qa_node.TestNodeList)
155

    
156
  # Test listing fields
157
  RunTestIf("node-list", qa_node.TestNodeListFields)
158
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
159
  RunTestIf("job-list", qa_job.TestJobListFields)
160
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
161

    
162
  RunTestIf("node-info", qa_node.TestNodeInfo)
163

    
164

    
165
def RunClusterTests():
166
  """Runs tests related to gnt-cluster.
167

168
  """
169
  for test, fn in [
170
    ("create-cluster", qa_cluster.TestClusterInitDisk),
171
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
172
    ("cluster-verify", qa_cluster.TestClusterVerify),
173
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
174
    # TODO: add more cluster modify tests
175
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
176
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
177
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
178
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
179
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
180
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
181
    ("cluster-rename", qa_cluster.TestClusterRename),
182
    ("cluster-info", qa_cluster.TestClusterVersion),
183
    ("cluster-info", qa_cluster.TestClusterInfo),
184
    ("cluster-info", qa_cluster.TestClusterGetmaster),
185
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
186
    (["cluster-copyfile", qa_config.NoVirtualCluster],
187
     qa_cluster.TestClusterCopyfile),
188
    ("cluster-command", qa_cluster.TestClusterCommand),
189
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
190
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
191
    ("cluster-master-failover",
192
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
193
    (["cluster-oob", qa_config.NoVirtualCluster],
194
     qa_cluster.TestClusterOob),
195
    (qa_rapi.Enabled, qa_rapi.TestVersion),
196
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
197
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
198
    ]:
199
    RunTestIf(test, fn)
200

    
201

    
202
def RunRepairDiskSizes():
203
  """Run the repair disk-sizes test.
204

205
  """
206
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
207

    
208

    
209
def RunOsTests():
210
  """Runs all tests related to gnt-os.
211

212
  """
213
  os_enabled = ["os", qa_config.NoVirtualCluster]
214

    
215
  if qa_config.TestEnabled(qa_rapi.Enabled):
216
    rapi_getos = qa_rapi.GetOperatingSystems
217
  else:
218
    rapi_getos = None
219

    
220
  for fn in [
221
    qa_os.TestOsList,
222
    qa_os.TestOsDiagnose,
223
    ]:
224
    RunTestIf(os_enabled, fn)
225

    
226
  for fn in [
227
    qa_os.TestOsValid,
228
    qa_os.TestOsInvalid,
229
    qa_os.TestOsPartiallyValid,
230
    ]:
231
    RunTestIf(os_enabled, fn, rapi_getos)
232

    
233
  for fn in [
234
    qa_os.TestOsModifyValid,
235
    qa_os.TestOsModifyInvalid,
236
    qa_os.TestOsStatesNonExisting,
237
    ]:
238
    RunTestIf(os_enabled, fn)
239

    
240

    
241
def RunCommonInstanceTests(instance):
242
  """Runs a few tests that are common to all disk types.
243

244
  """
245
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
246
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
247
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
248
  RunTestIf(["instance-shutdown", "instance-modify"],
249
            qa_instance.TestInstanceStoppedModify, instance)
250
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
251

    
252
  # Test shutdown/start via RAPI
253
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
254
            qa_rapi.TestRapiInstanceShutdown, instance)
255
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
256
            qa_rapi.TestRapiInstanceStartup, instance)
257

    
258
  RunTestIf("instance-list", qa_instance.TestInstanceList)
259

    
260
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
261

    
262
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
263
  RunTestIf(["instance-modify", qa_rapi.Enabled],
264
            qa_rapi.TestRapiInstanceModify, instance)
265

    
266
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
267
  RunTestIf(["instance-console", qa_rapi.Enabled],
268
            qa_rapi.TestRapiInstanceConsole, instance)
269

    
270
  DOWN_TESTS = qa_config.Either([
271
    "instance-reinstall",
272
    "instance-rename",
273
    "instance-grow-disk",
274
    ])
275

    
276
  # shutdown instance for any 'down' tests
277
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)
278

    
279
  # now run the 'down' state tests
280
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
281
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
282
            qa_rapi.TestRapiInstanceReinstall, instance)
283

    
284
  if qa_config.TestEnabled("instance-rename"):
285
    tgt_instance = qa_config.AcquireInstance()
286
    try:
287
      rename_source = instance.name
288
      rename_target = tgt_instance.name
289
      # perform instance rename to the same name
290
      RunTest(qa_instance.TestInstanceRenameAndBack,
291
              rename_source, rename_source)
292
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
293
                rename_source, rename_source)
294
      if rename_target is not None:
295
        # perform instance rename to a different name, if we have one configured
296
        RunTest(qa_instance.TestInstanceRenameAndBack,
297
                rename_source, rename_target)
298
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
299
                  rename_source, rename_target)
300
    finally:
301
      tgt_instance.Release()
302

    
303
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)
304

    
305
  # and now start the instance again
306
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)
307

    
308
  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
309

    
310
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
311

    
312
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
313

    
314
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
315

    
316
  # Lists instances, too
317
  RunTestIf("node-list", qa_node.TestNodeList)
318

    
319
  # Some jobs have been run, let's test listing them
320
  RunTestIf("job-list", qa_job.TestJobList)
321

    
322

    
323
def RunCommonNodeTests():
324
  """Run a few common node tests.
325

326
  """
327
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
328
  RunTestIf("node-storage", qa_node.TestNodeStorage)
329
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
330

    
331

    
332
def RunGroupListTests():
333
  """Run tests for listing node groups.
334

335
  """
336
  RunTestIf("group-list", qa_group.TestGroupList)
337
  RunTestIf("group-list", qa_group.TestGroupListFields)
338

    
339

    
340
def RunNetworkTests():
341
  """Run tests for network management.
342

343
  """
344
  RunTestIf("network", qa_network.TestNetworkAddRemove)
345
  RunTestIf("network", qa_network.TestNetworkConnect)
346

    
347

    
348
def RunGroupRwTests():
349
  """Run tests for adding/removing/renaming groups.
350

351
  """
352
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
353
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
354
  RunTestIf("group-rwops", qa_group.TestGroupModify)
355
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
356
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
357
            qa_group.GetDefaultGroup())
358

    
359

    
360
def RunExportImportTests(instance, inodes):
361
  """Tries to export and import the instance.
362

363
  @type inodes: list of nodes
364
  @param inodes: current nodes of the instance
365

366
  """
367
  # FIXME: export explicitly bails out on file based storage. other non-lvm
368
  # based storage types are untested, though. Also note that import could still
369
  # work, but is deeply embedded into the "export" case.
370
  if (qa_config.TestEnabled("instance-export") and
371
      instance.disk_template != constants.DT_FILE):
372
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)
373

    
374
    pnode = inodes[0]
375
    expnode = qa_config.AcquireNode(exclude=pnode)
376
    try:
377
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
378

    
379
      RunTest(qa_instance.TestBackupList, expnode)
380

    
381
      if qa_config.TestEnabled("instance-import"):
382
        newinst = qa_config.AcquireInstance()
383
        try:
384
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
385
                  expnode, name)
386
          # Check if starting the instance works
387
          RunTest(qa_instance.TestInstanceStartup, newinst)
388
          RunTest(qa_instance.TestInstanceRemove, newinst)
389
        finally:
390
          newinst.Release()
391
    finally:
392
      expnode.Release()
393

    
394
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
395
  # See Issue 414.
396
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
397
      and instance.disk_template != constants.DT_FILE):
398
    newinst = qa_config.AcquireInstance()
399
    try:
400
      tnode = qa_config.AcquireNode(exclude=inodes)
401
      try:
402
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
403
                inodes, tnode)
404
      finally:
405
        tnode.Release()
406
    finally:
407
      newinst.Release()
408

    
409

    
410
def RunDaemonTests(instance):
411
  """Test the ganeti-watcher script.
412

413
  """
414
  RunTest(qa_daemon.TestPauseWatcher)
415

    
416
  RunTestIf("instance-automatic-restart",
417
            qa_daemon.TestInstanceAutomaticRestart, instance)
418
  RunTestIf("instance-consecutive-failures",
419
            qa_daemon.TestInstanceConsecutiveFailures, instance)
420

    
421
  RunTest(qa_daemon.TestResumeWatcher)
422

    
423

    
424
def RunHardwareFailureTests(instance, inodes):
425
  """Test cluster internal hardware failure recovery.
426

427
  """
428
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
429
  RunTestIf(["instance-failover", qa_rapi.Enabled],
430
            qa_rapi.TestRapiInstanceFailover, instance)
431

    
432
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
433
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
434
            qa_rapi.TestRapiInstanceMigrate, instance)
435

    
436
  if qa_config.TestEnabled("instance-replace-disks"):
437
    # We just need alternative secondary nodes, hence "- 1"
438
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
439
    try:
440
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
441
      RunTest(qa_instance.TestReplaceDisks,
442
              instance, inodes, othernodes)
443
    finally:
444
      qa_config.ReleaseManyNodes(othernodes)
445
    del othernodes
446

    
447
  if qa_config.TestEnabled("instance-recreate-disks"):
448
    try:
449
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
450
      othernodes = acquirednodes
451
    except qa_error.OutOfNodesError:
452
      if len(inodes) > 1:
453
        # If the cluster is not big enough, let's reuse some of the nodes, but
454
        # with different roles. In this way, we can test a DRBD instance even on
455
        # a 3-node cluster.
456
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
457
        othernodes = acquirednodes + inodes[:-1]
458
      else:
459
        raise
460
    try:
461
      RunTest(qa_instance.TestRecreateDisks,
462
              instance, inodes, othernodes)
463
    finally:
464
      qa_config.ReleaseManyNodes(acquirednodes)
465

    
466
  if len(inodes) >= 2:
467
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
468
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
469

    
470

    
471
def RunExclusiveStorageTests():
472
  """Test exclusive storage."""
473
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
474
    return
475

    
476
  node = qa_config.AcquireNode()
477
  try:
478
    old_es = qa_cluster.TestSetExclStorCluster(False)
479
    qa_node.TestExclStorSingleNode(node)
480

    
481
    qa_cluster.TestSetExclStorCluster(True)
482
    qa_cluster.TestExclStorSharedPv(node)
483

    
484
    if qa_config.TestEnabled("instance-add-plain-disk"):
485
      # Make sure that the cluster doesn't have any pre-existing problem
486
      qa_cluster.AssertClusterVerify()
487

    
488
      # Create and allocate instances
489
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
490
      try:
491
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
492
        try:
493
          # cluster-verify checks that disks are allocated correctly
494
          qa_cluster.AssertClusterVerify()
495

    
496
          # Remove instances
497
          qa_instance.TestInstanceRemove(instance2)
498
          qa_instance.TestInstanceRemove(instance1)
499
        finally:
500
          instance2.Release()
501
      finally:
502
        instance1.Release()
503

    
504
    if qa_config.TestEnabled("instance-add-drbd-disk"):
505
      snode = qa_config.AcquireNode()
506
      try:
507
        qa_cluster.TestSetExclStorCluster(False)
508
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
509
        try:
510
          qa_cluster.TestSetExclStorCluster(True)
511
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
512
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
513
          qa_instance.TestInstanceRemove(instance)
514
        finally:
515
          instance.Release()
516
      finally:
517
        snode.Release()
518
    qa_cluster.TestSetExclStorCluster(old_es)
519
  finally:
520
    node.Release()
521

    
522

    
523
def _BuildSpecDict(par, mn, st, mx):
524
  return {par: {"min": mn, "std": st, "max": mx}}
525

    
526

    
527
def TestIPolicyPlainInstance():
528
  """Test instance policy interaction with instances"""
529
  params = ["mem-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
530
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
531
    print "Template %s not supported" % constants.DT_PLAIN
532
    return
533

    
534
  # This test assumes that the group policy is empty
535
  (_, old_specs) = qa_cluster.TestClusterSetISpecs({})
536
  node = qa_config.AcquireNode()
537
  try:
538
    # Log of policy changes, list of tuples: (change, policy_violated)
539
    history = []
540
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
541
    try:
542
      policyerror = [constants.CV_EINSTANCEPOLICY]
543
      for par in params:
544
        qa_cluster.AssertClusterVerify()
545
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
546
        # Some specs must be multiple of 4
547
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
548
        history.append((new_spec, True))
549
        qa_cluster.TestClusterSetISpecs(new_spec)
550
        qa_cluster.AssertClusterVerify(warnings=policyerror)
551
        if iminval > 0:
552
          # Some specs must be multiple of 4
553
          if iminval >= 4:
554
            upper = iminval - 4
555
          else:
556
            upper = iminval - 1
557
          new_spec = _BuildSpecDict(par, 0, upper, upper)
558
          history.append((new_spec, True))
559
          qa_cluster.TestClusterSetISpecs(new_spec)
560
          qa_cluster.AssertClusterVerify(warnings=policyerror)
561
        qa_cluster.TestClusterSetISpecs(old_specs)
562
        history.append((old_specs, False))
563
      qa_instance.TestInstanceRemove(instance)
564
    finally:
565
      instance.Release()
566

    
567
    # Now we replay the same policy changes, and we expect that the instance
568
    # cannot be created for the cases where we had a policy violation above
569
    for (change, failed) in history:
570
      qa_cluster.TestClusterSetISpecs(change)
571
      if failed:
572
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
573
      # Instance creation with no policy violation has been tested already
574
  finally:
575
    node.Release()
576

    
577

    
578
def RunInstanceTests():
579
  """Create and exercise instances."""
580
  instance_tests = [
581
    ("instance-add-plain-disk", constants.DT_PLAIN,
582
     qa_instance.TestInstanceAddWithPlainDisk, 1),
583
    ("instance-add-drbd-disk", constants.DT_DRBD8,
584
     qa_instance.TestInstanceAddWithDrbdDisk, 2),
585
    ("instance-add-diskless", constants.DT_DISKLESS,
586
     qa_instance.TestInstanceAddDiskless, 1),
587
    ("instance-add-file", constants.DT_FILE,
588
     qa_instance.TestInstanceAddFile, 1)
589
    ]
590

    
591
  for (test_name, templ, create_fun, num_nodes) in instance_tests:
592
    if (qa_config.TestEnabled(test_name) and
593
        qa_config.IsTemplateSupported(templ)):
594
      inodes = qa_config.AcquireManyNodes(num_nodes)
595
      try:
596
        instance = RunTest(create_fun, inodes)
597
        try:
598
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
599
          RunDaemonTests(instance)
600
          for node in inodes:
601
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
602
          if len(inodes) > 1:
603
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
604
                      constants.INITIAL_NODE_GROUP_NAME,
605
                      inodes[0].primary, inodes[1].primary)
606
          if qa_config.TestEnabled("instance-convert-disk"):
607
            RunTest(qa_instance.TestInstanceShutdown, instance)
608
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
609
                    instance, inodes)
610
            RunTest(qa_instance.TestInstanceStartup, instance)
611
          RunCommonInstanceTests(instance)
612
          if qa_config.TestEnabled("instance-modify-primary"):
613
            othernode = qa_config.AcquireNode()
614
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
615
                    instance, inodes[0], othernode)
616
            othernode.Release()
617
          RunGroupListTests()
618
          RunExportImportTests(instance, inodes)
619
          RunHardwareFailureTests(instance, inodes)
620
          RunRepairDiskSizes()
621
          RunTest(qa_instance.TestInstanceRemove, instance)
622
        finally:
623
          instance.Release()
624
        del instance
625
      finally:
626
        qa_config.ReleaseManyNodes(inodes)
627
      qa_cluster.AssertClusterVerify()
628

    
629

    
630
def RunQa():
631
  """Main QA body.
632

633
  """
634
  rapi_user = "ganeti-qa"
635
  rapi_secret = utils.GenerateSecret()
636

    
637
  RunEnvTests()
638
  SetupCluster(rapi_user, rapi_secret)
639

    
640
  # Load RAPI certificate
641
  qa_rapi.Setup(rapi_user, rapi_secret)
642

    
643
  RunClusterTests()
644
  RunOsTests()
645

    
646
  RunTestIf("tags", qa_tags.TestClusterTags)
647

    
648
  RunCommonNodeTests()
649
  RunGroupListTests()
650
  RunGroupRwTests()
651
  RunNetworkTests()
652

    
653
  # The master shouldn't be readded or put offline; "delay" needs a non-master
654
  # node to test
655
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
656
  try:
657
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
658
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
659
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
660
  finally:
661
    pnode.Release()
662

    
663
  # Make sure the cluster is clean before running instance tests
664
  qa_cluster.AssertClusterVerify()
665

    
666
  pnode = qa_config.AcquireNode()
667
  try:
668
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
669

    
670
    if qa_rapi.Enabled():
671
      RunTest(qa_rapi.TestNode, pnode)
672

    
673
      if qa_config.TestEnabled("instance-add-plain-disk"):
674
        for use_client in [True, False]:
675
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
676
                                  use_client)
677
          try:
678
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
679
              RunCommonInstanceTests(rapi_instance)
680
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
681
          finally:
682
            rapi_instance.Release()
683
          del rapi_instance
684

    
685
  finally:
686
    pnode.Release()
687

    
688
  config_list = [
689
    ("default-instance-tests", lambda: None, lambda _: None),
690
    ("exclusive-storage-instance-tests",
691
     lambda: qa_cluster.TestSetExclStorCluster(True),
692
     qa_cluster.TestSetExclStorCluster),
693
  ]
694
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
695
    if qa_config.TestEnabled(conf_name):
696
      oldconf = setup_conf_f()
697
      RunInstanceTests()
698
      restore_conf_f(oldconf)
699

    
700
  pnode = qa_config.AcquireNode()
701
  try:
702
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
703
      for shutdown in [False, True]:
704
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
705
        try:
706
          expnode = qa_config.AcquireNode(exclude=pnode)
707
          try:
708
            if shutdown:
709
              # Stop instance before exporting and removing it
710
              RunTest(qa_instance.TestInstanceShutdown, instance)
711
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
712
            RunTest(qa_instance.TestBackupList, expnode)
713
          finally:
714
            expnode.Release()
715
        finally:
716
          instance.Release()
717
        del expnode
718
        del instance
719
      qa_cluster.AssertClusterVerify()
720

    
721
  finally:
722
    pnode.Release()
723

    
724
  RunExclusiveStorageTests()
725
  RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
726
            TestIPolicyPlainInstance)
727

    
728
  RunTestIf(
729
    "instance-add-restricted-by-disktemplates",
730
    qa_instance.TestInstanceCreationRestrictedByDiskTemplates)
731

    
732
  # Test removing instance with offline drbd secondary
733
  if qa_config.TestEnabled(["instance-remove-drbd-offline",
734
                            "instance-add-drbd-disk"]):
735
    # Make sure the master is not put offline
736
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
737
    try:
738
      pnode = qa_config.AcquireNode(exclude=snode)
739
      try:
740
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
741
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
742
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
743
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
744
                set_offline, set_online)
745
      finally:
746
        pnode.Release()
747
    finally:
748
      snode.Release()
749
    qa_cluster.AssertClusterVerify()
750

    
751
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
752

    
753
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
754

    
755

    
756
@UsesRapiClient
757
def main():
758
  """Main program.
759

760
  """
761
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
762
  parser.add_option("--yes-do-it", dest="yes_do_it",
763
                    action="store_true",
764
                    help="Really execute the tests")
765
  (opts, args) = parser.parse_args()
766

    
767
  if len(args) == 1:
768
    (config_file, ) = args
769
  else:
770
    parser.error("Wrong number of arguments.")
771

    
772
  if not opts.yes_do_it:
773
    print ("Executing this script irreversibly destroys any Ganeti\n"
774
           "configuration on all nodes involved. If you really want\n"
775
           "to start testing, supply the --yes-do-it option.")
776
    sys.exit(1)
777

    
778
  qa_config.Load(config_file)
779

    
780
  primary = qa_config.GetMasterNode().primary
781
  qa_utils.StartMultiplexer(primary)
782
  print ("SSH command for primary node: %s" %
783
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
784
  print ("SSH command for other nodes: %s" %
785
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
786
  try:
787
    RunQa()
788
  finally:
789
    qa_utils.CloseMultiplexers()
790

    
791
if __name__ == "__main__":
792
  main()