Statistics
| Branch: | Tag: | Revision:

root / qa / ganeti-qa.py @ 0fdf247d

History | View | Annotate | Download (16.6 kB)

1
#!/usr/bin/python -u
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Script for doing QA on Ganeti.
23

24
"""
25

    
26
# pylint: disable=C0103
27
# due to invalid name
28

    
29
import sys
30
import datetime
31
import optparse
32

    
33
import qa_cluster
34
import qa_config
35
import qa_daemon
36
import qa_env
37
import qa_group
38
import qa_instance
39
import qa_node
40
import qa_os
41
import qa_job
42
import qa_rapi
43
import qa_tags
44
import qa_utils
45

    
46
from ganeti import utils
47
from ganeti import rapi
48
from ganeti import constants
49

    
50
import ganeti.rapi.client # pylint: disable=W0611
51

    
52

    
53
def _FormatHeader(line, end=72):
54
  """Fill a line up to the end column.
55

56
  """
57
  line = "---- " + line + " "
58
  line += "-" * (end - len(line))
59
  line = line.rstrip()
60
  return line
61

    
62

    
63
def _DescriptionOf(fn):
64
  """Computes the description of an item.
65

66
  """
67
  if fn.__doc__:
68
    desc = fn.__doc__.splitlines()[0].strip()
69
  else:
70
    desc = "%r" % fn
71

    
72
  return desc.rstrip(".")
73

    
74

    
75
def RunTest(fn, *args):
76
  """Runs a test after printing a header.
77

78
  """
79

    
80
  tstart = datetime.datetime.now()
81

    
82
  desc = _DescriptionOf(fn)
83

    
84
  print
85
  print _FormatHeader("%s start %s" % (tstart, desc))
86

    
87
  try:
88
    retval = fn(*args)
89
    return retval
90
  finally:
91
    tstop = datetime.datetime.now()
92
    tdelta = tstop - tstart
93
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
94

    
95

    
96
def RunTestIf(testnames, fn, *args):
97
  """Runs a test conditionally.
98

99
  @param testnames: either a single test name in the configuration
100
      file, or a list of testnames (which will be AND-ed together)
101

102
  """
103
  if qa_config.TestEnabled(testnames):
104
    RunTest(fn, *args)
105
  else:
106
    tstart = datetime.datetime.now()
107
    desc = _DescriptionOf(fn)
108
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
109
                        (tstart, desc, testnames))
110

    
111

    
112
def RunEnvTests():
113
  """Run several environment tests.
114

115
  """
116
  RunTestIf("env", qa_env.TestSshConnection)
117
  RunTestIf("env", qa_env.TestIcmpPing)
118
  RunTestIf("env", qa_env.TestGanetiCommands)
119

    
120

    
121
def SetupCluster(rapi_user, rapi_secret):
122
  """Initializes the cluster.
123

124
  @param rapi_user: Login user for RAPI
125
  @param rapi_secret: Login secret for RAPI
126

127
  """
128
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
129
            rapi_user, rapi_secret)
130

    
131
  # Test on empty cluster
132
  RunTestIf("node-list", qa_node.TestNodeList)
133
  RunTestIf("instance-list", qa_instance.TestInstanceList)
134
  RunTestIf("job-list", qa_job.TestJobList)
135

    
136
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
137
  if not qa_config.TestEnabled("create-cluster"):
138
    # consider the nodes are already there
139
    qa_node.MarkNodeAddedAll()
140

    
141
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
142

    
143
  # enable the watcher (unconditionally)
144
  RunTest(qa_daemon.TestResumeWatcher)
145

    
146
  RunTestIf("node-list", qa_node.TestNodeList)
147

    
148
  # Test listing fields
149
  RunTestIf("node-list", qa_node.TestNodeListFields)
150
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
151
  RunTestIf("job-list", qa_job.TestJobListFields)
152
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
153

    
154
  RunTestIf("node-info", qa_node.TestNodeInfo)
155

    
156

    
157
def RunClusterTests():
158
  """Runs tests related to gnt-cluster.
159

160
  """
161
  for test, fn in [
162
    ("create-cluster", qa_cluster.TestClusterInitDisk),
163
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
164
    ("cluster-verify", qa_cluster.TestClusterVerify),
165
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
166
    # TODO: add more cluster modify tests
167
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
168
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
169
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
170
    ("cluster-rename", qa_cluster.TestClusterRename),
171
    ("cluster-info", qa_cluster.TestClusterVersion),
172
    ("cluster-info", qa_cluster.TestClusterInfo),
173
    ("cluster-info", qa_cluster.TestClusterGetmaster),
174
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
175
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
176
    ("cluster-command", qa_cluster.TestClusterCommand),
177
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
178
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
179
    ("cluster-master-failover",
180
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
181
    ("cluster-oob", qa_cluster.TestClusterOob),
182
    ("rapi", qa_rapi.TestVersion),
183
    ("rapi", qa_rapi.TestEmptyCluster),
184
    ("rapi", qa_rapi.TestRapiQuery),
185
    ]:
186
    RunTestIf(test, fn)
187

    
188

    
189
def RunRepairDiskSizes():
190
  """Run the repair disk-sizes test.
191

192
  """
193
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
194

    
195

    
196
def RunOsTests():
197
  """Runs all tests related to gnt-os.
198

199
  """
200
  if qa_config.TestEnabled("rapi"):
201
    rapi_getos = qa_rapi.GetOperatingSystems
202
  else:
203
    rapi_getos = None
204

    
205
  for fn in [
206
    qa_os.TestOsList,
207
    qa_os.TestOsDiagnose,
208
    ]:
209
    RunTestIf("os", fn)
210

    
211
  for fn in [
212
    qa_os.TestOsValid,
213
    qa_os.TestOsInvalid,
214
    qa_os.TestOsPartiallyValid,
215
    ]:
216
    RunTestIf("os", fn, rapi_getos)
217

    
218
  for fn in [
219
    qa_os.TestOsModifyValid,
220
    qa_os.TestOsModifyInvalid,
221
    qa_os.TestOsStatesNonExisting,
222
    ]:
223
    RunTestIf("os", fn)
224

    
225

    
226
def RunCommonInstanceTests(instance):
227
  """Runs a few tests that are common to all disk types.
228

229
  """
230
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
231
  RunTestIf(["instance-shutdown", "instance-console", "rapi"],
232
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
233
  RunTestIf(["instance-shutdown", "instance-modify"],
234
            qa_instance.TestInstanceStoppedModify, instance)
235
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
236

    
237
  # Test shutdown/start via RAPI
238
  RunTestIf(["instance-shutdown", "rapi"],
239
            qa_rapi.TestRapiInstanceShutdown, instance)
240
  RunTestIf(["instance-shutdown", "rapi"],
241
            qa_rapi.TestRapiInstanceStartup, instance)
242

    
243
  RunTestIf("instance-list", qa_instance.TestInstanceList)
244

    
245
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
246

    
247
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
248
  RunTestIf(["instance-modify", "rapi"],
249
            qa_rapi.TestRapiInstanceModify, instance)
250

    
251
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
252
  RunTestIf(["instance-console", "rapi"],
253
            qa_rapi.TestRapiInstanceConsole, instance)
254

    
255
  DOWN_TESTS = ["instance-reinstall", "instance-rename"]
256
  # shutdown instance for any 'down' tests
257
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)
258

    
259
  # now run the 'down' state tests
260
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
261
  RunTestIf(["instance-reinstall", "rapi"],
262
            qa_rapi.TestRapiInstanceReinstall, instance)
263
  # RAPI reinstall will leave the instance up by default, so we have
264
  # to stop it again
265
  RunTestIf(["instance-reinstall", "rapi"],
266
            qa_rapi.TestRapiInstanceShutdown, instance)
267

    
268
  if qa_config.TestEnabled("instance-rename"):
269
    rename_source = instance["name"]
270
    rename_target = qa_config.get("rename", None)
271
    # perform instance rename to the same name
272
    RunTest(qa_instance.TestInstanceRenameAndBack,
273
            rename_source, rename_source)
274
    RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
275
              rename_source, rename_source)
276
    if rename_target is not None:
277
      # perform instance rename to a different name, if we have one configured
278
      RunTest(qa_instance.TestInstanceRenameAndBack,
279
              rename_source, rename_target)
280
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRenameAndBack,
281
                rename_source, rename_target)
282

    
283
  # and now start the instance again
284
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)
285

    
286
  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
287

    
288
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
289

    
290
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
291

    
292
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
293

    
294
  # Lists instances, too
295
  RunTestIf("node-list", qa_node.TestNodeList)
296

    
297
  # Some jobs have been run, let's test listing them
298
  RunTestIf("job-list", qa_job.TestJobList)
299

    
300

    
301
def RunCommonNodeTests():
302
  """Run a few common node tests.
303

304
  """
305
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
306
  RunTestIf("node-storage", qa_node.TestNodeStorage)
307
  RunTestIf("node-oob", qa_node.TestOutOfBand)
308

    
309

    
310
def RunGroupListTests():
311
  """Run tests for listing node groups.
312

313
  """
314
  RunTestIf("group-list", qa_group.TestGroupList)
315
  RunTestIf("group-list", qa_group.TestGroupListFields)
316

    
317

    
318
def RunGroupRwTests():
319
  """Run tests for adding/removing/renaming groups.
320

321
  """
322
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
323
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
324
  RunTestIf("group-rwops", qa_group.TestGroupModify)
325
  RunTestIf(["group-rwops", "rapi"], qa_rapi.TestRapiNodeGroups)
326
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
327
            qa_group.GetDefaultGroup())
328

    
329

    
330
def RunExportImportTests(instance, pnode, snode):
331
  """Tries to export and import the instance.
332

333
  @param pnode: current primary node of the instance
334
  @param snode: current secondary node of the instance, if any,
335
      otherwise None
336

337
  """
338
  if qa_config.TestEnabled("instance-export"):
339
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)
340

    
341
    expnode = qa_config.AcquireNode(exclude=pnode)
342
    try:
343
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
344

    
345
      RunTest(qa_instance.TestBackupList, expnode)
346

    
347
      if qa_config.TestEnabled("instance-import"):
348
        newinst = qa_config.AcquireInstance()
349
        try:
350
          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
351
                  expnode, name)
352
          RunTest(qa_instance.TestInstanceRemove, newinst)
353
        finally:
354
          qa_config.ReleaseInstance(newinst)
355
    finally:
356
      qa_config.ReleaseNode(expnode)
357

    
358
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
359
    newinst = qa_config.AcquireInstance()
360
    try:
361
      if snode is None:
362
        excl = [pnode]
363
      else:
364
        excl = [pnode, snode]
365
      tnode = qa_config.AcquireNode(exclude=excl)
366
      try:
367
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
368
                pnode, snode, tnode)
369
      finally:
370
        qa_config.ReleaseNode(tnode)
371
    finally:
372
      qa_config.ReleaseInstance(newinst)
373

    
374

    
375
def RunDaemonTests(instance):
376
  """Test the ganeti-watcher script.
377

378
  """
379
  RunTest(qa_daemon.TestPauseWatcher)
380

    
381
  RunTestIf("instance-automatic-restart",
382
            qa_daemon.TestInstanceAutomaticRestart, instance)
383
  RunTestIf("instance-consecutive-failures",
384
            qa_daemon.TestInstanceConsecutiveFailures, instance)
385

    
386
  RunTest(qa_daemon.TestResumeWatcher)
387

    
388

    
389
def RunHardwareFailureTests(instance, pnode, snode):
390
  """Test cluster internal hardware failure recovery.
391

392
  """
393
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
394
  RunTestIf(["instance-failover", "rapi"],
395
            qa_rapi.TestRapiInstanceFailover, instance)
396

    
397
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
398
  RunTestIf(["instance-migrate", "rapi"],
399
            qa_rapi.TestRapiInstanceMigrate, instance)
400

    
401
  if qa_config.TestEnabled("instance-replace-disks"):
402
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
403
    try:
404
      RunTestIf("rapi", qa_rapi.TestRapiInstanceReplaceDisks, instance)
405
      RunTest(qa_instance.TestReplaceDisks,
406
              instance, pnode, snode, othernode)
407
    finally:
408
      qa_config.ReleaseNode(othernode)
409

    
410
  RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, pnode, snode)
411

    
412
  RunTestIf("node-failover", qa_node.TestNodeFailover, pnode, snode)
413

    
414
  RunTestIf("instance-disk-failure", qa_instance.TestInstanceMasterDiskFailure,
415
            instance, pnode, snode)
416
  RunTestIf("instance-disk-failure",
417
            qa_instance.TestInstanceSecondaryDiskFailure, instance,
418
            pnode, snode)
419

    
420

    
421
def RunQa():
422
  """Main QA body.
423

424
  """
425
  rapi_user = "ganeti-qa"
426
  rapi_secret = utils.GenerateSecret()
427

    
428
  RunEnvTests()
429
  SetupCluster(rapi_user, rapi_secret)
430

    
431
  # Load RAPI certificate
432
  qa_rapi.Setup(rapi_user, rapi_secret)
433

    
434
  RunClusterTests()
435
  RunOsTests()
436

    
437
  RunTestIf("tags", qa_tags.TestClusterTags)
438

    
439
  RunCommonNodeTests()
440
  RunGroupListTests()
441
  RunGroupRwTests()
442

    
443
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
444
  try:
445
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
446
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
447
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
448
  finally:
449
    qa_config.ReleaseNode(pnode)
450

    
451
  pnode = qa_config.AcquireNode()
452
  try:
453
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
454

    
455
    if qa_rapi.Enabled():
456
      RunTest(qa_rapi.TestNode, pnode)
457

    
458
      if qa_config.TestEnabled("instance-add-plain-disk"):
459
        for use_client in [True, False]:
460
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
461
                                  use_client)
462
          RunCommonInstanceTests(rapi_instance)
463
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
464
          del rapi_instance
465

    
466
    if qa_config.TestEnabled("instance-add-plain-disk"):
467
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
468
      RunCommonInstanceTests(instance)
469
      RunGroupListTests()
470
      RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
471
      RunExportImportTests(instance, pnode, None)
472
      RunDaemonTests(instance)
473
      RunRepairDiskSizes()
474
      RunTest(qa_instance.TestInstanceRemove, instance)
475
      del instance
476

    
477
    multinode_tests = [
478
      ("instance-add-drbd-disk",
479
       qa_instance.TestInstanceAddWithDrbdDisk),
480
    ]
481

    
482
    for name, func in multinode_tests:
483
      if qa_config.TestEnabled(name):
484
        snode = qa_config.AcquireNode(exclude=pnode)
485
        try:
486
          instance = RunTest(func, pnode, snode)
487
          RunCommonInstanceTests(instance)
488
          RunGroupListTests()
489
          RunTest(qa_group.TestAssignNodesIncludingSplit,
490
                  constants.INITIAL_NODE_GROUP_NAME,
491
                  pnode["primary"], snode["primary"])
492
          if qa_config.TestEnabled("instance-convert-disk"):
493
            RunTest(qa_instance.TestInstanceShutdown, instance)
494
            RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
495
            RunTest(qa_instance.TestInstanceStartup, instance)
496
          RunExportImportTests(instance, pnode, snode)
497
          RunHardwareFailureTests(instance, pnode, snode)
498
          RunRepairDiskSizes()
499
          RunTest(qa_instance.TestInstanceRemove, instance)
500
          del instance
501
        finally:
502
          qa_config.ReleaseNode(snode)
503

    
504
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
505
      for shutdown in [False, True]:
506
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
507
        expnode = qa_config.AcquireNode(exclude=pnode)
508
        try:
509
          if shutdown:
510
            # Stop instance before exporting and removing it
511
            RunTest(qa_instance.TestInstanceShutdown, instance)
512
          RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
513
          RunTest(qa_instance.TestBackupList, expnode)
514
        finally:
515
          qa_config.ReleaseNode(expnode)
516
        del expnode
517
        del instance
518

    
519
  finally:
520
    qa_config.ReleaseNode(pnode)
521

    
522
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
523

    
524
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
525

    
526

    
527
@rapi.client.UsesRapiClient
528
def main():
529
  """Main program.
530

531
  """
532
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
533
  parser.add_option("--yes-do-it", dest="yes_do_it",
534
      action="store_true",
535
      help="Really execute the tests")
536
  (qa_config.options, args) = parser.parse_args()
537

    
538
  if len(args) == 1:
539
    (config_file, ) = args
540
  else:
541
    parser.error("Wrong number of arguments.")
542

    
543
  if not qa_config.options.yes_do_it:
544
    print ("Executing this script irreversibly destroys any Ganeti\n"
545
           "configuration on all nodes involved. If you really want\n"
546
           "to start testing, supply the --yes-do-it option.")
547
    sys.exit(1)
548

    
549
  qa_config.Load(config_file)
550

    
551
  qa_utils.StartMultiplexer(qa_config.GetMasterNode()["primary"])
552
  try:
553
    RunQa()
554
  finally:
555
    qa_utils.CloseMultiplexers()
556

    
557
if __name__ == "__main__":
558
  main()