Statistics
| Branch: | Tag: | Revision:

root / qa / ganeti-qa.py @ e5c2accd

History | View | Annotate | Download (13.2 kB)

1
#!/usr/bin/python -u
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Script for doing QA on Ganeti.
23

24
"""
25

    
26
import sys
27
import datetime
28
import optparse
29

    
30
import qa_cluster
31
import qa_config
32
import qa_daemon
33
import qa_env
34
import qa_group
35
import qa_instance
36
import qa_node
37
import qa_os
38
import qa_rapi
39
import qa_tags
40
import qa_utils
41

    
42
from ganeti import utils
43
from ganeti import rapi
44

    
45
import ganeti.rapi.client
46

    
47

    
48
def _FormatHeader(line, end=72):
49
  """Fill a line up to the end column.
50

51
  """
52
  line = "---- " + line + " "
53
  line += "-" * (end-len(line))
54
  line = line.rstrip()
55
  return line
56

    
57

    
58
def _DescriptionOf(fn):
59
  """Computes the description of an item.
60

61
  """
62
  if fn.__doc__:
63
    desc = fn.__doc__.splitlines()[0].strip()
64
  else:
65
    desc = "%r" % fn
66

    
67
  return desc.rstrip(".")
68

    
69
def RunTest(fn, *args):
70
  """Runs a test after printing a header.
71

72
  """
73

    
74
  tstart = datetime.datetime.now()
75

    
76
  desc = _DescriptionOf(fn)
77

    
78
  print
79
  print _FormatHeader("%s start %s" % (tstart, desc))
80

    
81
  try:
82
    retval = fn(*args)
83
    return retval
84
  finally:
85
    tstop = datetime.datetime.now()
86
    tdelta = tstop - tstart
87
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
88

    
89

    
90
def RunTestIf(testnames, fn, *args):
91
  """Runs a test conditionally.
92

93
  @param testnames: either a single test name in the configuration
94
      file, or a list of testnames (which will be AND-ed together)
95

96
  """
97
  if qa_config.TestEnabled(testnames):
98
    RunTest(fn, *args)
99
  else:
100
    tstart = datetime.datetime.now()
101
    desc = _DescriptionOf(fn)
102
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
103
                        (tstart, desc, testnames))
104

    
105

    
106
def RunEnvTests():
107
  """Run several environment tests.
108

109
  """
110
  RunTestIf("env", qa_env.TestSshConnection)
111
  RunTestIf("env", qa_env.TestIcmpPing)
112
  RunTestIf("env", qa_env.TestGanetiCommands)
113

    
114

    
115
def SetupCluster(rapi_user, rapi_secret):
116
  """Initializes the cluster.
117

118
  @param rapi_user: Login user for RAPI
119
  @param rapi_secret: Login secret for RAPI
120

121
  """
122
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
123
            rapi_user, rapi_secret)
124
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
125
  if not qa_config.TestEnabled("create-cluster"):
126
    # consider the nodes are already there
127
    qa_node.MarkNodeAddedAll()
128

    
129
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
130

    
131
  # enable the watcher (unconditionally)
132
  RunTest(qa_daemon.TestResumeWatcher)
133

    
134
  RunTestIf("node-info", qa_node.TestNodeInfo)
135

    
136

    
137
def RunClusterTests():
138
  """Runs tests related to gnt-cluster.
139

140
  """
141
  for test, fn in [
142
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
143
    ("cluster-verify", qa_cluster.TestClusterVerify),
144
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
145
    # TODO: add more cluster modify tests
146
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
147
    ("cluster-rename", qa_cluster.TestClusterRename),
148
    ("cluster-info", qa_cluster.TestClusterVersion),
149
    ("cluster-info", qa_cluster.TestClusterInfo),
150
    ("cluster-info", qa_cluster.TestClusterGetmaster),
151
    ("cluster-copyfile", qa_cluster.TestClusterCopyfile),
152
    ("cluster-command", qa_cluster.TestClusterCommand),
153
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
154
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
155
    ("rapi", qa_rapi.TestVersion),
156
    ("rapi", qa_rapi.TestEmptyCluster),
157
    ]:
158
    RunTestIf(test, fn)
159

    
160

    
161
def RunOsTests():
162
  """Runs all tests related to gnt-os.
163

164
  """
165
  for fn in [
166
    qa_os.TestOsList,
167
    qa_os.TestOsDiagnose,
168
    qa_os.TestOsValid,
169
    qa_os.TestOsInvalid,
170
    qa_os.TestOsPartiallyValid,
171
    qa_os.TestOsModifyValid,
172
    qa_os.TestOsModifyInvalid,
173
    qa_os.TestOsStates,
174
    ]:
175
    RunTestIf("os", fn)
176

    
177

    
178
def RunCommonInstanceTests(instance):
179
  """Runs a few tests that are common to all disk types.
180

181
  """
182
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
183
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
184

    
185
  RunTestIf("instance-list", qa_instance.TestInstanceList)
186

    
187
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
188

    
189
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
190
  RunTestIf(["instance-modify", "rapi"],
191
            qa_rapi.TestRapiInstanceModify, instance)
192

    
193
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
194

    
195
  RunTestIf("instance-reinstall", qa_instance.TestInstanceShutdown, instance)
196
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
197
  RunTestIf("instance-reinstall", qa_instance.TestInstanceStartup, instance)
198

    
199
  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
200

    
201
  if qa_config.TestEnabled('instance-rename'):
202
    rename_source = instance["name"]
203
    rename_target = qa_config.get("rename", None)
204
    if rename_target is None:
205
      print qa_utils.FormatError("Can rename instance, 'rename' entry is"
206
                                 " missing from configuration")
207
    else:
208
      RunTest(qa_instance.TestInstanceShutdown, instance)
209
      RunTest(qa_instance.TestInstanceRename, rename_source, rename_target)
210
      RunTest(qa_instance.TestInstanceRename, rename_target, rename_source)
211
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRename, rename_source, rename_target)
212
      RunTestIf("rapi", qa_rapi.TestRapiInstanceRename, rename_target, rename_source)
213
      RunTest(qa_instance.TestInstanceStartup, instance)
214

    
215
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
216

    
217
  RunTestIf("rapi", qa_rapi.TestInstance, instance)
218

    
219

    
220
def RunCommonNodeTests():
221
  """Run a few common node tests.
222

223
  """
224
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
225
  RunTestIf("node-storage", qa_node.TestNodeStorage)
226

    
227

    
228
def RunGroupListTests():
229
  """Run tests for listing node groups.
230

231
  """
232
  RunTestIf("group-list", qa_group.TestGroupListDefaultFields)
233
  RunTestIf("group-list", qa_group.TestGroupListAllFields)
234

    
235

    
236
def RunGroupRwTests():
237
  """Run tests for adding/removing/renaming groups.
238

239
  """
240
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
241

    
242
def RunExportImportTests(instance, pnode, snode):
243
  """Tries to export and import the instance.
244

245
  @param pnode: current primary node of the instance
246
  @param snode: current secondary node of the instance, if any,
247
      otherwise None
248

249
  """
250
  if qa_config.TestEnabled('instance-export'):
251
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)
252

    
253
    expnode = qa_config.AcquireNode(exclude=pnode)
254
    try:
255
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
256

    
257
      RunTest(qa_instance.TestBackupList, expnode)
258

    
259
      if qa_config.TestEnabled('instance-import'):
260
        newinst = qa_config.AcquireInstance()
261
        try:
262
          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
263
                  expnode, name)
264
          RunTest(qa_instance.TestInstanceRemove, newinst)
265
        finally:
266
          qa_config.ReleaseInstance(newinst)
267
    finally:
268
      qa_config.ReleaseNode(expnode)
269

    
270
  if qa_config.TestEnabled(["rapi", "inter-cluster-instance-move"]):
271
    newinst = qa_config.AcquireInstance()
272
    try:
273
      if snode is None:
274
        excl = [pnode]
275
      else:
276
        excl = [pnode, snode]
277
      tnode = qa_config.AcquireNode(exclude=excl)
278
      try:
279
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
280
                pnode, snode, tnode)
281
      finally:
282
        qa_config.ReleaseNode(tnode)
283
    finally:
284
      qa_config.ReleaseInstance(newinst)
285

    
286

    
287
def RunDaemonTests(instance, pnode):
288
  """Test the ganeti-watcher script.
289

290
  """
291
  RunTest(qa_daemon.TestPauseWatcher)
292

    
293
  RunTestIf("instance-automatic-restart",
294
            qa_daemon.TestInstanceAutomaticRestart, pnode, instance)
295
  RunTestIf("instance-consecutive-failures",
296
            qa_daemon.TestInstanceConsecutiveFailures, pnode, instance)
297

    
298
  RunTest(qa_daemon.TestResumeWatcher)
299

    
300

    
301
def RunHardwareFailureTests(instance, pnode, snode):
302
  """Test cluster internal hardware failure recovery.
303

304
  """
305
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
306

    
307
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
308
  RunTestIf(["instance-migrate", "rapi"],
309
            qa_rapi.TestRapiInstanceMigrate, instance)
310

    
311
  if qa_config.TestEnabled('instance-replace-disks'):
312
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
313
    try:
314
      RunTest(qa_instance.TestReplaceDisks,
315
              instance, pnode, snode, othernode)
316
    finally:
317
      qa_config.ReleaseNode(othernode)
318

    
319
  RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, pnode, snode)
320

    
321
  RunTestIf("node-failover", qa_node.TestNodeFailover, pnode, snode)
322

    
323
  RunTestIf("instance-disk-failure", qa_instance.TestInstanceMasterDiskFailure,
324
            instance, pnode, snode)
325
  RunTestIf("instance-disk-failure",
326
            qa_instance.TestInstanceSecondaryDiskFailure, instance,
327
            pnode, snode)
328

    
329

    
330
@rapi.client.UsesRapiClient
331
def main():
332
  """Main program.
333

334
  """
335
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
336
  parser.add_option('--yes-do-it', dest='yes_do_it',
337
      action="store_true",
338
      help="Really execute the tests")
339
  (qa_config.options, args) = parser.parse_args()
340

    
341
  if len(args) == 1:
342
    (config_file, ) = args
343
  else:
344
    parser.error("Wrong number of arguments.")
345

    
346
  if not qa_config.options.yes_do_it:
347
    print ("Executing this script irreversibly destroys any Ganeti\n"
348
           "configuration on all nodes involved. If you really want\n"
349
           "to start testing, supply the --yes-do-it option.")
350
    sys.exit(1)
351

    
352
  qa_config.Load(config_file)
353

    
354
  rapi_user = "ganeti-qa"
355
  rapi_secret = utils.GenerateSecret()
356

    
357
  RunEnvTests()
358
  SetupCluster(rapi_user, rapi_secret)
359

    
360
  # Load RAPI certificate
361
  qa_rapi.Setup(rapi_user, rapi_secret)
362

    
363
  RunClusterTests()
364
  RunOsTests()
365

    
366
  RunTestIf("tags", qa_tags.TestClusterTags)
367

    
368
  RunCommonNodeTests()
369
  RunGroupListTests()
370
  RunGroupRwTests()
371

    
372
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
373
  try:
374
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
375
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
376
  finally:
377
    qa_config.ReleaseNode(pnode)
378

    
379
  pnode = qa_config.AcquireNode()
380
  try:
381
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
382

    
383
    if qa_rapi.Enabled():
384
      RunTest(qa_rapi.TestNode, pnode)
385

    
386
      if qa_config.TestEnabled("instance-add-plain-disk"):
387
        for use_client in [True, False]:
388
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
389
                                  use_client)
390
          RunCommonInstanceTests(rapi_instance)
391
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
392
          del rapi_instance
393

    
394
    if qa_config.TestEnabled('instance-add-plain-disk'):
395
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
396
      RunCommonInstanceTests(instance)
397
      RunGroupListTests()
398
      RunExportImportTests(instance, pnode, None)
399
      RunDaemonTests(instance, pnode)
400
      RunTest(qa_instance.TestInstanceRemove, instance)
401
      del instance
402

    
403
    multinode_tests = [
404
      ('instance-add-drbd-disk',
405
       qa_instance.TestInstanceAddWithDrbdDisk),
406
    ]
407

    
408
    for name, func in multinode_tests:
409
      if qa_config.TestEnabled(name):
410
        snode = qa_config.AcquireNode(exclude=pnode)
411
        try:
412
          instance = RunTest(func, pnode, snode)
413
          RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
414
          RunCommonInstanceTests(instance)
415
          RunGroupListTests()
416
          if qa_config.TestEnabled('instance-convert-disk'):
417
            RunTest(qa_instance.TestInstanceShutdown, instance)
418
            RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
419
            RunTest(qa_instance.TestInstanceStartup, instance)
420
          RunExportImportTests(instance, pnode, snode)
421
          RunHardwareFailureTests(instance, pnode, snode)
422
          RunTest(qa_instance.TestInstanceRemove, instance)
423
          del instance
424
        finally:
425
          qa_config.ReleaseNode(snode)
426

    
427
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
428
      for shutdown in [False, True]:
429
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
430
        expnode = qa_config.AcquireNode(exclude=pnode)
431
        try:
432
          if shutdown:
433
            # Stop instance before exporting and removing it
434
            RunTest(qa_instance.TestInstanceShutdown, instance)
435
          RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
436
          RunTest(qa_instance.TestBackupList, expnode)
437
        finally:
438
          qa_config.ReleaseNode(expnode)
439
        del expnode
440
        del instance
441

    
442
  finally:
443
    qa_config.ReleaseNode(pnode)
444

    
445
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
446

    
447
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
448

    
449

    
450
if __name__ == '__main__':
451
  main()