Statistics
| Branch: | Tag: | Revision:

root / qa / ganeti-qa.py @ 8201b996

History | View | Annotate | Download (12.5 kB)

1
#!/usr/bin/python -u
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Script for doing QA on Ganeti.
23

24
"""
25

    
26
import sys
27
import datetime
28
import optparse
29

    
30
import qa_cluster
31
import qa_config
32
import qa_daemon
33
import qa_env
34
import qa_instance
35
import qa_node
36
import qa_os
37
import qa_rapi
38
import qa_tags
39
import qa_utils
40

    
41
from ganeti import utils
42
from ganeti import rapi
43

    
44
import ganeti.rapi.client
45

    
46

    
47
def _FormatHeader(line, end=72, char="-"):
48
  """Fill a line up to the end column.
49

50
  """
51
  line = "---- " + line + " "
52
  line += "-" * (end-len(line))
53
  line = line.rstrip()
54
  return line
55

    
56

    
57
def RunTest(fn, *args):
58
  """Runs a test after printing a header.
59

60
  """
61
  if fn.__doc__:
62
    desc = fn.__doc__.splitlines()[0].strip()
63
  else:
64
    desc = "%r" % fn
65

    
66
  desc = desc.rstrip(".")
67

    
68
  tstart = datetime.datetime.now()
69

    
70
  print
71
  print _FormatHeader("%s start %s" % (tstart, desc))
72

    
73
  try:
74
    retval = fn(*args)
75
    return retval
76
  finally:
77
    tstop = datetime.datetime.now()
78
    tdelta = tstop - tstart
79
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc))
80

    
81

    
82
def RunEnvTests():
83
  """Run several environment tests.
84

85
  """
86
  if not qa_config.TestEnabled('env'):
87
    return
88

    
89
  RunTest(qa_env.TestSshConnection)
90
  RunTest(qa_env.TestIcmpPing)
91
  RunTest(qa_env.TestGanetiCommands)
92

    
93

    
94
def SetupCluster(rapi_user, rapi_secret):
95
  """Initializes the cluster.
96

97
  @param rapi_user: Login user for RAPI
98
  @param rapi_secret: Login secret for RAPI
99

100
  """
101
  if qa_config.TestEnabled('create-cluster'):
102
    RunTest(qa_cluster.TestClusterInit, rapi_user, rapi_secret)
103
    RunTest(qa_node.TestNodeAddAll)
104
    RunTest(qa_cluster.TestJobqueue)
105
  else:
106
    # consider the nodes are already there
107
    qa_node.MarkNodeAddedAll()
108

    
109
  # enable the watcher (unconditionally)
110
  RunTest(qa_daemon.TestResumeWatcher)
111

    
112
  if qa_config.TestEnabled('node-info'):
113
    RunTest(qa_node.TestNodeInfo)
114

    
115

    
116
def RunClusterTests():
117
  """Runs tests related to gnt-cluster.
118

119
  """
120
  if qa_config.TestEnabled("cluster-renew-crypto"):
121
    RunTest(qa_cluster.TestClusterRenewCrypto)
122

    
123
  if qa_config.TestEnabled('cluster-verify'):
124
    RunTest(qa_cluster.TestClusterVerify)
125

    
126
  if qa_config.TestEnabled('cluster-reserved-lvs'):
127
    RunTest(qa_cluster.TestClusterReservedLvs)
128

    
129
  if qa_config.TestEnabled('cluster-rename'):
130
    RunTest(qa_cluster.TestClusterRename)
131

    
132
  if qa_config.TestEnabled('cluster-info'):
133
    RunTest(qa_cluster.TestClusterVersion)
134
    RunTest(qa_cluster.TestClusterInfo)
135
    RunTest(qa_cluster.TestClusterGetmaster)
136

    
137
  if qa_config.TestEnabled('cluster-copyfile'):
138
    RunTest(qa_cluster.TestClusterCopyfile)
139

    
140
  if qa_config.TestEnabled('cluster-command'):
141
    RunTest(qa_cluster.TestClusterCommand)
142

    
143
  if qa_config.TestEnabled('cluster-burnin'):
144
    RunTest(qa_cluster.TestClusterBurnin)
145

    
146
  if qa_config.TestEnabled('cluster-master-failover'):
147
    RunTest(qa_cluster.TestClusterMasterFailover)
148

    
149
  if qa_rapi.Enabled():
150
    RunTest(qa_rapi.TestVersion)
151
    RunTest(qa_rapi.TestEmptyCluster)
152

    
153

    
154
def RunOsTests():
155
  """Runs all tests related to gnt-os.
156

157
  """
158
  if not qa_config.TestEnabled('os'):
159
    return
160

    
161
  RunTest(qa_os.TestOsList)
162
  RunTest(qa_os.TestOsDiagnose)
163
  RunTest(qa_os.TestOsValid)
164
  RunTest(qa_os.TestOsInvalid)
165
  RunTest(qa_os.TestOsPartiallyValid)
166
  RunTest(qa_os.TestOsModifyValid)
167
  RunTest(qa_os.TestOsModifyInvalid)
168
  RunTest(qa_os.TestOsStates)
169

    
170

    
171
def RunCommonInstanceTests(instance):
172
  """Runs a few tests that are common to all disk types.
173

174
  """
175
  if qa_config.TestEnabled('instance-shutdown'):
176
    RunTest(qa_instance.TestInstanceShutdown, instance)
177
    RunTest(qa_instance.TestInstanceStartup, instance)
178

    
179
  if qa_config.TestEnabled('instance-list'):
180
    RunTest(qa_instance.TestInstanceList)
181

    
182
  if qa_config.TestEnabled('instance-info'):
183
    RunTest(qa_instance.TestInstanceInfo, instance)
184

    
185
  if qa_config.TestEnabled('instance-modify'):
186
    RunTest(qa_instance.TestInstanceModify, instance)
187
    if qa_rapi.Enabled():
188
      RunTest(qa_rapi.TestRapiInstanceModify, instance)
189

    
190
  if qa_config.TestEnabled('instance-console'):
191
    RunTest(qa_instance.TestInstanceConsole, instance)
192

    
193
  if qa_config.TestEnabled('instance-reinstall'):
194
    RunTest(qa_instance.TestInstanceShutdown, instance)
195
    RunTest(qa_instance.TestInstanceReinstall, instance)
196
    RunTest(qa_instance.TestInstanceStartup, instance)
197

    
198
  if qa_config.TestEnabled('instance-reboot'):
199
    RunTest(qa_instance.TestInstanceReboot, instance)
200

    
201
  if qa_config.TestEnabled('instance-rename'):
202
    rename_target = qa_config.get("rename", None)
203
    if rename_target is None:
204
      print qa_utils.FormatError("Can rename instance, 'rename' entry is"
205
                                 " missing from configuration")
206
    else:
207
      RunTest(qa_instance.TestInstanceShutdown, instance)
208
      RunTest(qa_instance.TestInstanceRename, instance, rename_target)
209
      if qa_rapi.Enabled():
210
        RunTest(qa_rapi.TestRapiInstanceRename, instance, rename_target)
211
      RunTest(qa_instance.TestInstanceStartup, instance)
212

    
213
  if qa_config.TestEnabled('tags'):
214
    RunTest(qa_tags.TestInstanceTags, instance)
215

    
216
  if qa_config.TestEnabled('node-volumes'):
217
    RunTest(qa_node.TestNodeVolumes)
218

    
219
  if qa_config.TestEnabled("node-storage"):
220
    RunTest(qa_node.TestNodeStorage)
221

    
222
  if qa_rapi.Enabled():
223
    RunTest(qa_rapi.TestInstance, instance)
224

    
225

    
226
def RunExportImportTests(instance, pnode, snode):
227
  """Tries to export and import the instance.
228

229
  @param pnode: current primary node of the instance
230
  @param snode: current secondary node of the instance, if any,
231
      otherwise None
232

233
  """
234
  if qa_config.TestEnabled('instance-export'):
235
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)
236

    
237
    expnode = qa_config.AcquireNode(exclude=pnode)
238
    try:
239
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
240

    
241
      RunTest(qa_instance.TestBackupList, expnode)
242

    
243
      if qa_config.TestEnabled('instance-import'):
244
        newinst = qa_config.AcquireInstance()
245
        try:
246
          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
247
                  expnode, name)
248
          RunTest(qa_instance.TestInstanceRemove, newinst)
249
        finally:
250
          qa_config.ReleaseInstance(newinst)
251
    finally:
252
      qa_config.ReleaseNode(expnode)
253

    
254
  if (qa_rapi.Enabled() and
255
      qa_config.TestEnabled("inter-cluster-instance-move")):
256
    newinst = qa_config.AcquireInstance()
257
    try:
258
      if snode is None:
259
        excl = [pnode]
260
      else:
261
        excl = [pnode, snode]
262
      tnode = qa_config.AcquireNode(exclude=excl)
263
      try:
264
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
265
                pnode, snode, tnode)
266
      finally:
267
        qa_config.ReleaseNode(tnode)
268
    finally:
269
      qa_config.ReleaseInstance(newinst)
270

    
271

    
272
def RunDaemonTests(instance, pnode):
273
  """Test the ganeti-watcher script.
274

275
  """
276
  automatic_restart = \
277
    qa_config.TestEnabled('instance-automatic-restart')
278
  consecutive_failures = \
279
    qa_config.TestEnabled('instance-consecutive-failures')
280

    
281
  RunTest(qa_daemon.TestPauseWatcher)
282
  if automatic_restart or consecutive_failures:
283

    
284
    if automatic_restart:
285
      RunTest(qa_daemon.TestInstanceAutomaticRestart, pnode, instance)
286

    
287
    if consecutive_failures:
288
      RunTest(qa_daemon.TestInstanceConsecutiveFailures, pnode, instance)
289

    
290
  RunTest(qa_daemon.TestResumeWatcher)
291

    
292

    
293
def RunHardwareFailureTests(instance, pnode, snode):
294
  """Test cluster internal hardware failure recovery.
295

296
  """
297
  if qa_config.TestEnabled('instance-failover'):
298
    RunTest(qa_instance.TestInstanceFailover, instance)
299

    
300
  if qa_config.TestEnabled("instance-migrate"):
301
    RunTest(qa_instance.TestInstanceMigrate, instance)
302

    
303
    if qa_rapi.Enabled():
304
      RunTest(qa_rapi.TestRapiInstanceMigrate, instance)
305

    
306
  if qa_config.TestEnabled('instance-replace-disks'):
307
    othernode = qa_config.AcquireNode(exclude=[pnode, snode])
308
    try:
309
      RunTest(qa_instance.TestReplaceDisks,
310
              instance, pnode, snode, othernode)
311
    finally:
312
      qa_config.ReleaseNode(othernode)
313

    
314
  if qa_config.TestEnabled('node-evacuate'):
315
    RunTest(qa_node.TestNodeEvacuate, pnode, snode)
316

    
317
  if qa_config.TestEnabled('node-failover'):
318
    RunTest(qa_node.TestNodeFailover, pnode, snode)
319

    
320
  if qa_config.TestEnabled('instance-disk-failure'):
321
    RunTest(qa_instance.TestInstanceMasterDiskFailure,
322
            instance, pnode, snode)
323
    RunTest(qa_instance.TestInstanceSecondaryDiskFailure,
324
            instance, pnode, snode)
325

    
326

    
327
@rapi.client.UsesRapiClient
328
def main():
329
  """Main program.
330

331
  """
332
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
333
  parser.add_option('--yes-do-it', dest='yes_do_it',
334
      action="store_true",
335
      help="Really execute the tests")
336
  (qa_config.options, args) = parser.parse_args()
337

    
338
  if len(args) == 1:
339
    (config_file, ) = args
340
  else:
341
    parser.error("Wrong number of arguments.")
342

    
343
  if not qa_config.options.yes_do_it:
344
    print ("Executing this script irreversibly destroys any Ganeti\n"
345
           "configuration on all nodes involved. If you really want\n"
346
           "to start testing, supply the --yes-do-it option.")
347
    sys.exit(1)
348

    
349
  qa_config.Load(config_file)
350

    
351
  rapi_user = "ganeti-qa"
352
  rapi_secret = utils.GenerateSecret()
353

    
354
  RunEnvTests()
355
  SetupCluster(rapi_user, rapi_secret)
356

    
357
  # Load RAPI certificate
358
  #qa_rapi.Setup(rapi_user, rapi_secret)
359

    
360
  RunClusterTests()
361
  RunOsTests()
362

    
363
  if qa_config.TestEnabled('tags'):
364
    RunTest(qa_tags.TestClusterTags)
365

    
366
  if qa_config.TestEnabled('node-readd'):
367
    master = qa_config.GetMasterNode()
368
    pnode = qa_config.AcquireNode(exclude=master)
369
    try:
370
      RunTest(qa_node.TestNodeReadd, pnode)
371
    finally:
372
      qa_config.ReleaseNode(pnode)
373

    
374
  pnode = qa_config.AcquireNode()
375
  try:
376
    if qa_config.TestEnabled('tags'):
377
      RunTest(qa_tags.TestNodeTags, pnode)
378

    
379
    if qa_rapi.Enabled():
380
      RunTest(qa_rapi.TestNode, pnode)
381

    
382
      if qa_config.TestEnabled("instance-add-plain-disk"):
383
        for use_client in [True, False]:
384
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
385
                                  use_client)
386
          RunCommonInstanceTests(rapi_instance)
387
          RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
388
          del rapi_instance
389

    
390
    if qa_config.TestEnabled('instance-add-plain-disk'):
391
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
392
      RunCommonInstanceTests(instance)
393
      RunExportImportTests(instance, pnode, None)
394
      RunDaemonTests(instance, pnode)
395
      RunTest(qa_instance.TestInstanceRemove, instance)
396
      del instance
397

    
398
    multinode_tests = [
399
      ('instance-add-drbd-disk',
400
       qa_instance.TestInstanceAddWithDrbdDisk),
401
    ]
402

    
403
    for name, func in multinode_tests:
404
      if qa_config.TestEnabled(name):
405
        snode = qa_config.AcquireNode(exclude=pnode)
406
        try:
407
          instance = RunTest(func, pnode, snode)
408
          RunCommonInstanceTests(instance)
409
          if qa_config.TestEnabled('instance-convert-disk'):
410
            RunTest(qa_instance.TestInstanceShutdown, instance)
411
            RunTest(qa_instance.TestInstanceConvertDisk, instance, snode)
412
            RunTest(qa_instance.TestInstanceStartup, instance)
413
          RunExportImportTests(instance, pnode, snode)
414
          RunHardwareFailureTests(instance, pnode, snode)
415
          RunTest(qa_instance.TestInstanceRemove, instance)
416
          del instance
417
        finally:
418
          qa_config.ReleaseNode(snode)
419

    
420
    if (qa_config.TestEnabled('instance-add-plain-disk') and
421
        qa_config.TestEnabled("instance-export")):
422
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
423
      expnode = qa_config.AcquireNode(exclude=pnode)
424
      try:
425
        RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
426
        RunTest(qa_instance.TestBackupList, expnode)
427
      finally:
428
        qa_config.ReleaseNode(expnode)
429
      del expnode
430
      del instance
431

    
432
  finally:
433
    qa_config.ReleaseNode(pnode)
434

    
435
  if qa_config.TestEnabled('create-cluster'):
436
    RunTest(qa_node.TestNodeRemoveAll)
437

    
438
  if qa_config.TestEnabled('cluster-destroy'):
439
    RunTest(qa_cluster.TestClusterDestroy)
440

    
441

    
442
if __name__ == '__main__':
443
  main()