Statistics
| Branch: | Tag: | Revision:

root / qa / ganeti-qa.py @ 1490a90c

History | View | Annotate | Download (31.9 kB)

1
#!/usr/bin/python -u
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Script for doing QA on Ganeti.
23

24
"""
25

    
26
# pylint: disable=C0103
27
# due to invalid name
28

    
29
import copy
30
import datetime
31
import optparse
32
import sys
33

    
34
import colors
35
import qa_cluster
36
import qa_config
37
import qa_daemon
38
import qa_env
39
import qa_error
40
import qa_group
41
import qa_instance
42
import qa_iptables
43
import qa_monitoring
44
import qa_network
45
import qa_node
46
import qa_os
47
import qa_job
48
import qa_rapi
49
import qa_tags
50
import qa_utils
51

    
52
from ganeti import utils
53
from ganeti import rapi # pylint: disable=W0611
54
from ganeti import constants
55
from ganeti import pathutils
56

    
57
from ganeti.http.auth import ParsePasswordFile
58
import ganeti.rapi.client # pylint: disable=W0611
59
from ganeti.rapi.client import UsesRapiClient
60

    
61

    
62
def _FormatHeader(line, end=72, mark="-", color=None):
63
  """Fill a line up to the end column.
64

65
  """
66
  line = (mark * 4) + " " + line + " "
67
  line += "-" * (end - len(line))
68
  line = line.rstrip()
69
  line = colors.colorize(line, color)
70
  return line
71

    
72

    
73
def _DescriptionOf(fn):
74
  """Computes the description of an item.
75

76
  """
77
  if fn.__doc__:
78
    desc = fn.__doc__.splitlines()[0].strip()
79
    desc = desc.rstrip(".")
80
    if fn.__name__:
81
      desc = "[" + fn.__name__ + "] " + desc
82
  else:
83
    desc = "%r" % fn
84

    
85
  return desc
86

    
87

    
88
def RunTest(fn, *args, **kwargs):
89
  """Runs a test after printing a header.
90

91
  """
92

    
93
  tstart = datetime.datetime.now()
94

    
95
  desc = _DescriptionOf(fn)
96

    
97
  print
98
  print _FormatHeader("%s start %s" % (tstart, desc),
99
                      color=colors.YELLOW, mark="<")
100

    
101
  try:
102
    retval = fn(*args, **kwargs)
103
    print _FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN)
104
    return retval
105
  except Exception, e:
106
    print _FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED)
107
    raise
108
  finally:
109
    tstop = datetime.datetime.now()
110
    tdelta = tstop - tstart
111
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc),
112
                        color=colors.MAGENTA, mark=">")
113

    
114

    
115
def RunTestIf(testnames, fn, *args, **kwargs):
116
  """Runs a test conditionally.
117

118
  @param testnames: either a single test name in the configuration
119
      file, or a list of testnames (which will be AND-ed together)
120

121
  """
122
  if qa_config.TestEnabled(testnames):
123
    RunTest(fn, *args, **kwargs)
124
  else:
125
    tstart = datetime.datetime.now()
126
    desc = _DescriptionOf(fn)
127
    # TODO: Formatting test names when non-string names are involved
128
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
129
                        (tstart, desc, testnames),
130
                        color=colors.BLUE, mark="*")
131

    
132

    
133
def RunEnvTests():
134
  """Run several environment tests.
135

136
  """
137
  RunTestIf("env", qa_env.TestSshConnection)
138
  RunTestIf("env", qa_env.TestIcmpPing)
139
  RunTestIf("env", qa_env.TestGanetiCommands)
140

    
141

    
142
def _LookupRapiSecret(rapi_user):
143
  """Find the RAPI secret for the given user.
144

145
  @param rapi_user: Login user
146
  @return: Login secret for the user
147

148
  """
149
  CTEXT = "{CLEARTEXT}"
150
  master = qa_config.GetMasterNode()
151
  cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)]
152
  file_content = qa_utils.GetCommandOutput(master.primary,
153
                                           utils.ShellQuoteArgs(cmd))
154
  users = ParsePasswordFile(file_content)
155
  entry = users.get(rapi_user)
156
  if not entry:
157
    raise qa_error.Error("User %s not found in RAPI users file" % rapi_user)
158
  secret = entry.password
159
  if secret.upper().startswith(CTEXT):
160
    secret = secret[len(CTEXT):]
161
  elif secret.startswith("{"):
162
    raise qa_error.Error("Unsupported password schema for RAPI user %s:"
163
                         " not a clear text password" % rapi_user)
164
  return secret
165

    
166

    
167
def SetupCluster(rapi_user):
168
  """Initializes the cluster.
169

170
  @param rapi_user: Login user for RAPI
171
  @return: Login secret for RAPI
172

173
  """
174
  rapi_secret = utils.GenerateSecret()
175
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
176
            rapi_user, rapi_secret)
177
  if not qa_config.TestEnabled("create-cluster"):
178
    # If the cluster is already in place, we assume that exclusive-storage is
179
    # already set according to the configuration
180
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
181
    if qa_rapi.Enabled():
182
      # To support RAPI on an existing cluster we have to find out the secret
183
      rapi_secret = _LookupRapiSecret(rapi_user)
184

    
185
  qa_group.ConfigureGroups()
186

    
187
  # Test on empty cluster
188
  RunTestIf("node-list", qa_node.TestNodeList)
189
  RunTestIf("instance-list", qa_instance.TestInstanceList)
190
  RunTestIf("job-list", qa_job.TestJobList)
191

    
192
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
193
  if not qa_config.TestEnabled("create-cluster"):
194
    # consider the nodes are already there
195
    qa_node.MarkNodeAddedAll()
196

    
197
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
198

    
199
  # enable the watcher (unconditionally)
200
  RunTest(qa_daemon.TestResumeWatcher)
201

    
202
  RunTestIf("node-list", qa_node.TestNodeList)
203

    
204
  # Test listing fields
205
  RunTestIf("node-list", qa_node.TestNodeListFields)
206
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
207
  RunTestIf("job-list", qa_job.TestJobListFields)
208
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
209

    
210
  RunTestIf("node-info", qa_node.TestNodeInfo)
211

    
212
  return rapi_secret
213

    
214

    
215
def RunClusterTests():
216
  """Runs tests related to gnt-cluster.
217

218
  """
219
  for test, fn in [
220
    ("create-cluster", qa_cluster.TestClusterInitDisk),
221
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
222
    ("cluster-verify", qa_cluster.TestClusterVerify),
223
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
224
    # TODO: add more cluster modify tests
225
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
226
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
227
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
228
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
229
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
230
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
231
    ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir),
232
    ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir),
233
    ("cluster-rename", qa_cluster.TestClusterRename),
234
    ("cluster-info", qa_cluster.TestClusterVersion),
235
    ("cluster-info", qa_cluster.TestClusterInfo),
236
    ("cluster-info", qa_cluster.TestClusterGetmaster),
237
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
238
    (["cluster-copyfile", qa_config.NoVirtualCluster],
239
     qa_cluster.TestClusterCopyfile),
240
    ("cluster-command", qa_cluster.TestClusterCommand),
241
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
242
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
243
    ("cluster-master-failover",
244
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
245
    (["cluster-oob", qa_config.NoVirtualCluster],
246
     qa_cluster.TestClusterOob),
247
    (qa_rapi.Enabled, qa_rapi.TestVersion),
248
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
249
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
250
    ]:
251
    RunTestIf(test, fn)
252

    
253

    
254
def RunRepairDiskSizes():
255
  """Run the repair disk-sizes test.
256

257
  """
258
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
259

    
260

    
261
def RunOsTests():
262
  """Runs all tests related to gnt-os.
263

264
  """
265
  os_enabled = ["os", qa_config.NoVirtualCluster]
266

    
267
  if qa_config.TestEnabled(qa_rapi.Enabled):
268
    rapi_getos = qa_rapi.GetOperatingSystems
269
  else:
270
    rapi_getos = None
271

    
272
  for fn in [
273
    qa_os.TestOsList,
274
    qa_os.TestOsDiagnose,
275
    ]:
276
    RunTestIf(os_enabled, fn)
277

    
278
  for fn in [
279
    qa_os.TestOsValid,
280
    qa_os.TestOsInvalid,
281
    qa_os.TestOsPartiallyValid,
282
    ]:
283
    RunTestIf(os_enabled, fn, rapi_getos)
284

    
285
  for fn in [
286
    qa_os.TestOsModifyValid,
287
    qa_os.TestOsModifyInvalid,
288
    qa_os.TestOsStatesNonExisting,
289
    ]:
290
    RunTestIf(os_enabled, fn)
291

    
292

    
293
def RunCommonInstanceTests(instance, inst_nodes):
294
  """Runs a few tests that are common to all disk types.
295

296
  """
297
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
298
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
299
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
300
  RunTestIf(["instance-shutdown", "instance-modify"],
301
            qa_instance.TestInstanceStoppedModify, instance)
302
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
303

    
304
  # Test shutdown/start via RAPI
305
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
306
            qa_rapi.TestRapiInstanceShutdown, instance)
307
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
308
            qa_rapi.TestRapiInstanceStartup, instance)
309

    
310
  RunTestIf("instance-list", qa_instance.TestInstanceList)
311

    
312
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
313

    
314
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
315
  RunTestIf(["instance-modify", qa_rapi.Enabled],
316
            qa_rapi.TestRapiInstanceModify, instance)
317

    
318
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
319
  RunTestIf(["instance-console", qa_rapi.Enabled],
320
            qa_rapi.TestRapiInstanceConsole, instance)
321

    
322
  RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
323
            instance)
324
  DOWN_TESTS = qa_config.Either([
325
    "instance-reinstall",
326
    "instance-rename",
327
    "instance-grow-disk",
328
    ])
329

    
330
  # shutdown instance for any 'down' tests
331
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)
332

    
333
  # now run the 'down' state tests
334
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
335
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
336
            qa_rapi.TestRapiInstanceReinstall, instance)
337

    
338
  if qa_config.TestEnabled("instance-rename"):
339
    tgt_instance = qa_config.AcquireInstance()
340
    try:
341
      rename_source = instance.name
342
      rename_target = tgt_instance.name
343
      # perform instance rename to the same name
344
      RunTest(qa_instance.TestInstanceRenameAndBack,
345
              rename_source, rename_source)
346
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
347
                rename_source, rename_source)
348
      if rename_target is not None:
349
        # perform instance rename to a different name, if we have one configured
350
        RunTest(qa_instance.TestInstanceRenameAndBack,
351
                rename_source, rename_target)
352
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
353
                  rename_source, rename_target)
354
    finally:
355
      tgt_instance.Release()
356

    
357
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)
358

    
359
  # and now start the instance again
360
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)
361

    
362
  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
363

    
364
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
365

    
366
  if instance.disk_template == constants.DT_DRBD8:
367
    RunTestIf("cluster-verify",
368
              qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes)
369
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
370

    
371
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
372

    
373
  # Lists instances, too
374
  RunTestIf("node-list", qa_node.TestNodeList)
375

    
376
  # Some jobs have been run, let's test listing them
377
  RunTestIf("job-list", qa_job.TestJobList)
378

    
379

    
380
def RunCommonNodeTests():
381
  """Run a few common node tests.
382

383
  """
384
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
385
  RunTestIf("node-storage", qa_node.TestNodeStorage)
386
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
387

    
388

    
389
def RunGroupListTests():
390
  """Run tests for listing node groups.
391

392
  """
393
  RunTestIf("group-list", qa_group.TestGroupList)
394
  RunTestIf("group-list", qa_group.TestGroupListFields)
395

    
396

    
397
def RunNetworkTests():
398
  """Run tests for network management.
399

400
  """
401
  RunTestIf("network", qa_network.TestNetworkAddRemove)
402
  RunTestIf("network", qa_network.TestNetworkConnect)
403

    
404

    
405
def RunGroupRwTests():
406
  """Run tests for adding/removing/renaming groups.
407

408
  """
409
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
410
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
411
  RunTestIf("group-rwops", qa_group.TestGroupModify)
412
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
413
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
414
            qa_group.GetDefaultGroup())
415

    
416

    
417
def RunExportImportTests(instance, inodes):
418
  """Tries to export and import the instance.
419

420
  @type inodes: list of nodes
421
  @param inodes: current nodes of the instance
422

423
  """
424
  # FIXME: export explicitly bails out on file based storage. other non-lvm
425
  # based storage types are untested, though. Also note that import could still
426
  # work, but is deeply embedded into the "export" case.
427
  if (qa_config.TestEnabled("instance-export") and
428
      instance.disk_template not in constants.DTS_FILEBASED):
429
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)
430

    
431
    pnode = inodes[0]
432
    expnode = qa_config.AcquireNode(exclude=pnode)
433
    try:
434
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
435

    
436
      RunTest(qa_instance.TestBackupList, expnode)
437

    
438
      if qa_config.TestEnabled("instance-import"):
439
        newinst = qa_config.AcquireInstance()
440
        try:
441
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
442
                  expnode, name)
443
          # Check if starting the instance works
444
          RunTest(qa_instance.TestInstanceStartup, newinst)
445
          RunTest(qa_instance.TestInstanceRemove, newinst)
446
        finally:
447
          newinst.Release()
448
    finally:
449
      expnode.Release()
450

    
451
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
452
  # See Issue 414.
453
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
454
      and (instance.disk_template not in constants.DTS_FILEBASED)):
455
    newinst = qa_config.AcquireInstance()
456
    try:
457
      tnode = qa_config.AcquireNode(exclude=inodes)
458
      try:
459
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
460
                inodes, tnode)
461
      finally:
462
        tnode.Release()
463
    finally:
464
      newinst.Release()
465

    
466

    
467
def RunDaemonTests(instance):
468
  """Test the ganeti-watcher script.
469

470
  """
471
  RunTest(qa_daemon.TestPauseWatcher)
472

    
473
  RunTestIf("instance-automatic-restart",
474
            qa_daemon.TestInstanceAutomaticRestart, instance)
475
  RunTestIf("instance-consecutive-failures",
476
            qa_daemon.TestInstanceConsecutiveFailures, instance)
477

    
478
  RunTest(qa_daemon.TestResumeWatcher)
479

    
480

    
481
def RunHardwareFailureTests(instance, inodes):
482
  """Test cluster internal hardware failure recovery.
483

484
  """
485
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
486
  RunTestIf(["instance-failover", qa_rapi.Enabled],
487
            qa_rapi.TestRapiInstanceFailover, instance)
488

    
489
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
490
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
491
            qa_rapi.TestRapiInstanceMigrate, instance)
492

    
493
  if qa_config.TestEnabled("instance-replace-disks"):
494
    # We just need alternative secondary nodes, hence "- 1"
495
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
496
    try:
497
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
498
      RunTest(qa_instance.TestReplaceDisks,
499
              instance, inodes, othernodes)
500
    finally:
501
      qa_config.ReleaseManyNodes(othernodes)
502
    del othernodes
503

    
504
  if qa_config.TestEnabled("instance-recreate-disks"):
505
    try:
506
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
507
      othernodes = acquirednodes
508
    except qa_error.OutOfNodesError:
509
      if len(inodes) > 1:
510
        # If the cluster is not big enough, let's reuse some of the nodes, but
511
        # with different roles. In this way, we can test a DRBD instance even on
512
        # a 3-node cluster.
513
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
514
        othernodes = acquirednodes + inodes[:-1]
515
      else:
516
        raise
517
    try:
518
      RunTest(qa_instance.TestRecreateDisks,
519
              instance, inodes, othernodes)
520
    finally:
521
      qa_config.ReleaseManyNodes(acquirednodes)
522

    
523
  if len(inodes) >= 2:
524
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
525
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
526
    RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1])
527

    
528

    
529
def RunExclusiveStorageTests():
530
  """Test exclusive storage."""
531
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
532
    return
533

    
534
  node = qa_config.AcquireNode()
535
  try:
536
    old_es = qa_cluster.TestSetExclStorCluster(False)
537
    qa_node.TestExclStorSingleNode(node)
538

    
539
    qa_cluster.TestSetExclStorCluster(True)
540
    qa_cluster.TestExclStorSharedPv(node)
541

    
542
    if qa_config.TestEnabled("instance-add-plain-disk"):
543
      # Make sure that the cluster doesn't have any pre-existing problem
544
      qa_cluster.AssertClusterVerify()
545

    
546
      # Create and allocate instances
547
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
548
      try:
549
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
550
        try:
551
          # cluster-verify checks that disks are allocated correctly
552
          qa_cluster.AssertClusterVerify()
553

    
554
          # Remove instances
555
          qa_instance.TestInstanceRemove(instance2)
556
          qa_instance.TestInstanceRemove(instance1)
557
        finally:
558
          instance2.Release()
559
      finally:
560
        instance1.Release()
561

    
562
    if qa_config.TestEnabled("instance-add-drbd-disk"):
563
      snode = qa_config.AcquireNode()
564
      try:
565
        qa_cluster.TestSetExclStorCluster(False)
566
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
567
        try:
568
          qa_cluster.TestSetExclStorCluster(True)
569
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
570
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
571
          qa_instance.TestInstanceRemove(instance)
572
        finally:
573
          instance.Release()
574
      finally:
575
        snode.Release()
576
    qa_cluster.TestSetExclStorCluster(old_es)
577
  finally:
578
    node.Release()
579

    
580

    
581
def RunCustomSshPortTests():
582
  """Test accessing nodes with custom SSH ports.
583

584
  This requires removing nodes, adding them to a new group, and then undoing
585
  the change.
586
  """
587
  if not qa_config.TestEnabled("group-custom-ssh-port"):
588
    return
589

    
590
  port = 211
591
  master = qa_config.GetMasterNode()
592
  with qa_config.AcquireManyNodesCtx(1, exclude=master) as nodes:
593
    for node in nodes:
594
      qa_node.NodeRemove(node)
595
    with qa_iptables.RulesContext(nodes) as r:
596
      with qa_group.NewGroupCtx() as group:
597
        qa_group.ModifyGroupSshPort(r, group, nodes, port)
598

    
599
        for node in nodes:
600
          qa_node.NodeAdd(node, group=group)
601

    
602
        # Make sure that the cluster doesn't have any pre-existing problem
603
        qa_cluster.AssertClusterVerify()
604

    
605
        # Create and allocate instances
606
        instance1 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
607
        try:
608
          instance2 = qa_instance.TestInstanceAddWithPlainDisk(nodes)
609
          try:
610
            # cluster-verify checks that disks are allocated correctly
611
            qa_cluster.AssertClusterVerify()
612

    
613
            # Remove instances
614
            qa_instance.TestInstanceRemove(instance2)
615
            qa_instance.TestInstanceRemove(instance1)
616
          finally:
617
            instance2.Release()
618
        finally:
619
          instance1.Release()
620

    
621
        for node in nodes:
622
          qa_node.NodeRemove(node)
623

    
624
    for node in nodes:
625
      qa_node.NodeAdd(node)
626

    
627
    qa_cluster.AssertClusterVerify()
628

    
629

    
630
def _BuildSpecDict(par, mn, st, mx):
631
  return {
632
    constants.ISPECS_MINMAX: [{
633
      constants.ISPECS_MIN: {par: mn},
634
      constants.ISPECS_MAX: {par: mx},
635
      }],
636
    constants.ISPECS_STD: {par: st},
637
    }
638

    
639

    
640
def _BuildDoubleSpecDict(index, par, mn, st, mx):
641
  new_spec = {
642
    constants.ISPECS_MINMAX: [{}, {}],
643
    }
644
  if st is not None:
645
    new_spec[constants.ISPECS_STD] = {par: st}
646
  new_spec[constants.ISPECS_MINMAX][index] = {
647
    constants.ISPECS_MIN: {par: mn},
648
    constants.ISPECS_MAX: {par: mx},
649
    }
650
  return new_spec
651

    
652

    
653
def TestIPolicyPlainInstance():
654
  """Test instance policy interaction with instances"""
655
  params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
656
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
657
    print "Template %s not supported" % constants.DT_PLAIN
658
    return
659

    
660
  # This test assumes that the group policy is empty
661
  (_, old_specs) = qa_cluster.TestClusterSetISpecs()
662
  # We also assume to have only one min/max bound
663
  assert len(old_specs[constants.ISPECS_MINMAX]) == 1
664
  node = qa_config.AcquireNode()
665
  try:
666
    # Log of policy changes, list of tuples:
667
    # (full_change, incremental_change, policy_violated)
668
    history = []
669
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
670
    try:
671
      policyerror = [constants.CV_EINSTANCEPOLICY]
672
      for par in params:
673
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
674
        # Some specs must be multiple of 4
675
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
676
        history.append((None, new_spec, True))
677
        if iminval > 0:
678
          # Some specs must be multiple of 4
679
          if iminval >= 4:
680
            upper = iminval - 4
681
          else:
682
            upper = iminval - 1
683
          new_spec = _BuildSpecDict(par, 0, upper, upper)
684
          history.append((None, new_spec, True))
685
        history.append((old_specs, None, False))
686

    
687
      # Test with two instance specs
688
      double_specs = copy.deepcopy(old_specs)
689
      double_specs[constants.ISPECS_MINMAX] = \
690
          double_specs[constants.ISPECS_MINMAX] * 2
691
      (par1, par2) = params[0:2]
692
      (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1)
693
      (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2)
694
      old_minmax = old_specs[constants.ISPECS_MINMAX][0]
695
      history.extend([
696
        (double_specs, None, False),
697
        # The first min/max limit is being violated
698
        (None,
699
         _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4,
700
                              imaxval1 + 4),
701
         False),
702
        # Both min/max limits are being violated
703
        (None,
704
         _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4),
705
         True),
706
        # The second min/max limit is being violated
707
        (None,
708
         _BuildDoubleSpecDict(0, par1,
709
                              old_minmax[constants.ISPECS_MIN][par1],
710
                              old_specs[constants.ISPECS_STD][par1],
711
                              old_minmax[constants.ISPECS_MAX][par1]),
712
         False),
713
        (old_specs, None, False),
714
        ])
715

    
716
      # Apply the changes, and check policy violations after each change
717
      qa_cluster.AssertClusterVerify()
718
      for (new_specs, diff_specs, failed) in history:
719
        qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
720
                                        diff_specs=diff_specs)
721
        if failed:
722
          qa_cluster.AssertClusterVerify(warnings=policyerror)
723
        else:
724
          qa_cluster.AssertClusterVerify()
725

    
726
      qa_instance.TestInstanceRemove(instance)
727
    finally:
728
      instance.Release()
729

    
730
    # Now we replay the same policy changes, and we expect that the instance
731
    # cannot be created for the cases where we had a policy violation above
732
    for (new_specs, diff_specs, failed) in history:
733
      qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
734
                                      diff_specs=diff_specs)
735
      if failed:
736
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
737
      # Instance creation with no policy violation has been tested already
738
  finally:
739
    node.Release()
740

    
741

    
742
def IsExclusiveStorageInstanceTestEnabled():
743
  test_name = "exclusive-storage-instance-tests"
744
  if qa_config.TestEnabled(test_name):
745
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
746
    vgscmd = utils.ShellQuoteArgs([
747
      "vgs", "--noheadings", "-o", "pv_count", vgname,
748
      ])
749
    nodes = qa_config.GetConfig()["nodes"]
750
    for node in nodes:
751
      try:
752
        pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
753
      except Exception, e:
754
        msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
755
               (node.primary, test_name, e))
756
        raise qa_error.Error(msg)
757
      if pvnum < 2:
758
        raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" %
759
                             (node.primary, pvnum, test_name))
760
    res = True
761
  else:
762
    res = False
763
  return res
764

    
765

    
766
def RunInstanceTests():
767
  """Create and exercise instances."""
768

    
769
  for (test_name, templ, create_fun, num_nodes) in \
770
      qa_instance.available_instance_tests:
771
    if (qa_config.TestEnabled(test_name) and
772
        qa_config.IsTemplateSupported(templ)):
773
      inodes = qa_config.AcquireManyNodes(num_nodes)
774
      try:
775
        instance = RunTest(create_fun, inodes)
776
        try:
777
          RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
778
                    instance, qa_config.GetMasterNode())
779
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
780
          RunDaemonTests(instance)
781
          for node in inodes:
782
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
783
          if len(inodes) > 1:
784
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
785
                      constants.INITIAL_NODE_GROUP_NAME,
786
                      inodes[0].primary, inodes[1].primary)
787
          if qa_config.TestEnabled("instance-convert-disk"):
788
            RunTest(qa_instance.TestInstanceShutdown, instance)
789
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
790
                    instance, inodes)
791
            RunTest(qa_instance.TestInstanceStartup, instance)
792
          RunTestIf("instance-modify-disks",
793
                    qa_instance.TestInstanceModifyDisks, instance)
794
          RunCommonInstanceTests(instance, inodes)
795
          if qa_config.TestEnabled("instance-modify-primary"):
796
            othernode = qa_config.AcquireNode()
797
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
798
                    instance, inodes[0], othernode)
799
            othernode.Release()
800
          RunGroupListTests()
801
          RunExportImportTests(instance, inodes)
802
          RunHardwareFailureTests(instance, inodes)
803
          RunRepairDiskSizes()
804
          RunTest(qa_instance.TestInstanceRemove, instance)
805
        finally:
806
          instance.Release()
807
        del instance
808
      finally:
809
        qa_config.ReleaseManyNodes(inodes)
810
      qa_cluster.AssertClusterVerify()
811

    
812

    
813
def RunMonitoringTests():
814
  if qa_config.TestEnabled("mon-collector"):
815
    RunTest(qa_monitoring.TestInstStatusCollector)
816

    
817

    
818
def RunQa():
819
  """Main QA body.
820

821
  """
822
  rapi_user = "ganeti-qa"
823

    
824
  RunEnvTests()
825
  rapi_secret = SetupCluster(rapi_user)
826

    
827
  if qa_rapi.Enabled():
828
    # Load RAPI certificate
829
    qa_rapi.Setup(rapi_user, rapi_secret)
830

    
831
  RunClusterTests()
832
  RunOsTests()
833

    
834
  RunTestIf("tags", qa_tags.TestClusterTags)
835

    
836
  RunCommonNodeTests()
837
  RunGroupListTests()
838
  RunGroupRwTests()
839
  RunNetworkTests()
840

    
841
  # The master shouldn't be readded or put offline; "delay" needs a non-master
842
  # node to test
843
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
844
  try:
845
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
846
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
847
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
848
  finally:
849
    pnode.Release()
850

    
851
  # Make sure the cluster is clean before running instance tests
852
  qa_cluster.AssertClusterVerify()
853

    
854
  pnode = qa_config.AcquireNode()
855
  try:
856
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
857

    
858
    if qa_rapi.Enabled():
859
      RunTest(qa_rapi.TestNode, pnode)
860

    
861
      if (qa_config.TestEnabled("instance-add-plain-disk")
862
          and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
863
        for use_client in [True, False]:
864
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
865
                                  use_client)
866
          try:
867
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
868
              RunCommonInstanceTests(rapi_instance, [pnode])
869
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
870
          finally:
871
            rapi_instance.Release()
872
          del rapi_instance
873

    
874
  finally:
875
    pnode.Release()
876

    
877
  config_list = [
878
    ("default-instance-tests", lambda: None, lambda _: None),
879
    (IsExclusiveStorageInstanceTestEnabled,
880
     lambda: qa_cluster.TestSetExclStorCluster(True),
881
     qa_cluster.TestSetExclStorCluster),
882
  ]
883
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
884
    if qa_config.TestEnabled(conf_name):
885
      oldconf = setup_conf_f()
886
      RunInstanceTests()
887
      restore_conf_f(oldconf)
888

    
889
  pnode = qa_config.AcquireNode()
890
  try:
891
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
892
      for shutdown in [False, True]:
893
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
894
        try:
895
          expnode = qa_config.AcquireNode(exclude=pnode)
896
          try:
897
            if shutdown:
898
              # Stop instance before exporting and removing it
899
              RunTest(qa_instance.TestInstanceShutdown, instance)
900
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
901
            RunTest(qa_instance.TestBackupList, expnode)
902
          finally:
903
            expnode.Release()
904
        finally:
905
          instance.Release()
906
        del expnode
907
        del instance
908
      qa_cluster.AssertClusterVerify()
909

    
910
  finally:
911
    pnode.Release()
912

    
913
  RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade)
914

    
915
  RunExclusiveStorageTests()
916
  RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
917
            TestIPolicyPlainInstance)
918

    
919
  RunCustomSshPortTests()
920

    
921
  RunTestIf(
922
    "instance-add-restricted-by-disktemplates",
923
    qa_instance.TestInstanceCreationRestrictedByDiskTemplates)
924

    
925
  # Test removing instance with offline drbd secondary
926
  if qa_config.TestEnabled(["instance-remove-drbd-offline",
927
                            "instance-add-drbd-disk"]):
928
    # Make sure the master is not put offline
929
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
930
    try:
931
      pnode = qa_config.AcquireNode(exclude=snode)
932
      try:
933
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
934
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
935
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
936
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
937
                set_offline, set_online)
938
      finally:
939
        pnode.Release()
940
    finally:
941
      snode.Release()
942
    qa_cluster.AssertClusterVerify()
943

    
944
  RunMonitoringTests()
945

    
946
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
947

    
948
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
949

    
950

    
951
@UsesRapiClient
952
def main():
953
  """Main program.
954

955
  """
956
  colors.check_for_colors()
957

    
958
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
959
  parser.add_option("--yes-do-it", dest="yes_do_it",
960
                    action="store_true",
961
                    help="Really execute the tests")
962
  (opts, args) = parser.parse_args()
963

    
964
  if len(args) == 1:
965
    (config_file, ) = args
966
  else:
967
    parser.error("Wrong number of arguments.")
968

    
969
  if not opts.yes_do_it:
970
    print ("Executing this script irreversibly destroys any Ganeti\n"
971
           "configuration on all nodes involved. If you really want\n"
972
           "to start testing, supply the --yes-do-it option.")
973
    sys.exit(1)
974

    
975
  qa_config.Load(config_file)
976

    
977
  primary = qa_config.GetMasterNode().primary
978
  qa_utils.StartMultiplexer(primary)
979
  print ("SSH command for primary node: %s" %
980
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
981
  print ("SSH command for other nodes: %s" %
982
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
983
  try:
984
    RunQa()
985
  finally:
986
    qa_utils.CloseMultiplexers()
987

    
988
if __name__ == "__main__":
989
  main()