Statistics
| Branch: | Tag: | Revision:

root / qa / ganeti-qa.py @ ec7b6d63

History | View | Annotate | Download (30.4 kB)

1
#!/usr/bin/python -u
2
#
3

    
4
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Script for doing QA on Ganeti.
23

24
"""
25

    
26
# pylint: disable=C0103
27
# due to invalid name
28

    
29
import copy
30
import datetime
31
import optparse
32
import sys
33

    
34
import colors
35
import qa_cluster
36
import qa_config
37
import qa_daemon
38
import qa_env
39
import qa_error
40
import qa_group
41
import qa_instance
42
import qa_monitoring
43
import qa_network
44
import qa_node
45
import qa_os
46
import qa_job
47
import qa_rapi
48
import qa_tags
49
import qa_utils
50

    
51
from ganeti import utils
52
from ganeti import rapi # pylint: disable=W0611
53
from ganeti import constants
54
from ganeti import pathutils
55

    
56
from ganeti.http.auth import ParsePasswordFile
57
import ganeti.rapi.client # pylint: disable=W0611
58
from ganeti.rapi.client import UsesRapiClient
59

    
60

    
61
def _FormatHeader(line, end=72, mark="-", color=None):
62
  """Fill a line up to the end column.
63

64
  """
65
  line = (mark * 4) + " " + line + " "
66
  line += "-" * (end - len(line))
67
  line = line.rstrip()
68
  line = colors.colorize(line, color)
69
  return line
70

    
71

    
72
def _DescriptionOf(fn):
73
  """Computes the description of an item.
74

75
  """
76
  if fn.__doc__:
77
    desc = fn.__doc__.splitlines()[0].strip()
78
    desc = desc.rstrip(".")
79
    if fn.__name__:
80
      desc = "[" + fn.__name__ + "] " + desc
81
  else:
82
    desc = "%r" % fn
83

    
84
  return desc
85

    
86

    
87
def RunTest(fn, *args, **kwargs):
88
  """Runs a test after printing a header.
89

90
  """
91

    
92
  tstart = datetime.datetime.now()
93

    
94
  desc = _DescriptionOf(fn)
95

    
96
  print
97
  print _FormatHeader("%s start %s" % (tstart, desc),
98
                      color=colors.YELLOW, mark="<")
99

    
100
  try:
101
    retval = fn(*args, **kwargs)
102
    print _FormatHeader("PASSED %s" % (desc, ), color=colors.GREEN)
103
    return retval
104
  except Exception, e:
105
    print _FormatHeader("FAILED %s: %s" % (desc, e), color=colors.RED)
106
    raise
107
  finally:
108
    tstop = datetime.datetime.now()
109
    tdelta = tstop - tstart
110
    print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc),
111
                        color=colors.MAGENTA, mark=">")
112

    
113

    
114
def RunTestIf(testnames, fn, *args, **kwargs):
115
  """Runs a test conditionally.
116

117
  @param testnames: either a single test name in the configuration
118
      file, or a list of testnames (which will be AND-ed together)
119

120
  """
121
  if qa_config.TestEnabled(testnames):
122
    RunTest(fn, *args, **kwargs)
123
  else:
124
    tstart = datetime.datetime.now()
125
    desc = _DescriptionOf(fn)
126
    # TODO: Formatting test names when non-string names are involved
127
    print _FormatHeader("%s skipping %s, test(s) %s disabled" %
128
                        (tstart, desc, testnames),
129
                        color=colors.BLUE, mark="*")
130

    
131

    
132
def RunEnvTests():
133
  """Run several environment tests.
134

135
  """
136
  RunTestIf("env", qa_env.TestSshConnection)
137
  RunTestIf("env", qa_env.TestIcmpPing)
138
  RunTestIf("env", qa_env.TestGanetiCommands)
139

    
140

    
141
def _LookupRapiSecret(rapi_user):
142
  """Find the RAPI secret for the given user.
143

144
  @param rapi_user: Login user
145
  @return: Login secret for the user
146

147
  """
148
  CTEXT = "{CLEARTEXT}"
149
  master = qa_config.GetMasterNode()
150
  cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)]
151
  file_content = qa_utils.GetCommandOutput(master.primary,
152
                                           utils.ShellQuoteArgs(cmd))
153
  users = ParsePasswordFile(file_content)
154
  entry = users.get(rapi_user)
155
  if not entry:
156
    raise qa_error.Error("User %s not found in RAPI users file" % rapi_user)
157
  secret = entry.password
158
  if secret.upper().startswith(CTEXT):
159
    secret = secret[len(CTEXT):]
160
  elif secret.startswith("{"):
161
    raise qa_error.Error("Unsupported password schema for RAPI user %s:"
162
                         " not a clear text password" % rapi_user)
163
  return secret
164

    
165

    
166
def SetupCluster(rapi_user):
167
  """Initializes the cluster.
168

169
  @param rapi_user: Login user for RAPI
170
  @return: Login secret for RAPI
171

172
  """
173
  rapi_secret = utils.GenerateSecret()
174
  RunTestIf("create-cluster", qa_cluster.TestClusterInit,
175
            rapi_user, rapi_secret)
176
  if not qa_config.TestEnabled("create-cluster"):
177
    # If the cluster is already in place, we assume that exclusive-storage is
178
    # already set according to the configuration
179
    qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False))
180
    if qa_rapi.Enabled():
181
      # To support RAPI on an existing cluster we have to find out the secret
182
      rapi_secret = _LookupRapiSecret(rapi_user)
183

    
184
  qa_group.ConfigureGroups()
185

    
186
  # Test on empty cluster
187
  RunTestIf("node-list", qa_node.TestNodeList)
188
  RunTestIf("instance-list", qa_instance.TestInstanceList)
189
  RunTestIf("job-list", qa_job.TestJobList)
190

    
191
  RunTestIf("create-cluster", qa_node.TestNodeAddAll)
192
  if not qa_config.TestEnabled("create-cluster"):
193
    # consider the nodes are already there
194
    qa_node.MarkNodeAddedAll()
195

    
196
  RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
197

    
198
  # enable the watcher (unconditionally)
199
  RunTest(qa_daemon.TestResumeWatcher)
200

    
201
  RunTestIf("node-list", qa_node.TestNodeList)
202

    
203
  # Test listing fields
204
  RunTestIf("node-list", qa_node.TestNodeListFields)
205
  RunTestIf("instance-list", qa_instance.TestInstanceListFields)
206
  RunTestIf("job-list", qa_job.TestJobListFields)
207
  RunTestIf("instance-export", qa_instance.TestBackupListFields)
208

    
209
  RunTestIf("node-info", qa_node.TestNodeInfo)
210

    
211
  return rapi_secret
212

    
213

    
214
def RunClusterTests():
215
  """Runs tests related to gnt-cluster.
216

217
  """
218
  for test, fn in [
219
    ("create-cluster", qa_cluster.TestClusterInitDisk),
220
    ("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
221
    ("cluster-verify", qa_cluster.TestClusterVerify),
222
    ("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
223
    # TODO: add more cluster modify tests
224
    ("cluster-modify", qa_cluster.TestClusterModifyEmpty),
225
    ("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
226
    ("cluster-modify", qa_cluster.TestClusterModifyISpecs),
227
    ("cluster-modify", qa_cluster.TestClusterModifyBe),
228
    ("cluster-modify", qa_cluster.TestClusterModifyDisk),
229
    ("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
230
    ("cluster-modify", qa_cluster.TestClusterModifyFileStorageDir),
231
    ("cluster-modify", qa_cluster.TestClusterModifySharedFileStorageDir),
232
    ("cluster-rename", qa_cluster.TestClusterRename),
233
    ("cluster-info", qa_cluster.TestClusterVersion),
234
    ("cluster-info", qa_cluster.TestClusterInfo),
235
    ("cluster-info", qa_cluster.TestClusterGetmaster),
236
    ("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
237
    (["cluster-copyfile", qa_config.NoVirtualCluster],
238
     qa_cluster.TestClusterCopyfile),
239
    ("cluster-command", qa_cluster.TestClusterCommand),
240
    ("cluster-burnin", qa_cluster.TestClusterBurnin),
241
    ("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
242
    ("cluster-master-failover",
243
     qa_cluster.TestClusterMasterFailoverWithDrainedQueue),
244
    (["cluster-oob", qa_config.NoVirtualCluster],
245
     qa_cluster.TestClusterOob),
246
    (qa_rapi.Enabled, qa_rapi.TestVersion),
247
    (qa_rapi.Enabled, qa_rapi.TestEmptyCluster),
248
    (qa_rapi.Enabled, qa_rapi.TestRapiQuery),
249
    ]:
250
    RunTestIf(test, fn)
251

    
252

    
253
def RunRepairDiskSizes():
254
  """Run the repair disk-sizes test.
255

256
  """
257
  RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
258

    
259

    
260
def RunOsTests():
261
  """Runs all tests related to gnt-os.
262

263
  """
264
  os_enabled = ["os", qa_config.NoVirtualCluster]
265

    
266
  if qa_config.TestEnabled(qa_rapi.Enabled):
267
    rapi_getos = qa_rapi.GetOperatingSystems
268
  else:
269
    rapi_getos = None
270

    
271
  for fn in [
272
    qa_os.TestOsList,
273
    qa_os.TestOsDiagnose,
274
    ]:
275
    RunTestIf(os_enabled, fn)
276

    
277
  for fn in [
278
    qa_os.TestOsValid,
279
    qa_os.TestOsInvalid,
280
    qa_os.TestOsPartiallyValid,
281
    ]:
282
    RunTestIf(os_enabled, fn, rapi_getos)
283

    
284
  for fn in [
285
    qa_os.TestOsModifyValid,
286
    qa_os.TestOsModifyInvalid,
287
    qa_os.TestOsStatesNonExisting,
288
    ]:
289
    RunTestIf(os_enabled, fn)
290

    
291

    
292
def RunCommonInstanceTests(instance, inst_nodes):
293
  """Runs a few tests that are common to all disk types.
294

295
  """
296
  RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
297
  RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled],
298
            qa_rapi.TestRapiStoppedInstanceConsole, instance)
299
  RunTestIf(["instance-shutdown", "instance-modify"],
300
            qa_instance.TestInstanceStoppedModify, instance)
301
  RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
302

    
303
  # Test shutdown/start via RAPI
304
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
305
            qa_rapi.TestRapiInstanceShutdown, instance)
306
  RunTestIf(["instance-shutdown", qa_rapi.Enabled],
307
            qa_rapi.TestRapiInstanceStartup, instance)
308

    
309
  RunTestIf("instance-list", qa_instance.TestInstanceList)
310

    
311
  RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
312

    
313
  RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
314
  RunTestIf(["instance-modify", qa_rapi.Enabled],
315
            qa_rapi.TestRapiInstanceModify, instance)
316

    
317
  RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
318
  RunTestIf(["instance-console", qa_rapi.Enabled],
319
            qa_rapi.TestRapiInstanceConsole, instance)
320

    
321
  RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
322
            instance)
323
  DOWN_TESTS = qa_config.Either([
324
    "instance-reinstall",
325
    "instance-rename",
326
    "instance-grow-disk",
327
    ])
328

    
329
  # shutdown instance for any 'down' tests
330
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance)
331

    
332
  # now run the 'down' state tests
333
  RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
334
  RunTestIf(["instance-reinstall", qa_rapi.Enabled],
335
            qa_rapi.TestRapiInstanceReinstall, instance)
336

    
337
  if qa_config.TestEnabled("instance-rename"):
338
    tgt_instance = qa_config.AcquireInstance()
339
    try:
340
      rename_source = instance.name
341
      rename_target = tgt_instance.name
342
      # perform instance rename to the same name
343
      RunTest(qa_instance.TestInstanceRenameAndBack,
344
              rename_source, rename_source)
345
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
346
                rename_source, rename_source)
347
      if rename_target is not None:
348
        # perform instance rename to a different name, if we have one configured
349
        RunTest(qa_instance.TestInstanceRenameAndBack,
350
                rename_source, rename_target)
351
        RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack,
352
                  rename_source, rename_target)
353
    finally:
354
      tgt_instance.Release()
355

    
356
  RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)
357

    
358
  # and now start the instance again
359
  RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance)
360

    
361
  RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
362

    
363
  RunTestIf("tags", qa_tags.TestInstanceTags, instance)
364

    
365
  if instance.disk_template == constants.DT_DRBD8:
366
    RunTestIf("cluster-verify",
367
              qa_cluster.TestClusterVerifyDisksBrokenDRBD, instance, inst_nodes)
368
  RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
369

    
370
  RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance)
371

    
372
  # Lists instances, too
373
  RunTestIf("node-list", qa_node.TestNodeList)
374

    
375
  # Some jobs have been run, let's test listing them
376
  RunTestIf("job-list", qa_job.TestJobList)
377

    
378

    
379
def RunCommonNodeTests():
380
  """Run a few common node tests.
381

382
  """
383
  RunTestIf("node-volumes", qa_node.TestNodeVolumes)
384
  RunTestIf("node-storage", qa_node.TestNodeStorage)
385
  RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
386

    
387

    
388
def RunGroupListTests():
389
  """Run tests for listing node groups.
390

391
  """
392
  RunTestIf("group-list", qa_group.TestGroupList)
393
  RunTestIf("group-list", qa_group.TestGroupListFields)
394

    
395

    
396
def RunNetworkTests():
397
  """Run tests for network management.
398

399
  """
400
  RunTestIf("network", qa_network.TestNetworkAddRemove)
401
  RunTestIf("network", qa_network.TestNetworkConnect)
402

    
403

    
404
def RunGroupRwTests():
405
  """Run tests for adding/removing/renaming groups.
406

407
  """
408
  RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
409
  RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
410
  RunTestIf("group-rwops", qa_group.TestGroupModify)
411
  RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
412
  RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags,
413
            qa_group.GetDefaultGroup())
414

    
415

    
416
def RunExportImportTests(instance, inodes):
417
  """Tries to export and import the instance.
418

419
  @type inodes: list of nodes
420
  @param inodes: current nodes of the instance
421

422
  """
423
  # FIXME: export explicitly bails out on file based storage. other non-lvm
424
  # based storage types are untested, though. Also note that import could still
425
  # work, but is deeply embedded into the "export" case.
426
  if (qa_config.TestEnabled("instance-export") and
427
      instance.disk_template not in constants.DTS_FILEBASED):
428
    RunTest(qa_instance.TestInstanceExportNoTarget, instance)
429

    
430
    pnode = inodes[0]
431
    expnode = qa_config.AcquireNode(exclude=pnode)
432
    try:
433
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
434

    
435
      RunTest(qa_instance.TestBackupList, expnode)
436

    
437
      if qa_config.TestEnabled("instance-import"):
438
        newinst = qa_config.AcquireInstance()
439
        try:
440
          RunTest(qa_instance.TestInstanceImport, newinst, pnode,
441
                  expnode, name)
442
          # Check if starting the instance works
443
          RunTest(qa_instance.TestInstanceStartup, newinst)
444
          RunTest(qa_instance.TestInstanceRemove, newinst)
445
        finally:
446
          newinst.Release()
447
    finally:
448
      expnode.Release()
449

    
450
  # FIXME: inter-cluster-instance-move crashes on file based instances :/
451
  # See Issue 414.
452
  if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"])
453
      and (instance.disk_template not in constants.DTS_FILEBASED)):
454
    newinst = qa_config.AcquireInstance()
455
    try:
456
      tnode = qa_config.AcquireNode(exclude=inodes)
457
      try:
458
        RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst,
459
                inodes, tnode)
460
      finally:
461
        tnode.Release()
462
    finally:
463
      newinst.Release()
464

    
465

    
466
def RunDaemonTests(instance):
467
  """Test the ganeti-watcher script.
468

469
  """
470
  RunTest(qa_daemon.TestPauseWatcher)
471

    
472
  RunTestIf("instance-automatic-restart",
473
            qa_daemon.TestInstanceAutomaticRestart, instance)
474
  RunTestIf("instance-consecutive-failures",
475
            qa_daemon.TestInstanceConsecutiveFailures, instance)
476

    
477
  RunTest(qa_daemon.TestResumeWatcher)
478

    
479

    
480
def RunHardwareFailureTests(instance, inodes):
481
  """Test cluster internal hardware failure recovery.
482

483
  """
484
  RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
485
  RunTestIf(["instance-failover", qa_rapi.Enabled],
486
            qa_rapi.TestRapiInstanceFailover, instance)
487

    
488
  RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
489
  RunTestIf(["instance-migrate", qa_rapi.Enabled],
490
            qa_rapi.TestRapiInstanceMigrate, instance)
491

    
492
  if qa_config.TestEnabled("instance-replace-disks"):
493
    # We just need alternative secondary nodes, hence "- 1"
494
    othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes)
495
    try:
496
      RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance)
497
      RunTest(qa_instance.TestReplaceDisks,
498
              instance, inodes, othernodes)
499
    finally:
500
      qa_config.ReleaseManyNodes(othernodes)
501
    del othernodes
502

    
503
  if qa_config.TestEnabled("instance-recreate-disks"):
504
    try:
505
      acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
506
      othernodes = acquirednodes
507
    except qa_error.OutOfNodesError:
508
      if len(inodes) > 1:
509
        # If the cluster is not big enough, let's reuse some of the nodes, but
510
        # with different roles. In this way, we can test a DRBD instance even on
511
        # a 3-node cluster.
512
        acquirednodes = [qa_config.AcquireNode(exclude=inodes)]
513
        othernodes = acquirednodes + inodes[:-1]
514
      else:
515
        raise
516
    try:
517
      RunTest(qa_instance.TestRecreateDisks,
518
              instance, inodes, othernodes)
519
    finally:
520
      qa_config.ReleaseManyNodes(acquirednodes)
521

    
522
  if len(inodes) >= 2:
523
    RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1])
524
    RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1])
525
    RunTestIf("node-migrate", qa_node.TestNodeMigrate, inodes[0], inodes[1])
526

    
527

    
528
def RunExclusiveStorageTests():
529
  """Test exclusive storage."""
530
  if not qa_config.TestEnabled("cluster-exclusive-storage"):
531
    return
532

    
533
  node = qa_config.AcquireNode()
534
  try:
535
    old_es = qa_cluster.TestSetExclStorCluster(False)
536
    qa_node.TestExclStorSingleNode(node)
537

    
538
    qa_cluster.TestSetExclStorCluster(True)
539
    qa_cluster.TestExclStorSharedPv(node)
540

    
541
    if qa_config.TestEnabled("instance-add-plain-disk"):
542
      # Make sure that the cluster doesn't have any pre-existing problem
543
      qa_cluster.AssertClusterVerify()
544

    
545
      # Create and allocate instances
546
      instance1 = qa_instance.TestInstanceAddWithPlainDisk([node])
547
      try:
548
        instance2 = qa_instance.TestInstanceAddWithPlainDisk([node])
549
        try:
550
          # cluster-verify checks that disks are allocated correctly
551
          qa_cluster.AssertClusterVerify()
552

    
553
          # Remove instances
554
          qa_instance.TestInstanceRemove(instance2)
555
          qa_instance.TestInstanceRemove(instance1)
556
        finally:
557
          instance2.Release()
558
      finally:
559
        instance1.Release()
560

    
561
    if qa_config.TestEnabled("instance-add-drbd-disk"):
562
      snode = qa_config.AcquireNode()
563
      try:
564
        qa_cluster.TestSetExclStorCluster(False)
565
        instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode])
566
        try:
567
          qa_cluster.TestSetExclStorCluster(True)
568
          exp_err = [constants.CV_EINSTANCEUNSUITABLENODE]
569
          qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
570
          qa_instance.TestInstanceRemove(instance)
571
        finally:
572
          instance.Release()
573
      finally:
574
        snode.Release()
575
    qa_cluster.TestSetExclStorCluster(old_es)
576
  finally:
577
    node.Release()
578

    
579

    
580
def _BuildSpecDict(par, mn, st, mx):
581
  return {
582
    constants.ISPECS_MINMAX: [{
583
      constants.ISPECS_MIN: {par: mn},
584
      constants.ISPECS_MAX: {par: mx},
585
      }],
586
    constants.ISPECS_STD: {par: st},
587
    }
588

    
589

    
590
def _BuildDoubleSpecDict(index, par, mn, st, mx):
591
  new_spec = {
592
    constants.ISPECS_MINMAX: [{}, {}],
593
    }
594
  if st is not None:
595
    new_spec[constants.ISPECS_STD] = {par: st}
596
  new_spec[constants.ISPECS_MINMAX][index] = {
597
    constants.ISPECS_MIN: {par: mn},
598
    constants.ISPECS_MAX: {par: mx},
599
    }
600
  return new_spec
601

    
602

    
603
def TestIPolicyPlainInstance():
604
  """Test instance policy interaction with instances"""
605
  params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"]
606
  if not qa_config.IsTemplateSupported(constants.DT_PLAIN):
607
    print "Template %s not supported" % constants.DT_PLAIN
608
    return
609

    
610
  # This test assumes that the group policy is empty
611
  (_, old_specs) = qa_cluster.TestClusterSetISpecs()
612
  # We also assume to have only one min/max bound
613
  assert len(old_specs[constants.ISPECS_MINMAX]) == 1
614
  node = qa_config.AcquireNode()
615
  try:
616
    # Log of policy changes, list of tuples:
617
    # (full_change, incremental_change, policy_violated)
618
    history = []
619
    instance = qa_instance.TestInstanceAddWithPlainDisk([node])
620
    try:
621
      policyerror = [constants.CV_EINSTANCEPOLICY]
622
      for par in params:
623
        (iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par)
624
        # Some specs must be multiple of 4
625
        new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4)
626
        history.append((None, new_spec, True))
627
        if iminval > 0:
628
          # Some specs must be multiple of 4
629
          if iminval >= 4:
630
            upper = iminval - 4
631
          else:
632
            upper = iminval - 1
633
          new_spec = _BuildSpecDict(par, 0, upper, upper)
634
          history.append((None, new_spec, True))
635
        history.append((old_specs, None, False))
636

    
637
      # Test with two instance specs
638
      double_specs = copy.deepcopy(old_specs)
639
      double_specs[constants.ISPECS_MINMAX] = \
640
          double_specs[constants.ISPECS_MINMAX] * 2
641
      (par1, par2) = params[0:2]
642
      (_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1)
643
      (_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2)
644
      old_minmax = old_specs[constants.ISPECS_MINMAX][0]
645
      history.extend([
646
        (double_specs, None, False),
647
        # The first min/max limit is being violated
648
        (None,
649
         _BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4,
650
                              imaxval1 + 4),
651
         False),
652
        # Both min/max limits are being violated
653
        (None,
654
         _BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4),
655
         True),
656
        # The second min/max limit is being violated
657
        (None,
658
         _BuildDoubleSpecDict(0, par1,
659
                              old_minmax[constants.ISPECS_MIN][par1],
660
                              old_specs[constants.ISPECS_STD][par1],
661
                              old_minmax[constants.ISPECS_MAX][par1]),
662
         False),
663
        (old_specs, None, False),
664
        ])
665

    
666
      # Apply the changes, and check policy violations after each change
667
      qa_cluster.AssertClusterVerify()
668
      for (new_specs, diff_specs, failed) in history:
669
        qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
670
                                        diff_specs=diff_specs)
671
        if failed:
672
          qa_cluster.AssertClusterVerify(warnings=policyerror)
673
        else:
674
          qa_cluster.AssertClusterVerify()
675

    
676
      qa_instance.TestInstanceRemove(instance)
677
    finally:
678
      instance.Release()
679

    
680
    # Now we replay the same policy changes, and we expect that the instance
681
    # cannot be created for the cases where we had a policy violation above
682
    for (new_specs, diff_specs, failed) in history:
683
      qa_cluster.TestClusterSetISpecs(new_specs=new_specs,
684
                                      diff_specs=diff_specs)
685
      if failed:
686
        qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
687
      # Instance creation with no policy violation has been tested already
688
  finally:
689
    node.Release()
690

    
691

    
692
def IsExclusiveStorageInstanceTestEnabled():
693
  test_name = "exclusive-storage-instance-tests"
694
  if qa_config.TestEnabled(test_name):
695
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
696
    vgscmd = utils.ShellQuoteArgs([
697
      "vgs", "--noheadings", "-o", "pv_count", vgname,
698
      ])
699
    nodes = qa_config.GetConfig()["nodes"]
700
    for node in nodes:
701
      try:
702
        pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
703
      except Exception, e:
704
        msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
705
               (node.primary, test_name, e))
706
        raise qa_error.Error(msg)
707
      if pvnum < 2:
708
        raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" %
709
                             (node.primary, pvnum, test_name))
710
    res = True
711
  else:
712
    res = False
713
  return res
714

    
715

    
716
def RunInstanceTests():
717
  """Create and exercise instances."""
718

    
719
  for (test_name, templ, create_fun, num_nodes) in \
720
      qa_instance.available_instance_tests:
721
    if (qa_config.TestEnabled(test_name) and
722
        qa_config.IsTemplateSupported(templ)):
723
      inodes = qa_config.AcquireManyNodes(num_nodes)
724
      try:
725
        instance = RunTest(create_fun, inodes)
726
        try:
727
          RunTestIf("instance-user-down", qa_instance.TestInstanceUserDown,
728
                    instance, qa_config.GetMasterNode())
729
          RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
730
          RunDaemonTests(instance)
731
          for node in inodes:
732
            RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
733
          if len(inodes) > 1:
734
            RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
735
                      constants.INITIAL_NODE_GROUP_NAME,
736
                      inodes[0].primary, inodes[1].primary)
737
          if qa_config.TestEnabled("instance-convert-disk"):
738
            RunTest(qa_instance.TestInstanceShutdown, instance)
739
            RunTest(qa_instance.TestInstanceConvertDiskToPlain,
740
                    instance, inodes)
741
            RunTest(qa_instance.TestInstanceStartup, instance)
742
          RunTestIf("instance-modify-disks",
743
                    qa_instance.TestInstanceModifyDisks, instance)
744
          RunCommonInstanceTests(instance, inodes)
745
          if qa_config.TestEnabled("instance-modify-primary"):
746
            othernode = qa_config.AcquireNode()
747
            RunTest(qa_instance.TestInstanceModifyPrimaryAndBack,
748
                    instance, inodes[0], othernode)
749
            othernode.Release()
750
          RunGroupListTests()
751
          RunExportImportTests(instance, inodes)
752
          RunHardwareFailureTests(instance, inodes)
753
          RunRepairDiskSizes()
754
          RunTest(qa_instance.TestInstanceRemove, instance)
755
        finally:
756
          instance.Release()
757
        del instance
758
      finally:
759
        qa_config.ReleaseManyNodes(inodes)
760
      qa_cluster.AssertClusterVerify()
761

    
762

    
763
def RunMonitoringTests():
764
  if qa_config.TestEnabled("mon-collector"):
765
    RunTest(qa_monitoring.TestInstStatusCollector)
766

    
767

    
768
def RunQa():
769
  """Main QA body.
770

771
  """
772
  rapi_user = "ganeti-qa"
773

    
774
  RunEnvTests()
775
  rapi_secret = SetupCluster(rapi_user)
776

    
777
  if qa_rapi.Enabled():
778
    # Load RAPI certificate
779
    qa_rapi.Setup(rapi_user, rapi_secret)
780

    
781
  RunClusterTests()
782
  RunOsTests()
783

    
784
  RunTestIf("tags", qa_tags.TestClusterTags)
785

    
786
  RunCommonNodeTests()
787
  RunGroupListTests()
788
  RunGroupRwTests()
789
  RunNetworkTests()
790

    
791
  # The master shouldn't be readded or put offline; "delay" needs a non-master
792
  # node to test
793
  pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
794
  try:
795
    RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
796
    RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
797
    RunTestIf("delay", qa_cluster.TestDelay, pnode)
798
  finally:
799
    pnode.Release()
800

    
801
  # Make sure the cluster is clean before running instance tests
802
  qa_cluster.AssertClusterVerify()
803

    
804
  pnode = qa_config.AcquireNode()
805
  try:
806
    RunTestIf("tags", qa_tags.TestNodeTags, pnode)
807

    
808
    if qa_rapi.Enabled():
809
      RunTest(qa_rapi.TestNode, pnode)
810

    
811
      if (qa_config.TestEnabled("instance-add-plain-disk")
812
          and qa_config.IsTemplateSupported(constants.DT_PLAIN)):
813
        for use_client in [True, False]:
814
          rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode,
815
                                  use_client)
816
          try:
817
            if qa_config.TestEnabled("instance-plain-rapi-common-tests"):
818
              RunCommonInstanceTests(rapi_instance, [pnode])
819
            RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client)
820
          finally:
821
            rapi_instance.Release()
822
          del rapi_instance
823

    
824
  finally:
825
    pnode.Release()
826

    
827
  config_list = [
828
    ("default-instance-tests", lambda: None, lambda _: None),
829
    (IsExclusiveStorageInstanceTestEnabled,
830
     lambda: qa_cluster.TestSetExclStorCluster(True),
831
     qa_cluster.TestSetExclStorCluster),
832
  ]
833
  for (conf_name, setup_conf_f, restore_conf_f) in config_list:
834
    if qa_config.TestEnabled(conf_name):
835
      oldconf = setup_conf_f()
836
      RunInstanceTests()
837
      restore_conf_f(oldconf)
838

    
839
  pnode = qa_config.AcquireNode()
840
  try:
841
    if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]):
842
      for shutdown in [False, True]:
843
        instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode])
844
        try:
845
          expnode = qa_config.AcquireNode(exclude=pnode)
846
          try:
847
            if shutdown:
848
              # Stop instance before exporting and removing it
849
              RunTest(qa_instance.TestInstanceShutdown, instance)
850
            RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode)
851
            RunTest(qa_instance.TestBackupList, expnode)
852
          finally:
853
            expnode.Release()
854
        finally:
855
          instance.Release()
856
        del expnode
857
        del instance
858
      qa_cluster.AssertClusterVerify()
859

    
860
  finally:
861
    pnode.Release()
862

    
863
  RunTestIf("cluster-upgrade", qa_cluster.TestUpgrade)
864

    
865
  RunExclusiveStorageTests()
866
  RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"],
867
            TestIPolicyPlainInstance)
868

    
869
  RunTestIf(
870
    "instance-add-restricted-by-disktemplates",
871
    qa_instance.TestInstanceCreationRestrictedByDiskTemplates)
872

    
873
  # Test removing instance with offline drbd secondary
874
  if qa_config.TestEnabled(["instance-remove-drbd-offline",
875
                            "instance-add-drbd-disk"]):
876
    # Make sure the master is not put offline
877
    snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode())
878
    try:
879
      pnode = qa_config.AcquireNode(exclude=snode)
880
      try:
881
        instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode])
882
        set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes")
883
        set_online = lambda node: qa_node.MakeNodeOffline(node, "no")
884
        RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode,
885
                set_offline, set_online)
886
      finally:
887
        pnode.Release()
888
    finally:
889
      snode.Release()
890
    qa_cluster.AssertClusterVerify()
891

    
892
  RunMonitoringTests()
893

    
894
  RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
895

    
896
  RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
897

    
898

    
899
@UsesRapiClient
900
def main():
901
  """Main program.
902

903
  """
904
  parser = optparse.OptionParser(usage="%prog [options] <config-file>")
905
  parser.add_option("--yes-do-it", dest="yes_do_it",
906
                    action="store_true",
907
                    help="Really execute the tests")
908
  (opts, args) = parser.parse_args()
909

    
910
  if len(args) == 1:
911
    (config_file, ) = args
912
  else:
913
    parser.error("Wrong number of arguments.")
914

    
915
  if not opts.yes_do_it:
916
    print ("Executing this script irreversibly destroys any Ganeti\n"
917
           "configuration on all nodes involved. If you really want\n"
918
           "to start testing, supply the --yes-do-it option.")
919
    sys.exit(1)
920

    
921
  qa_config.Load(config_file)
922

    
923
  primary = qa_config.GetMasterNode().primary
924
  qa_utils.StartMultiplexer(primary)
925
  print ("SSH command for primary node: %s" %
926
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
927
  print ("SSH command for other nodes: %s" %
928
         utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", "")))
929
  try:
930
    RunQa()
931
  finally:
932
    qa_utils.CloseMultiplexers()
933

    
934
if __name__ == "__main__":
935
  main()