root / qa / ganeti-qa.py @ 35ff270b
History | View | Annotate | Download (29.4 kB)
1 |
#!/usr/bin/python -u
|
---|---|
2 |
#
|
3 |
|
4 |
# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
|
5 |
#
|
6 |
# This program is free software; you can redistribute it and/or modify
|
7 |
# it under the terms of the GNU General Public License as published by
|
8 |
# the Free Software Foundation; either version 2 of the License, or
|
9 |
# (at your option) any later version.
|
10 |
#
|
11 |
# This program is distributed in the hope that it will be useful, but
|
12 |
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14 |
# General Public License for more details.
|
15 |
#
|
16 |
# You should have received a copy of the GNU General Public License
|
17 |
# along with this program; if not, write to the Free Software
|
18 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
19 |
# 02110-1301, USA.
|
20 |
|
21 |
|
22 |
"""Script for doing QA on Ganeti.
|
23 |
|
24 |
"""
|
25 |
|
26 |
# pylint: disable=C0103
|
27 |
# due to invalid name
|
28 |
|
29 |
import copy |
30 |
import datetime |
31 |
import optparse |
32 |
import sys |
33 |
|
34 |
import qa_cluster |
35 |
import qa_config |
36 |
import qa_daemon |
37 |
import qa_env |
38 |
import qa_error |
39 |
import qa_group |
40 |
import qa_instance |
41 |
import qa_network |
42 |
import qa_node |
43 |
import qa_os |
44 |
import qa_job |
45 |
import qa_rapi |
46 |
import qa_tags |
47 |
import qa_utils |
48 |
|
49 |
from ganeti import utils |
50 |
from ganeti import rapi # pylint: disable=W0611 |
51 |
from ganeti import constants |
52 |
from ganeti import pathutils |
53 |
|
54 |
from ganeti.http.auth import ParsePasswordFile |
55 |
import ganeti.rapi.client # pylint: disable=W0611 |
56 |
from ganeti.rapi.client import UsesRapiClient |
57 |
|
58 |
|
59 |
def _FormatHeader(line, end=72): |
60 |
"""Fill a line up to the end column.
|
61 |
|
62 |
"""
|
63 |
line = "---- " + line + " " |
64 |
line += "-" * (end - len(line)) |
65 |
line = line.rstrip() |
66 |
return line
|
67 |
|
68 |
|
69 |
def _DescriptionOf(fn): |
70 |
"""Computes the description of an item.
|
71 |
|
72 |
"""
|
73 |
if fn.__doc__:
|
74 |
desc = fn.__doc__.splitlines()[0].strip()
|
75 |
else:
|
76 |
desc = "%r" % fn
|
77 |
|
78 |
return desc.rstrip(".") |
79 |
|
80 |
|
81 |
def RunTest(fn, *args, **kwargs): |
82 |
"""Runs a test after printing a header.
|
83 |
|
84 |
"""
|
85 |
|
86 |
tstart = datetime.datetime.now() |
87 |
|
88 |
desc = _DescriptionOf(fn) |
89 |
|
90 |
print
|
91 |
print _FormatHeader("%s start %s" % (tstart, desc)) |
92 |
|
93 |
try:
|
94 |
retval = fn(*args, **kwargs) |
95 |
return retval
|
96 |
finally:
|
97 |
tstop = datetime.datetime.now() |
98 |
tdelta = tstop - tstart |
99 |
print _FormatHeader("%s time=%s %s" % (tstop, tdelta, desc)) |
100 |
|
101 |
|
102 |
def RunTestIf(testnames, fn, *args, **kwargs): |
103 |
"""Runs a test conditionally.
|
104 |
|
105 |
@param testnames: either a single test name in the configuration
|
106 |
file, or a list of testnames (which will be AND-ed together)
|
107 |
|
108 |
"""
|
109 |
if qa_config.TestEnabled(testnames):
|
110 |
RunTest(fn, *args, **kwargs) |
111 |
else:
|
112 |
tstart = datetime.datetime.now() |
113 |
desc = _DescriptionOf(fn) |
114 |
# TODO: Formatting test names when non-string names are involved
|
115 |
print _FormatHeader("%s skipping %s, test(s) %s disabled" % |
116 |
(tstart, desc, testnames)) |
117 |
|
118 |
|
119 |
def RunEnvTests(): |
120 |
"""Run several environment tests.
|
121 |
|
122 |
"""
|
123 |
RunTestIf("env", qa_env.TestSshConnection)
|
124 |
RunTestIf("env", qa_env.TestIcmpPing)
|
125 |
RunTestIf("env", qa_env.TestGanetiCommands)
|
126 |
|
127 |
|
128 |
def _LookupRapiSecret(rapi_user): |
129 |
"""Find the RAPI secret for the given user.
|
130 |
|
131 |
@param rapi_user: Login user
|
132 |
@return: Login secret for the user
|
133 |
|
134 |
"""
|
135 |
CTEXT = "{CLEARTEXT}"
|
136 |
master = qa_config.GetMasterNode() |
137 |
cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)]
|
138 |
file_content = qa_utils.GetCommandOutput(master.primary, |
139 |
utils.ShellQuoteArgs(cmd)) |
140 |
users = ParsePasswordFile(file_content) |
141 |
entry = users.get(rapi_user) |
142 |
if not entry: |
143 |
raise qa_error.Error("User %s not found in RAPI users file" % rapi_user) |
144 |
secret = entry.password |
145 |
if secret.upper().startswith(CTEXT):
|
146 |
secret = secret[len(CTEXT):]
|
147 |
elif secret.startswith("{"): |
148 |
raise qa_error.Error("Unsupported password schema for RAPI user %s:" |
149 |
" not a clear text password" % rapi_user)
|
150 |
return secret
|
151 |
|
152 |
|
153 |
def SetupCluster(rapi_user): |
154 |
"""Initializes the cluster.
|
155 |
|
156 |
@param rapi_user: Login user for RAPI
|
157 |
@return: Login secret for RAPI
|
158 |
|
159 |
"""
|
160 |
rapi_secret = utils.GenerateSecret() |
161 |
RunTestIf("create-cluster", qa_cluster.TestClusterInit,
|
162 |
rapi_user, rapi_secret) |
163 |
if not qa_config.TestEnabled("create-cluster"): |
164 |
# If the cluster is already in place, we assume that exclusive-storage is
|
165 |
# already set according to the configuration
|
166 |
qa_config.SetExclusiveStorage(qa_config.get("exclusive-storage", False)) |
167 |
if qa_rapi.Enabled():
|
168 |
# To support RAPI on an existing cluster we have to find out the secret
|
169 |
rapi_secret = _LookupRapiSecret(rapi_user) |
170 |
|
171 |
# Test on empty cluster
|
172 |
RunTestIf("node-list", qa_node.TestNodeList)
|
173 |
RunTestIf("instance-list", qa_instance.TestInstanceList)
|
174 |
RunTestIf("job-list", qa_job.TestJobList)
|
175 |
|
176 |
RunTestIf("create-cluster", qa_node.TestNodeAddAll)
|
177 |
if not qa_config.TestEnabled("create-cluster"): |
178 |
# consider the nodes are already there
|
179 |
qa_node.MarkNodeAddedAll() |
180 |
|
181 |
RunTestIf("test-jobqueue", qa_cluster.TestJobqueue)
|
182 |
|
183 |
# enable the watcher (unconditionally)
|
184 |
RunTest(qa_daemon.TestResumeWatcher) |
185 |
|
186 |
RunTestIf("node-list", qa_node.TestNodeList)
|
187 |
|
188 |
# Test listing fields
|
189 |
RunTestIf("node-list", qa_node.TestNodeListFields)
|
190 |
RunTestIf("instance-list", qa_instance.TestInstanceListFields)
|
191 |
RunTestIf("job-list", qa_job.TestJobListFields)
|
192 |
RunTestIf("instance-export", qa_instance.TestBackupListFields)
|
193 |
|
194 |
RunTestIf("node-info", qa_node.TestNodeInfo)
|
195 |
|
196 |
return rapi_secret
|
197 |
|
198 |
|
199 |
def RunClusterTests(): |
200 |
"""Runs tests related to gnt-cluster.
|
201 |
|
202 |
"""
|
203 |
for test, fn in [ |
204 |
("create-cluster", qa_cluster.TestClusterInitDisk),
|
205 |
("cluster-renew-crypto", qa_cluster.TestClusterRenewCrypto),
|
206 |
("cluster-verify", qa_cluster.TestClusterVerify),
|
207 |
("cluster-reserved-lvs", qa_cluster.TestClusterReservedLvs),
|
208 |
# TODO: add more cluster modify tests
|
209 |
("cluster-modify", qa_cluster.TestClusterModifyEmpty),
|
210 |
("cluster-modify", qa_cluster.TestClusterModifyIPolicy),
|
211 |
("cluster-modify", qa_cluster.TestClusterModifyISpecs),
|
212 |
("cluster-modify", qa_cluster.TestClusterModifyBe),
|
213 |
("cluster-modify", qa_cluster.TestClusterModifyDisk),
|
214 |
("cluster-modify", qa_cluster.TestClusterModifyDiskTemplates),
|
215 |
("cluster-rename", qa_cluster.TestClusterRename),
|
216 |
("cluster-info", qa_cluster.TestClusterVersion),
|
217 |
("cluster-info", qa_cluster.TestClusterInfo),
|
218 |
("cluster-info", qa_cluster.TestClusterGetmaster),
|
219 |
("cluster-redist-conf", qa_cluster.TestClusterRedistConf),
|
220 |
(["cluster-copyfile", qa_config.NoVirtualCluster],
|
221 |
qa_cluster.TestClusterCopyfile), |
222 |
("cluster-command", qa_cluster.TestClusterCommand),
|
223 |
("cluster-burnin", qa_cluster.TestClusterBurnin),
|
224 |
("cluster-master-failover", qa_cluster.TestClusterMasterFailover),
|
225 |
("cluster-master-failover",
|
226 |
qa_cluster.TestClusterMasterFailoverWithDrainedQueue), |
227 |
(["cluster-oob", qa_config.NoVirtualCluster],
|
228 |
qa_cluster.TestClusterOob), |
229 |
(qa_rapi.Enabled, qa_rapi.TestVersion), |
230 |
(qa_rapi.Enabled, qa_rapi.TestEmptyCluster), |
231 |
(qa_rapi.Enabled, qa_rapi.TestRapiQuery), |
232 |
]: |
233 |
RunTestIf(test, fn) |
234 |
|
235 |
|
236 |
def RunRepairDiskSizes(): |
237 |
"""Run the repair disk-sizes test.
|
238 |
|
239 |
"""
|
240 |
RunTestIf("cluster-repair-disk-sizes", qa_cluster.TestClusterRepairDiskSizes)
|
241 |
|
242 |
|
243 |
def RunOsTests(): |
244 |
"""Runs all tests related to gnt-os.
|
245 |
|
246 |
"""
|
247 |
os_enabled = ["os", qa_config.NoVirtualCluster]
|
248 |
|
249 |
if qa_config.TestEnabled(qa_rapi.Enabled):
|
250 |
rapi_getos = qa_rapi.GetOperatingSystems |
251 |
else:
|
252 |
rapi_getos = None
|
253 |
|
254 |
for fn in [ |
255 |
qa_os.TestOsList, |
256 |
qa_os.TestOsDiagnose, |
257 |
]: |
258 |
RunTestIf(os_enabled, fn) |
259 |
|
260 |
for fn in [ |
261 |
qa_os.TestOsValid, |
262 |
qa_os.TestOsInvalid, |
263 |
qa_os.TestOsPartiallyValid, |
264 |
]: |
265 |
RunTestIf(os_enabled, fn, rapi_getos) |
266 |
|
267 |
for fn in [ |
268 |
qa_os.TestOsModifyValid, |
269 |
qa_os.TestOsModifyInvalid, |
270 |
qa_os.TestOsStatesNonExisting, |
271 |
]: |
272 |
RunTestIf(os_enabled, fn) |
273 |
|
274 |
|
275 |
def RunCommonInstanceTests(instance): |
276 |
"""Runs a few tests that are common to all disk types.
|
277 |
|
278 |
"""
|
279 |
RunTestIf("instance-shutdown", qa_instance.TestInstanceShutdown, instance)
|
280 |
RunTestIf(["instance-shutdown", "instance-console", qa_rapi.Enabled], |
281 |
qa_rapi.TestRapiStoppedInstanceConsole, instance) |
282 |
RunTestIf(["instance-shutdown", "instance-modify"], |
283 |
qa_instance.TestInstanceStoppedModify, instance) |
284 |
RunTestIf("instance-shutdown", qa_instance.TestInstanceStartup, instance)
|
285 |
|
286 |
# Test shutdown/start via RAPI
|
287 |
RunTestIf(["instance-shutdown", qa_rapi.Enabled],
|
288 |
qa_rapi.TestRapiInstanceShutdown, instance) |
289 |
RunTestIf(["instance-shutdown", qa_rapi.Enabled],
|
290 |
qa_rapi.TestRapiInstanceStartup, instance) |
291 |
|
292 |
RunTestIf("instance-list", qa_instance.TestInstanceList)
|
293 |
|
294 |
RunTestIf("instance-info", qa_instance.TestInstanceInfo, instance)
|
295 |
|
296 |
RunTestIf("instance-modify", qa_instance.TestInstanceModify, instance)
|
297 |
RunTestIf(["instance-modify", qa_rapi.Enabled],
|
298 |
qa_rapi.TestRapiInstanceModify, instance) |
299 |
|
300 |
RunTestIf("instance-console", qa_instance.TestInstanceConsole, instance)
|
301 |
RunTestIf(["instance-console", qa_rapi.Enabled],
|
302 |
qa_rapi.TestRapiInstanceConsole, instance) |
303 |
|
304 |
RunTestIf("instance-device-names", qa_instance.TestInstanceDeviceNames,
|
305 |
instance) |
306 |
DOWN_TESTS = qa_config.Either([ |
307 |
"instance-reinstall",
|
308 |
"instance-rename",
|
309 |
"instance-grow-disk",
|
310 |
]) |
311 |
|
312 |
# shutdown instance for any 'down' tests
|
313 |
RunTestIf(DOWN_TESTS, qa_instance.TestInstanceShutdown, instance) |
314 |
|
315 |
# now run the 'down' state tests
|
316 |
RunTestIf("instance-reinstall", qa_instance.TestInstanceReinstall, instance)
|
317 |
RunTestIf(["instance-reinstall", qa_rapi.Enabled],
|
318 |
qa_rapi.TestRapiInstanceReinstall, instance) |
319 |
|
320 |
if qa_config.TestEnabled("instance-rename"): |
321 |
tgt_instance = qa_config.AcquireInstance() |
322 |
try:
|
323 |
rename_source = instance.name |
324 |
rename_target = tgt_instance.name |
325 |
# perform instance rename to the same name
|
326 |
RunTest(qa_instance.TestInstanceRenameAndBack, |
327 |
rename_source, rename_source) |
328 |
RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack, |
329 |
rename_source, rename_source) |
330 |
if rename_target is not None: |
331 |
# perform instance rename to a different name, if we have one configured
|
332 |
RunTest(qa_instance.TestInstanceRenameAndBack, |
333 |
rename_source, rename_target) |
334 |
RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceRenameAndBack, |
335 |
rename_source, rename_target) |
336 |
finally:
|
337 |
tgt_instance.Release() |
338 |
|
339 |
RunTestIf(["instance-grow-disk"], qa_instance.TestInstanceGrowDisk, instance)
|
340 |
|
341 |
# and now start the instance again
|
342 |
RunTestIf(DOWN_TESTS, qa_instance.TestInstanceStartup, instance) |
343 |
|
344 |
RunTestIf("instance-reboot", qa_instance.TestInstanceReboot, instance)
|
345 |
|
346 |
RunTestIf("tags", qa_tags.TestInstanceTags, instance)
|
347 |
|
348 |
RunTestIf("cluster-verify", qa_cluster.TestClusterVerify)
|
349 |
|
350 |
RunTestIf(qa_rapi.Enabled, qa_rapi.TestInstance, instance) |
351 |
|
352 |
# Lists instances, too
|
353 |
RunTestIf("node-list", qa_node.TestNodeList)
|
354 |
|
355 |
# Some jobs have been run, let's test listing them
|
356 |
RunTestIf("job-list", qa_job.TestJobList)
|
357 |
|
358 |
|
359 |
def RunCommonNodeTests(): |
360 |
"""Run a few common node tests.
|
361 |
|
362 |
"""
|
363 |
RunTestIf("node-volumes", qa_node.TestNodeVolumes)
|
364 |
RunTestIf("node-storage", qa_node.TestNodeStorage)
|
365 |
RunTestIf(["node-oob", qa_config.NoVirtualCluster], qa_node.TestOutOfBand)
|
366 |
|
367 |
|
368 |
def RunGroupListTests(): |
369 |
"""Run tests for listing node groups.
|
370 |
|
371 |
"""
|
372 |
RunTestIf("group-list", qa_group.TestGroupList)
|
373 |
RunTestIf("group-list", qa_group.TestGroupListFields)
|
374 |
|
375 |
|
376 |
def RunNetworkTests(): |
377 |
"""Run tests for network management.
|
378 |
|
379 |
"""
|
380 |
RunTestIf("network", qa_network.TestNetworkAddRemove)
|
381 |
RunTestIf("network", qa_network.TestNetworkConnect)
|
382 |
|
383 |
|
384 |
def RunGroupRwTests(): |
385 |
"""Run tests for adding/removing/renaming groups.
|
386 |
|
387 |
"""
|
388 |
RunTestIf("group-rwops", qa_group.TestGroupAddRemoveRename)
|
389 |
RunTestIf("group-rwops", qa_group.TestGroupAddWithOptions)
|
390 |
RunTestIf("group-rwops", qa_group.TestGroupModify)
|
391 |
RunTestIf(["group-rwops", qa_rapi.Enabled], qa_rapi.TestRapiNodeGroups)
|
392 |
RunTestIf(["group-rwops", "tags"], qa_tags.TestGroupTags, |
393 |
qa_group.GetDefaultGroup()) |
394 |
|
395 |
|
396 |
def RunExportImportTests(instance, inodes): |
397 |
"""Tries to export and import the instance.
|
398 |
|
399 |
@type inodes: list of nodes
|
400 |
@param inodes: current nodes of the instance
|
401 |
|
402 |
"""
|
403 |
# FIXME: export explicitly bails out on file based storage. other non-lvm
|
404 |
# based storage types are untested, though. Also note that import could still
|
405 |
# work, but is deeply embedded into the "export" case.
|
406 |
if (qa_config.TestEnabled("instance-export") and |
407 |
instance.disk_template != constants.DT_FILE): |
408 |
RunTest(qa_instance.TestInstanceExportNoTarget, instance) |
409 |
|
410 |
pnode = inodes[0]
|
411 |
expnode = qa_config.AcquireNode(exclude=pnode) |
412 |
try:
|
413 |
name = RunTest(qa_instance.TestInstanceExport, instance, expnode) |
414 |
|
415 |
RunTest(qa_instance.TestBackupList, expnode) |
416 |
|
417 |
if qa_config.TestEnabled("instance-import"): |
418 |
newinst = qa_config.AcquireInstance() |
419 |
try:
|
420 |
RunTest(qa_instance.TestInstanceImport, newinst, pnode, |
421 |
expnode, name) |
422 |
# Check if starting the instance works
|
423 |
RunTest(qa_instance.TestInstanceStartup, newinst) |
424 |
RunTest(qa_instance.TestInstanceRemove, newinst) |
425 |
finally:
|
426 |
newinst.Release() |
427 |
finally:
|
428 |
expnode.Release() |
429 |
|
430 |
# FIXME: inter-cluster-instance-move crashes on file based instances :/
|
431 |
# See Issue 414.
|
432 |
if (qa_config.TestEnabled([qa_rapi.Enabled, "inter-cluster-instance-move"]) |
433 |
and instance.disk_template != constants.DT_FILE):
|
434 |
newinst = qa_config.AcquireInstance() |
435 |
try:
|
436 |
tnode = qa_config.AcquireNode(exclude=inodes) |
437 |
try:
|
438 |
RunTest(qa_rapi.TestInterClusterInstanceMove, instance, newinst, |
439 |
inodes, tnode) |
440 |
finally:
|
441 |
tnode.Release() |
442 |
finally:
|
443 |
newinst.Release() |
444 |
|
445 |
|
446 |
def RunDaemonTests(instance): |
447 |
"""Test the ganeti-watcher script.
|
448 |
|
449 |
"""
|
450 |
RunTest(qa_daemon.TestPauseWatcher) |
451 |
|
452 |
RunTestIf("instance-automatic-restart",
|
453 |
qa_daemon.TestInstanceAutomaticRestart, instance) |
454 |
RunTestIf("instance-consecutive-failures",
|
455 |
qa_daemon.TestInstanceConsecutiveFailures, instance) |
456 |
|
457 |
RunTest(qa_daemon.TestResumeWatcher) |
458 |
|
459 |
|
460 |
def RunHardwareFailureTests(instance, inodes): |
461 |
"""Test cluster internal hardware failure recovery.
|
462 |
|
463 |
"""
|
464 |
RunTestIf("instance-failover", qa_instance.TestInstanceFailover, instance)
|
465 |
RunTestIf(["instance-failover", qa_rapi.Enabled],
|
466 |
qa_rapi.TestRapiInstanceFailover, instance) |
467 |
|
468 |
RunTestIf("instance-migrate", qa_instance.TestInstanceMigrate, instance)
|
469 |
RunTestIf(["instance-migrate", qa_rapi.Enabled],
|
470 |
qa_rapi.TestRapiInstanceMigrate, instance) |
471 |
|
472 |
if qa_config.TestEnabled("instance-replace-disks"): |
473 |
# We just need alternative secondary nodes, hence "- 1"
|
474 |
othernodes = qa_config.AcquireManyNodes(len(inodes) - 1, exclude=inodes) |
475 |
try:
|
476 |
RunTestIf(qa_rapi.Enabled, qa_rapi.TestRapiInstanceReplaceDisks, instance) |
477 |
RunTest(qa_instance.TestReplaceDisks, |
478 |
instance, inodes, othernodes) |
479 |
finally:
|
480 |
qa_config.ReleaseManyNodes(othernodes) |
481 |
del othernodes
|
482 |
|
483 |
if qa_config.TestEnabled("instance-recreate-disks"): |
484 |
try:
|
485 |
acquirednodes = qa_config.AcquireManyNodes(len(inodes), exclude=inodes)
|
486 |
othernodes = acquirednodes |
487 |
except qa_error.OutOfNodesError:
|
488 |
if len(inodes) > 1: |
489 |
# If the cluster is not big enough, let's reuse some of the nodes, but
|
490 |
# with different roles. In this way, we can test a DRBD instance even on
|
491 |
# a 3-node cluster.
|
492 |
acquirednodes = [qa_config.AcquireNode(exclude=inodes)] |
493 |
othernodes = acquirednodes + inodes[:-1]
|
494 |
else:
|
495 |
raise
|
496 |
try:
|
497 |
RunTest(qa_instance.TestRecreateDisks, |
498 |
instance, inodes, othernodes) |
499 |
finally:
|
500 |
qa_config.ReleaseManyNodes(acquirednodes) |
501 |
|
502 |
if len(inodes) >= 2: |
503 |
RunTestIf("node-evacuate", qa_node.TestNodeEvacuate, inodes[0], inodes[1]) |
504 |
RunTestIf("node-failover", qa_node.TestNodeFailover, inodes[0], inodes[1]) |
505 |
|
506 |
|
507 |
def RunExclusiveStorageTests(): |
508 |
"""Test exclusive storage."""
|
509 |
if not qa_config.TestEnabled("cluster-exclusive-storage"): |
510 |
return
|
511 |
|
512 |
node = qa_config.AcquireNode() |
513 |
try:
|
514 |
old_es = qa_cluster.TestSetExclStorCluster(False)
|
515 |
qa_node.TestExclStorSingleNode(node) |
516 |
|
517 |
qa_cluster.TestSetExclStorCluster(True)
|
518 |
qa_cluster.TestExclStorSharedPv(node) |
519 |
|
520 |
if qa_config.TestEnabled("instance-add-plain-disk"): |
521 |
# Make sure that the cluster doesn't have any pre-existing problem
|
522 |
qa_cluster.AssertClusterVerify() |
523 |
|
524 |
# Create and allocate instances
|
525 |
instance1 = qa_instance.TestInstanceAddWithPlainDisk([node]) |
526 |
try:
|
527 |
instance2 = qa_instance.TestInstanceAddWithPlainDisk([node]) |
528 |
try:
|
529 |
# cluster-verify checks that disks are allocated correctly
|
530 |
qa_cluster.AssertClusterVerify() |
531 |
|
532 |
# Remove instances
|
533 |
qa_instance.TestInstanceRemove(instance2) |
534 |
qa_instance.TestInstanceRemove(instance1) |
535 |
finally:
|
536 |
instance2.Release() |
537 |
finally:
|
538 |
instance1.Release() |
539 |
|
540 |
if qa_config.TestEnabled("instance-add-drbd-disk"): |
541 |
snode = qa_config.AcquireNode() |
542 |
try:
|
543 |
qa_cluster.TestSetExclStorCluster(False)
|
544 |
instance = qa_instance.TestInstanceAddWithDrbdDisk([node, snode]) |
545 |
try:
|
546 |
qa_cluster.TestSetExclStorCluster(True)
|
547 |
exp_err = [constants.CV_EINSTANCEUNSUITABLENODE] |
548 |
qa_cluster.AssertClusterVerify(fail=True, errors=exp_err)
|
549 |
qa_instance.TestInstanceRemove(instance) |
550 |
finally:
|
551 |
instance.Release() |
552 |
finally:
|
553 |
snode.Release() |
554 |
qa_cluster.TestSetExclStorCluster(old_es) |
555 |
finally:
|
556 |
node.Release() |
557 |
|
558 |
|
559 |
def _BuildSpecDict(par, mn, st, mx): |
560 |
return {
|
561 |
constants.ISPECS_MINMAX: [{ |
562 |
constants.ISPECS_MIN: {par: mn}, |
563 |
constants.ISPECS_MAX: {par: mx}, |
564 |
}], |
565 |
constants.ISPECS_STD: {par: st}, |
566 |
} |
567 |
|
568 |
|
569 |
def _BuildDoubleSpecDict(index, par, mn, st, mx): |
570 |
new_spec = { |
571 |
constants.ISPECS_MINMAX: [{}, {}], |
572 |
} |
573 |
if st is not None: |
574 |
new_spec[constants.ISPECS_STD] = {par: st} |
575 |
new_spec[constants.ISPECS_MINMAX][index] = { |
576 |
constants.ISPECS_MIN: {par: mn}, |
577 |
constants.ISPECS_MAX: {par: mx}, |
578 |
} |
579 |
return new_spec
|
580 |
|
581 |
|
582 |
def TestIPolicyPlainInstance(): |
583 |
"""Test instance policy interaction with instances"""
|
584 |
params = ["memory-size", "cpu-count", "disk-count", "disk-size", "nic-count"] |
585 |
if not qa_config.IsTemplateSupported(constants.DT_PLAIN): |
586 |
print "Template %s not supported" % constants.DT_PLAIN |
587 |
return
|
588 |
|
589 |
# This test assumes that the group policy is empty
|
590 |
(_, old_specs) = qa_cluster.TestClusterSetISpecs() |
591 |
# We also assume to have only one min/max bound
|
592 |
assert len(old_specs[constants.ISPECS_MINMAX]) == 1 |
593 |
node = qa_config.AcquireNode() |
594 |
try:
|
595 |
# Log of policy changes, list of tuples:
|
596 |
# (full_change, incremental_change, policy_violated)
|
597 |
history = [] |
598 |
instance = qa_instance.TestInstanceAddWithPlainDisk([node]) |
599 |
try:
|
600 |
policyerror = [constants.CV_EINSTANCEPOLICY] |
601 |
for par in params: |
602 |
(iminval, imaxval) = qa_instance.GetInstanceSpec(instance.name, par) |
603 |
# Some specs must be multiple of 4
|
604 |
new_spec = _BuildSpecDict(par, imaxval + 4, imaxval + 4, imaxval + 4) |
605 |
history.append((None, new_spec, True)) |
606 |
if iminval > 0: |
607 |
# Some specs must be multiple of 4
|
608 |
if iminval >= 4: |
609 |
upper = iminval - 4
|
610 |
else:
|
611 |
upper = iminval - 1
|
612 |
new_spec = _BuildSpecDict(par, 0, upper, upper)
|
613 |
history.append((None, new_spec, True)) |
614 |
history.append((old_specs, None, False)) |
615 |
|
616 |
# Test with two instance specs
|
617 |
double_specs = copy.deepcopy(old_specs) |
618 |
double_specs[constants.ISPECS_MINMAX] = \ |
619 |
double_specs[constants.ISPECS_MINMAX] * 2
|
620 |
(par1, par2) = params[0:2] |
621 |
(_, imaxval1) = qa_instance.GetInstanceSpec(instance.name, par1) |
622 |
(_, imaxval2) = qa_instance.GetInstanceSpec(instance.name, par2) |
623 |
old_minmax = old_specs[constants.ISPECS_MINMAX][0]
|
624 |
history.extend([ |
625 |
(double_specs, None, False), |
626 |
# The first min/max limit is being violated
|
627 |
(None,
|
628 |
_BuildDoubleSpecDict(0, par1, imaxval1 + 4, imaxval1 + 4, |
629 |
imaxval1 + 4),
|
630 |
False),
|
631 |
# Both min/max limits are being violated
|
632 |
(None,
|
633 |
_BuildDoubleSpecDict(1, par2, imaxval2 + 4, None, imaxval2 + 4), |
634 |
True),
|
635 |
# The second min/max limit is being violated
|
636 |
(None,
|
637 |
_BuildDoubleSpecDict(0, par1,
|
638 |
old_minmax[constants.ISPECS_MIN][par1], |
639 |
old_specs[constants.ISPECS_STD][par1], |
640 |
old_minmax[constants.ISPECS_MAX][par1]), |
641 |
False),
|
642 |
(old_specs, None, False), |
643 |
]) |
644 |
|
645 |
# Apply the changes, and check policy violations after each change
|
646 |
qa_cluster.AssertClusterVerify() |
647 |
for (new_specs, diff_specs, failed) in history: |
648 |
qa_cluster.TestClusterSetISpecs(new_specs=new_specs, |
649 |
diff_specs=diff_specs) |
650 |
if failed:
|
651 |
qa_cluster.AssertClusterVerify(warnings=policyerror) |
652 |
else:
|
653 |
qa_cluster.AssertClusterVerify() |
654 |
|
655 |
qa_instance.TestInstanceRemove(instance) |
656 |
finally:
|
657 |
instance.Release() |
658 |
|
659 |
# Now we replay the same policy changes, and we expect that the instance
|
660 |
# cannot be created for the cases where we had a policy violation above
|
661 |
for (new_specs, diff_specs, failed) in history: |
662 |
qa_cluster.TestClusterSetISpecs(new_specs=new_specs, |
663 |
diff_specs=diff_specs) |
664 |
if failed:
|
665 |
qa_instance.TestInstanceAddWithPlainDisk([node], fail=True)
|
666 |
# Instance creation with no policy violation has been tested already
|
667 |
finally:
|
668 |
node.Release() |
669 |
|
670 |
|
671 |
def IsExclusiveStorageInstanceTestEnabled(): |
672 |
test_name = "exclusive-storage-instance-tests"
|
673 |
if qa_config.TestEnabled(test_name):
|
674 |
vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
|
675 |
vgscmd = utils.ShellQuoteArgs([ |
676 |
"vgs", "--noheadings", "-o", "pv_count", vgname, |
677 |
]) |
678 |
nodes = qa_config.GetConfig()["nodes"]
|
679 |
for node in nodes: |
680 |
try:
|
681 |
pvnum = int(qa_utils.GetCommandOutput(node.primary, vgscmd))
|
682 |
except Exception, e: |
683 |
msg = ("Cannot get the number of PVs on %s, needed by '%s': %s" %
|
684 |
(node.primary, test_name, e)) |
685 |
raise qa_error.Error(msg)
|
686 |
if pvnum < 2: |
687 |
raise qa_error.Error("Node %s has not enough PVs (%s) to run '%s'" % |
688 |
(node.primary, pvnum, test_name)) |
689 |
res = True
|
690 |
else:
|
691 |
res = False
|
692 |
return res
|
693 |
|
694 |
|
695 |
def RunInstanceTests(): |
696 |
"""Create and exercise instances."""
|
697 |
instance_tests = [ |
698 |
("instance-add-plain-disk", constants.DT_PLAIN,
|
699 |
qa_instance.TestInstanceAddWithPlainDisk, 1),
|
700 |
("instance-add-drbd-disk", constants.DT_DRBD8,
|
701 |
qa_instance.TestInstanceAddWithDrbdDisk, 2),
|
702 |
("instance-add-diskless", constants.DT_DISKLESS,
|
703 |
qa_instance.TestInstanceAddDiskless, 1),
|
704 |
("instance-add-file", constants.DT_FILE,
|
705 |
qa_instance.TestInstanceAddFile, 1)
|
706 |
] |
707 |
|
708 |
for (test_name, templ, create_fun, num_nodes) in instance_tests: |
709 |
if (qa_config.TestEnabled(test_name) and |
710 |
qa_config.IsTemplateSupported(templ)): |
711 |
inodes = qa_config.AcquireManyNodes(num_nodes) |
712 |
try:
|
713 |
instance = RunTest(create_fun, inodes) |
714 |
try:
|
715 |
RunTestIf("cluster-epo", qa_cluster.TestClusterEpo)
|
716 |
RunDaemonTests(instance) |
717 |
for node in inodes: |
718 |
RunTestIf("haskell-confd", qa_node.TestNodeListDrbd, node)
|
719 |
if len(inodes) > 1: |
720 |
RunTestIf("group-rwops", qa_group.TestAssignNodesIncludingSplit,
|
721 |
constants.INITIAL_NODE_GROUP_NAME, |
722 |
inodes[0].primary, inodes[1].primary) |
723 |
if qa_config.TestEnabled("instance-convert-disk"): |
724 |
RunTest(qa_instance.TestInstanceShutdown, instance) |
725 |
RunTest(qa_instance.TestInstanceConvertDiskToPlain, |
726 |
instance, inodes) |
727 |
RunTest(qa_instance.TestInstanceStartup, instance) |
728 |
RunTestIf("instance-modify-disks",
|
729 |
qa_instance.TestInstanceModifyDisks, instance) |
730 |
RunCommonInstanceTests(instance) |
731 |
if qa_config.TestEnabled("instance-modify-primary"): |
732 |
othernode = qa_config.AcquireNode() |
733 |
RunTest(qa_instance.TestInstanceModifyPrimaryAndBack, |
734 |
instance, inodes[0], othernode)
|
735 |
othernode.Release() |
736 |
RunGroupListTests() |
737 |
RunExportImportTests(instance, inodes) |
738 |
RunHardwareFailureTests(instance, inodes) |
739 |
RunRepairDiskSizes() |
740 |
RunTest(qa_instance.TestInstanceRemove, instance) |
741 |
finally:
|
742 |
instance.Release() |
743 |
del instance
|
744 |
finally:
|
745 |
qa_config.ReleaseManyNodes(inodes) |
746 |
qa_cluster.AssertClusterVerify() |
747 |
|
748 |
|
749 |
def RunQa(): |
750 |
"""Main QA body.
|
751 |
|
752 |
"""
|
753 |
rapi_user = "ganeti-qa"
|
754 |
|
755 |
RunEnvTests() |
756 |
rapi_secret = SetupCluster(rapi_user) |
757 |
|
758 |
if qa_rapi.Enabled():
|
759 |
# Load RAPI certificate
|
760 |
qa_rapi.Setup(rapi_user, rapi_secret) |
761 |
|
762 |
RunClusterTests() |
763 |
RunOsTests() |
764 |
|
765 |
RunTestIf("tags", qa_tags.TestClusterTags)
|
766 |
|
767 |
RunCommonNodeTests() |
768 |
RunGroupListTests() |
769 |
RunGroupRwTests() |
770 |
RunNetworkTests() |
771 |
|
772 |
# The master shouldn't be readded or put offline; "delay" needs a non-master
|
773 |
# node to test
|
774 |
pnode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode()) |
775 |
try:
|
776 |
RunTestIf("node-readd", qa_node.TestNodeReadd, pnode)
|
777 |
RunTestIf("node-modify", qa_node.TestNodeModify, pnode)
|
778 |
RunTestIf("delay", qa_cluster.TestDelay, pnode)
|
779 |
finally:
|
780 |
pnode.Release() |
781 |
|
782 |
# Make sure the cluster is clean before running instance tests
|
783 |
qa_cluster.AssertClusterVerify() |
784 |
|
785 |
pnode = qa_config.AcquireNode() |
786 |
try:
|
787 |
RunTestIf("tags", qa_tags.TestNodeTags, pnode)
|
788 |
|
789 |
if qa_rapi.Enabled():
|
790 |
RunTest(qa_rapi.TestNode, pnode) |
791 |
|
792 |
if qa_config.TestEnabled("instance-add-plain-disk"): |
793 |
for use_client in [True, False]: |
794 |
rapi_instance = RunTest(qa_rapi.TestRapiInstanceAdd, pnode, |
795 |
use_client) |
796 |
try:
|
797 |
if qa_config.TestEnabled("instance-plain-rapi-common-tests"): |
798 |
RunCommonInstanceTests(rapi_instance) |
799 |
RunTest(qa_rapi.TestRapiInstanceRemove, rapi_instance, use_client) |
800 |
finally:
|
801 |
rapi_instance.Release() |
802 |
del rapi_instance
|
803 |
|
804 |
finally:
|
805 |
pnode.Release() |
806 |
|
807 |
config_list = [ |
808 |
("default-instance-tests", lambda: None, lambda _: None), |
809 |
(IsExclusiveStorageInstanceTestEnabled, |
810 |
lambda: qa_cluster.TestSetExclStorCluster(True), |
811 |
qa_cluster.TestSetExclStorCluster), |
812 |
] |
813 |
for (conf_name, setup_conf_f, restore_conf_f) in config_list: |
814 |
if qa_config.TestEnabled(conf_name):
|
815 |
oldconf = setup_conf_f() |
816 |
RunInstanceTests() |
817 |
restore_conf_f(oldconf) |
818 |
|
819 |
pnode = qa_config.AcquireNode() |
820 |
try:
|
821 |
if qa_config.TestEnabled(["instance-add-plain-disk", "instance-export"]): |
822 |
for shutdown in [False, True]: |
823 |
instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, [pnode]) |
824 |
try:
|
825 |
expnode = qa_config.AcquireNode(exclude=pnode) |
826 |
try:
|
827 |
if shutdown:
|
828 |
# Stop instance before exporting and removing it
|
829 |
RunTest(qa_instance.TestInstanceShutdown, instance) |
830 |
RunTest(qa_instance.TestInstanceExportWithRemove, instance, expnode) |
831 |
RunTest(qa_instance.TestBackupList, expnode) |
832 |
finally:
|
833 |
expnode.Release() |
834 |
finally:
|
835 |
instance.Release() |
836 |
del expnode
|
837 |
del instance
|
838 |
qa_cluster.AssertClusterVerify() |
839 |
|
840 |
finally:
|
841 |
pnode.Release() |
842 |
|
843 |
RunExclusiveStorageTests() |
844 |
RunTestIf(["cluster-instance-policy", "instance-add-plain-disk"], |
845 |
TestIPolicyPlainInstance) |
846 |
|
847 |
RunTestIf( |
848 |
"instance-add-restricted-by-disktemplates",
|
849 |
qa_instance.TestInstanceCreationRestrictedByDiskTemplates) |
850 |
|
851 |
# Test removing instance with offline drbd secondary
|
852 |
if qa_config.TestEnabled(["instance-remove-drbd-offline", |
853 |
"instance-add-drbd-disk"]):
|
854 |
# Make sure the master is not put offline
|
855 |
snode = qa_config.AcquireNode(exclude=qa_config.GetMasterNode()) |
856 |
try:
|
857 |
pnode = qa_config.AcquireNode(exclude=snode) |
858 |
try:
|
859 |
instance = qa_instance.TestInstanceAddWithDrbdDisk([pnode, snode]) |
860 |
set_offline = lambda node: qa_node.MakeNodeOffline(node, "yes") |
861 |
set_online = lambda node: qa_node.MakeNodeOffline(node, "no") |
862 |
RunTest(qa_instance.TestRemoveInstanceOfflineNode, instance, snode, |
863 |
set_offline, set_online) |
864 |
finally:
|
865 |
pnode.Release() |
866 |
finally:
|
867 |
snode.Release() |
868 |
qa_cluster.AssertClusterVerify() |
869 |
|
870 |
RunTestIf("create-cluster", qa_node.TestNodeRemoveAll)
|
871 |
|
872 |
RunTestIf("cluster-destroy", qa_cluster.TestClusterDestroy)
|
873 |
|
874 |
|
875 |
@UsesRapiClient
|
876 |
def main(): |
877 |
"""Main program.
|
878 |
|
879 |
"""
|
880 |
parser = optparse.OptionParser(usage="%prog [options] <config-file>")
|
881 |
parser.add_option("--yes-do-it", dest="yes_do_it", |
882 |
action="store_true",
|
883 |
help="Really execute the tests")
|
884 |
(opts, args) = parser.parse_args() |
885 |
|
886 |
if len(args) == 1: |
887 |
(config_file, ) = args |
888 |
else:
|
889 |
parser.error("Wrong number of arguments.")
|
890 |
|
891 |
if not opts.yes_do_it: |
892 |
print ("Executing this script irreversibly destroys any Ganeti\n" |
893 |
"configuration on all nodes involved. If you really want\n"
|
894 |
"to start testing, supply the --yes-do-it option.")
|
895 |
sys.exit(1)
|
896 |
|
897 |
qa_config.Load(config_file) |
898 |
|
899 |
primary = qa_config.GetMasterNode().primary |
900 |
qa_utils.StartMultiplexer(primary) |
901 |
print ("SSH command for primary node: %s" % |
902 |
utils.ShellQuoteArgs(qa_utils.GetSSHCommand(primary, "")))
|
903 |
print ("SSH command for other nodes: %s" % |
904 |
utils.ShellQuoteArgs(qa_utils.GetSSHCommand("NODE", ""))) |
905 |
try:
|
906 |
RunQa() |
907 |
finally:
|
908 |
qa_utils.CloseMultiplexers() |
909 |
|
910 |
if __name__ == "__main__": |
911 |
main() |