Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ f356202a

History | View | Annotate | Download (16.8 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import tempfile
27
import os.path
28

    
29
from ganeti import constants
30
from ganeti import compat
31
from ganeti import utils
32

    
33
import qa_config
34
import qa_utils
35
import qa_error
36

    
37
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
38

    
39

    
40
#: cluster verify command
41
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
42

    
43

    
44
def _RemoveFileFromAllNodes(filename):
45
  """Removes a file from all nodes.
46

47
  """
48
  for node in qa_config.get("nodes"):
49
    AssertCommand(["rm", "-f", filename], node=node)
50

    
51

    
52
def _CheckFileOnAllNodes(filename, content):
53
  """Verifies the content of the given file on all nodes.
54

55
  """
56
  cmd = utils.ShellQuoteArgs(["cat", filename])
57
  for node in qa_config.get("nodes"):
58
    AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content)
59

    
60

    
61
# data for testing failures due to bad keys/values for disk parameters
62
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
63
                "drbd:nonexistent=1",
64
                "drbd:resync-rate=invalid",
65
                ]
66

    
67

    
68
def TestClusterInitDisk():
69
  """gnt-cluster init -D"""
70
  name = qa_config.get("name")
71
  for param in _FAIL_PARAMS:
72
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
73

    
74

    
75
def TestClusterInit(rapi_user, rapi_secret):
76
  """gnt-cluster init"""
77
  master = qa_config.GetMasterNode()
78

    
79
  rapi_dir = os.path.dirname(constants.RAPI_USERS_FILE)
80

    
81
  # First create the RAPI credentials
82
  fh = tempfile.NamedTemporaryFile()
83
  try:
84
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
85
    fh.flush()
86

    
87
    tmpru = qa_utils.UploadFile(master["primary"], fh.name)
88
    try:
89
      AssertCommand(["mkdir", "-p", rapi_dir])
90
      AssertCommand(["mv", tmpru, constants.RAPI_USERS_FILE])
91
    finally:
92
      AssertCommand(["rm", "-f", tmpru])
93
  finally:
94
    fh.close()
95

    
96
  # Initialize cluster
97
  cmd = ["gnt-cluster", "init"]
98

    
99
  cmd.append("--primary-ip-version=%d" %
100
             qa_config.get("primary_ip_version", 4))
101

    
102
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
103
                    "nic-count"):
104
    for spec_val in ("min", "max", "std"):
105
      spec = qa_config.get("ispec_%s_%s" %
106
                           (spec_type.replace('-', '_'), spec_val), None)
107
      if spec:
108
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
109

    
110
  if master.get("secondary", None):
111
    cmd.append("--secondary-ip=%s" % master["secondary"])
112

    
113
  bridge = qa_config.get("bridge", None)
114
  if bridge:
115
    cmd.append("--bridge=%s" % bridge)
116
    cmd.append("--master-netdev=%s" % bridge)
117

    
118
  htype = qa_config.get("enabled-hypervisors", None)
119
  if htype:
120
    cmd.append("--enabled-hypervisors=%s" % htype)
121

    
122
  cmd.append(qa_config.get("name"))
123
  AssertCommand(cmd)
124

    
125
  cmd = ["gnt-cluster", "modify"]
126

    
127
  # hypervisor parameter modifications
128
  hvp = qa_config.get("hypervisor-parameters", {})
129
  for k, v in hvp.items():
130
    cmd.extend(["-H", "%s:%s" % (k, v)])
131
  # backend parameter modifications
132
  bep = qa_config.get("backend-parameters", "")
133
  if bep:
134
    cmd.extend(["-B", bep])
135

    
136
  if len(cmd) > 2:
137
    AssertCommand(cmd)
138

    
139
  # OS parameters
140
  osp = qa_config.get("os-parameters", {})
141
  for k, v in osp.items():
142
    AssertCommand(["gnt-os", "modify", "-O", v, k])
143

    
144
  # OS hypervisor parameters
145
  os_hvp = qa_config.get("os-hvp", {})
146
  for os_name in os_hvp:
147
    for hv, hvp in os_hvp[os_name].items():
148
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
149

    
150

    
151
def TestClusterRename():
152
  """gnt-cluster rename"""
153
  cmd = ["gnt-cluster", "rename", "-f"]
154

    
155
  original_name = qa_config.get("name")
156
  rename_target = qa_config.get("rename", None)
157
  if rename_target is None:
158
    print qa_utils.FormatError('"rename" entry is missing')
159
    return
160

    
161
  for data in [
162
    cmd + [rename_target],
163
    _CLUSTER_VERIFY,
164
    cmd + [original_name],
165
    _CLUSTER_VERIFY,
166
    ]:
167
    AssertCommand(data)
168

    
169

    
170
def TestClusterOob():
171
  """out-of-band framework"""
172
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
173

    
174
  AssertCommand(_CLUSTER_VERIFY)
175
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
176
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
177
                 utils.NewUUID()])
178

    
179
  AssertCommand(_CLUSTER_VERIFY, fail=True)
180

    
181
  AssertCommand(["touch", oob_path_exists])
182
  AssertCommand(["chmod", "0400", oob_path_exists])
183
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
184

    
185
  try:
186
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
187
                   "oob_program=%s" % oob_path_exists])
188

    
189
    AssertCommand(_CLUSTER_VERIFY, fail=True)
190

    
191
    AssertCommand(["chmod", "0500", oob_path_exists])
192
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
193

    
194
    AssertCommand(_CLUSTER_VERIFY)
195
  finally:
196
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
197

    
198
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
199
                 "oob_program="])
200

    
201

    
202
def TestClusterEpo():
203
  """gnt-cluster epo"""
204
  master = qa_config.GetMasterNode()
205

    
206
  # Assert that OOB is unavailable for all nodes
207
  result_output = GetCommandOutput(master["primary"],
208
                                   "gnt-node list --verbose --no-header -o"
209
                                   " powered")
210
  AssertEqual(compat.all(powered == "(unavail)"
211
                         for powered in result_output.splitlines()), True)
212

    
213
  # Conflicting
214
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
215
  # --all doesn't expect arguments
216
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
217

    
218
  # Unless --all is given master is not allowed to be in the list
219
  AssertCommand(["gnt-cluster", "epo", "-f", master["primary"]], fail=True)
220

    
221
  # This shouldn't fail
222
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
223

    
224
  # All instances should have been stopped now
225
  result_output = GetCommandOutput(master["primary"],
226
                                   "gnt-instance list --no-header -o status")
227
  AssertEqual(compat.all(status == "ADMIN_down"
228
                         for status in result_output.splitlines()), True)
229

    
230
  # Now start everything again
231
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
232

    
233
  # All instances should have been started now
234
  result_output = GetCommandOutput(master["primary"],
235
                                   "gnt-instance list --no-header -o status")
236
  AssertEqual(compat.all(status == "running"
237
                         for status in result_output.splitlines()), True)
238

    
239

    
240
def TestClusterVerify():
241
  """gnt-cluster verify"""
242
  AssertCommand(_CLUSTER_VERIFY)
243
  AssertCommand(["gnt-cluster", "verify-disks"])
244

    
245

    
246
def TestJobqueue():
247
  """gnt-debug test-jobqueue"""
248
  AssertCommand(["gnt-debug", "test-jobqueue"])
249

    
250

    
251
def TestDelay(node):
252
  """gnt-debug delay"""
253
  AssertCommand(["gnt-debug", "delay", "1"])
254
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
255
  AssertCommand(["gnt-debug", "delay", "--no-master",
256
                 "-n", node["primary"], "1"])
257

    
258

    
259
def TestClusterReservedLvs():
260
  """gnt-cluster reserved lvs"""
261
  for fail, cmd in [
262
    (False, _CLUSTER_VERIFY),
263
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
264
    (False, ["lvcreate", "-L1G", "-nqa-test", "xenvg"]),
265
    (True, _CLUSTER_VERIFY),
266
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
267
             "xenvg/qa-test,.*/other-test"]),
268
    (False, _CLUSTER_VERIFY),
269
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ".*/qa-.*"]),
270
    (False, _CLUSTER_VERIFY),
271
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
272
    (True, _CLUSTER_VERIFY),
273
    (False, ["lvremove", "-f", "xenvg/qa-test"]),
274
    (False, _CLUSTER_VERIFY),
275
    ]:
276
    AssertCommand(cmd, fail=fail)
277

    
278

    
279
def TestClusterModifyDisk():
280
  """gnt-cluster modify -D"""
281
  for param in _FAIL_PARAMS:
282
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
283

    
284

    
285
def TestClusterModifyBe():
286
  """gnt-cluster modify -B"""
287
  for fail, cmd in [
288
    # max/min mem
289
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
290
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
291
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
292
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
293
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
294
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
295
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
296
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
297
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
298
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
299
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
300
    # vcpus
301
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
302
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
303
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
304
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
305
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
306
    # auto_balance
307
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
308
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
309
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
310
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
311
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
312
    ]:
313
    AssertCommand(cmd, fail=fail)
314

    
315
  # redo the original-requested BE parameters, if any
316
  bep = qa_config.get("backend-parameters", "")
317
  if bep:
318
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
319

    
320

    
321
def TestClusterInfo():
322
  """gnt-cluster info"""
323
  AssertCommand(["gnt-cluster", "info"])
324

    
325

    
326
def TestClusterRedistConf():
327
  """gnt-cluster redist-conf"""
328
  AssertCommand(["gnt-cluster", "redist-conf"])
329

    
330

    
331
def TestClusterGetmaster():
332
  """gnt-cluster getmaster"""
333
  AssertCommand(["gnt-cluster", "getmaster"])
334

    
335

    
336
def TestClusterVersion():
337
  """gnt-cluster version"""
338
  AssertCommand(["gnt-cluster", "version"])
339

    
340

    
341
def TestClusterRenewCrypto():
342
  """gnt-cluster renew-crypto"""
343
  master = qa_config.GetMasterNode()
344

    
345
  # Conflicting options
346
  cmd = ["gnt-cluster", "renew-crypto", "--force",
347
         "--new-cluster-certificate", "--new-confd-hmac-key"]
348
  conflicting = [
349
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
350
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
351
    ]
352
  for i in conflicting:
353
    AssertCommand(cmd + i, fail=True)
354

    
355
  # Invalid RAPI certificate
356
  cmd = ["gnt-cluster", "renew-crypto", "--force",
357
         "--rapi-certificate=/dev/null"]
358
  AssertCommand(cmd, fail=True)
359

    
360
  rapi_cert_backup = qa_utils.BackupFile(master["primary"],
361
                                         constants.RAPI_CERT_FILE)
362
  try:
363
    # Custom RAPI certificate
364
    fh = tempfile.NamedTemporaryFile()
365

    
366
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
367
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
368

    
369
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
370

    
371
    tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
372
    try:
373
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
374
                     "--rapi-certificate=%s" % tmpcert])
375
    finally:
376
      AssertCommand(["rm", "-f", tmpcert])
377

    
378
    # Custom cluster domain secret
379
    cds_fh = tempfile.NamedTemporaryFile()
380
    cds_fh.write(utils.GenerateSecret())
381
    cds_fh.write("\n")
382
    cds_fh.flush()
383

    
384
    tmpcds = qa_utils.UploadFile(master["primary"], cds_fh.name)
385
    try:
386
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
387
                     "--cluster-domain-secret=%s" % tmpcds])
388
    finally:
389
      AssertCommand(["rm", "-f", tmpcds])
390

    
391
    # Normal case
392
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
393
                   "--new-cluster-certificate", "--new-confd-hmac-key",
394
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
395

    
396
    # Restore RAPI certificate
397
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
398
                   "--rapi-certificate=%s" % rapi_cert_backup])
399
  finally:
400
    AssertCommand(["rm", "-f", rapi_cert_backup])
401

    
402

    
403
def TestClusterBurnin():
404
  """Burnin"""
405
  master = qa_config.GetMasterNode()
406

    
407
  options = qa_config.get("options", {})
408
  disk_template = options.get("burnin-disk-template", "drbd")
409
  parallel = options.get("burnin-in-parallel", False)
410
  check_inst = options.get("burnin-check-instances", False)
411
  do_rename = options.get("burnin-rename", "")
412
  do_reboot = options.get("burnin-reboot", True)
413
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
414

    
415
  # Get as many instances as we need
416
  instances = []
417
  try:
418
    try:
419
      num = qa_config.get("options", {}).get("burnin-instances", 1)
420
      for _ in range(0, num):
421
        instances.append(qa_config.AcquireInstance())
422
    except qa_error.OutOfInstancesError:
423
      print "Not enough instances, continuing anyway."
424

    
425
    if len(instances) < 1:
426
      raise qa_error.Error("Burnin needs at least one instance")
427

    
428
    script = qa_utils.UploadFile(master["primary"], "../tools/burnin")
429
    try:
430
      # Run burnin
431
      cmd = [script,
432
             "--os=%s" % qa_config.get("os"),
433
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
434
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
435
             "--disk-size=%s" % ",".join(qa_config.get("disk")),
436
             "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
437
             "--disk-template=%s" % disk_template]
438
      if parallel:
439
        cmd.append("--parallel")
440
        cmd.append("--early-release")
441
      if check_inst:
442
        cmd.append("--http-check")
443
      if do_rename:
444
        cmd.append("--rename=%s" % do_rename)
445
      if not do_reboot:
446
        cmd.append("--no-reboot")
447
      else:
448
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
449
      cmd += [inst["name"] for inst in instances]
450
      AssertCommand(cmd)
451
    finally:
452
      AssertCommand(["rm", "-f", script])
453

    
454
  finally:
455
    for inst in instances:
456
      qa_config.ReleaseInstance(inst)
457

    
458

    
459
def TestClusterMasterFailover():
460
  """gnt-cluster master-failover"""
461
  master = qa_config.GetMasterNode()
462
  failovermaster = qa_config.AcquireNode(exclude=master)
463

    
464
  cmd = ["gnt-cluster", "master-failover"]
465
  try:
466
    AssertCommand(cmd, node=failovermaster)
467
    # Back to original master node
468
    AssertCommand(cmd, node=master)
469
  finally:
470
    qa_config.ReleaseNode(failovermaster)
471

    
472

    
473
def TestClusterMasterFailoverWithDrainedQueue():
474
  """gnt-cluster master-failover with drained queue"""
475
  drain_check = ["test", "-f", constants.JOB_QUEUE_DRAIN_FILE]
476

    
477
  master = qa_config.GetMasterNode()
478
  failovermaster = qa_config.AcquireNode(exclude=master)
479

    
480
  # Ensure queue is not drained
481
  for node in [master, failovermaster]:
482
    AssertCommand(drain_check, node=node, fail=True)
483

    
484
  # Drain queue on failover master
485
  AssertCommand(["touch", constants.JOB_QUEUE_DRAIN_FILE], node=failovermaster)
486

    
487
  cmd = ["gnt-cluster", "master-failover"]
488
  try:
489
    AssertCommand(drain_check, node=failovermaster)
490
    AssertCommand(cmd, node=failovermaster)
491
    AssertCommand(drain_check, fail=True)
492
    AssertCommand(drain_check, node=failovermaster, fail=True)
493

    
494
    # Back to original master node
495
    AssertCommand(cmd, node=master)
496
  finally:
497
    qa_config.ReleaseNode(failovermaster)
498

    
499
  AssertCommand(drain_check, fail=True)
500
  AssertCommand(drain_check, node=failovermaster, fail=True)
501

    
502

    
503
def TestClusterCopyfile():
504
  """gnt-cluster copyfile"""
505
  master = qa_config.GetMasterNode()
506

    
507
  uniqueid = utils.NewUUID()
508

    
509
  # Create temporary file
510
  f = tempfile.NamedTemporaryFile()
511
  f.write(uniqueid)
512
  f.flush()
513
  f.seek(0)
514

    
515
  # Upload file to master node
516
  testname = qa_utils.UploadFile(master["primary"], f.name)
517
  try:
518
    # Copy file to all nodes
519
    AssertCommand(["gnt-cluster", "copyfile", testname])
520
    _CheckFileOnAllNodes(testname, uniqueid)
521
  finally:
522
    _RemoveFileFromAllNodes(testname)
523

    
524

    
525
def TestClusterCommand():
526
  """gnt-cluster command"""
527
  uniqueid = utils.NewUUID()
528
  rfile = "/tmp/gnt%s" % utils.NewUUID()
529
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
530
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
531
                              "%s >%s" % (rcmd, rfile)])
532

    
533
  try:
534
    AssertCommand(cmd)
535
    _CheckFileOnAllNodes(rfile, uniqueid)
536
  finally:
537
    _RemoveFileFromAllNodes(rfile)
538

    
539

    
540
def TestClusterDestroy():
541
  """gnt-cluster destroy"""
542
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
543

    
544

    
545
def TestClusterRepairDiskSizes():
546
  """gnt-cluster repair-disk-sizes"""
547
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])