Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ 6f88e076

History | View | Annotate | Download (22 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import constants
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34

    
35
import qa_config
36
import qa_utils
37
import qa_error
38

    
39
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
40

    
41

    
42
# Prefix for LVM volumes created by QA code during tests
43
_QA_LV_PREFIX = "qa-"
44

    
45
#: cluster verify command
46
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
47

    
48

    
49
def _RemoveFileFromAllNodes(filename):
50
  """Removes a file from all nodes.
51

52
  """
53
  for node in qa_config.get("nodes"):
54
    AssertCommand(["rm", "-f", filename], node=node)
55

    
56

    
57
def _CheckFileOnAllNodes(filename, content):
58
  """Verifies the content of the given file on all nodes.
59

60
  """
61
  cmd = utils.ShellQuoteArgs(["cat", filename])
62
  for node in qa_config.get("nodes"):
63
    AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content)
64

    
65

    
66
# "gnt-cluster info" fields
67
_CIFIELD_RE = re.compile(r"^[-\s]*(?P<field>[^\s:]+):\s*(?P<value>\S.*)$")
68

    
69

    
70
def _GetBoolClusterField(field):
71
  """Get the Boolean value of a cluster field.
72

73
  This function currently assumes that the field name is unique in the cluster
74
  configuration. An assertion checks this assumption.
75

76
  @type field: string
77
  @param field: Name of the field
78
  @rtype: bool
79
  @return: The effective value of the field
80

81
  """
82
  master = qa_config.GetMasterNode()
83
  infocmd = "gnt-cluster info"
84
  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
85
  ret = None
86
  for l in info_out.splitlines():
87
    m = _CIFIELD_RE.match(l)
88
    # FIXME: There should be a way to specify a field through a hierarchy
89
    if m and m.group("field") == field:
90
      # Make sure that ignoring the hierarchy doesn't cause a double match
91
      assert ret is None
92
      ret = (m.group("value").lower() == "true")
93
  if ret is not None:
94
    return ret
95
  raise qa_error.Error("Field not found in cluster configuration: %s" % field)
96

    
97

    
98
# Cluster-verify errors (date, "ERROR", then error code)
99
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- ERROR:([A-Z0-9_-]+):")
100

    
101

    
102
def _GetCVErrorCodes(cvout):
103
  ret = set()
104
  for l in cvout.splitlines():
105
    m = _CVERROR_RE.match(l)
106
    if m:
107
      ecode = m.group(1)
108
      ret.add(ecode)
109
  return ret
110

    
111

    
112
def AssertClusterVerify(fail=False, errors=None):
113
  """Run cluster-verify and check the result
114

115
  @type fail: bool
116
  @param fail: if cluster-verify is expected to fail instead of succeeding
117
  @type errors: list of tuples
118
  @param errors: List of CV_XXX errors that are expected; if specified, all the
119
      errors listed must appear in cluster-verify output. A non-empty value
120
      implies C{fail=True}.
121

122
  """
123
  cvcmd = "gnt-cluster verify"
124
  mnode = qa_config.GetMasterNode()
125
  if errors:
126
    cvout = GetCommandOutput(mnode["primary"], cvcmd + " --error-codes",
127
                             fail=True)
128
    actual = _GetCVErrorCodes(cvout)
129
    expected = compat.UniqueFrozenset(e for (_, e, _) in errors)
130
    if not actual.issuperset(expected):
131
      missing = expected.difference(actual)
132
      raise qa_error.Error("Cluster-verify didn't return these expected"
133
                           " errors: %s" % utils.CommaJoin(missing))
134
  else:
135
    AssertCommand(cvcmd, fail=fail, node=mnode)
136

    
137

    
138
# data for testing failures due to bad keys/values for disk parameters
139
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
140
                "drbd:nonexistent=1",
141
                "drbd:resync-rate=invalid",
142
                ]
143

    
144

    
145
def TestClusterInitDisk():
146
  """gnt-cluster init -D"""
147
  name = qa_config.get("name")
148
  for param in _FAIL_PARAMS:
149
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
150

    
151

    
152
def TestClusterInit(rapi_user, rapi_secret):
153
  """gnt-cluster init"""
154
  master = qa_config.GetMasterNode()
155

    
156
  rapi_dir = os.path.dirname(pathutils.RAPI_USERS_FILE)
157

    
158
  # First create the RAPI credentials
159
  fh = tempfile.NamedTemporaryFile()
160
  try:
161
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
162
    fh.flush()
163

    
164
    tmpru = qa_utils.UploadFile(master["primary"], fh.name)
165
    try:
166
      AssertCommand(["mkdir", "-p", rapi_dir])
167
      AssertCommand(["mv", tmpru, pathutils.RAPI_USERS_FILE])
168
    finally:
169
      AssertCommand(["rm", "-f", tmpru])
170
  finally:
171
    fh.close()
172

    
173
  # Initialize cluster
174
  cmd = [
175
    "gnt-cluster", "init",
176
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
177
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
178
    ]
179

    
180
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
181
                    "nic-count"):
182
    for spec_val in ("min", "max", "std"):
183
      spec = qa_config.get("ispec_%s_%s" %
184
                           (spec_type.replace("-", "_"), spec_val), None)
185
      if spec:
186
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
187

    
188
  if master.get("secondary", None):
189
    cmd.append("--secondary-ip=%s" % master["secondary"])
190

    
191
  vgname = qa_config.get("vg-name", None)
192
  if vgname:
193
    cmd.append("--vg-name=%s" % vgname)
194

    
195
  master_netdev = qa_config.get("master-netdev", None)
196
  if master_netdev:
197
    cmd.append("--master-netdev=%s" % master_netdev)
198

    
199
  nicparams = qa_config.get("default-nicparams", None)
200
  if nicparams:
201
    cmd.append("--nic-parameters=%s" %
202
               ",".join(utils.FormatKeyValue(nicparams)))
203

    
204
  # Cluster value of the exclusive-storage node parameter
205
  e_s = qa_config.get("exclusive-storage")
206
  if e_s is not None:
207
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
208
  else:
209
    e_s = False
210
  qa_config.SetExclusiveStorage(e_s)
211

    
212
  cmd.append(qa_config.get("name"))
213
  AssertCommand(cmd)
214

    
215
  cmd = ["gnt-cluster", "modify"]
216

    
217
  # hypervisor parameter modifications
218
  hvp = qa_config.get("hypervisor-parameters", {})
219
  for k, v in hvp.items():
220
    cmd.extend(["-H", "%s:%s" % (k, v)])
221
  # backend parameter modifications
222
  bep = qa_config.get("backend-parameters", "")
223
  if bep:
224
    cmd.extend(["-B", bep])
225

    
226
  if len(cmd) > 2:
227
    AssertCommand(cmd)
228

    
229
  # OS parameters
230
  osp = qa_config.get("os-parameters", {})
231
  for k, v in osp.items():
232
    AssertCommand(["gnt-os", "modify", "-O", v, k])
233

    
234
  # OS hypervisor parameters
235
  os_hvp = qa_config.get("os-hvp", {})
236
  for os_name in os_hvp:
237
    for hv, hvp in os_hvp[os_name].items():
238
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
239

    
240

    
241
def TestClusterRename():
242
  """gnt-cluster rename"""
243
  cmd = ["gnt-cluster", "rename", "-f"]
244

    
245
  original_name = qa_config.get("name")
246
  rename_target = qa_config.get("rename", None)
247
  if rename_target is None:
248
    print qa_utils.FormatError('"rename" entry is missing')
249
    return
250

    
251
  for data in [
252
    cmd + [rename_target],
253
    _CLUSTER_VERIFY,
254
    cmd + [original_name],
255
    _CLUSTER_VERIFY,
256
    ]:
257
    AssertCommand(data)
258

    
259

    
260
def TestClusterOob():
261
  """out-of-band framework"""
262
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
263

    
264
  AssertCommand(_CLUSTER_VERIFY)
265
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
266
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
267
                 utils.NewUUID()])
268

    
269
  AssertCommand(_CLUSTER_VERIFY, fail=True)
270

    
271
  AssertCommand(["touch", oob_path_exists])
272
  AssertCommand(["chmod", "0400", oob_path_exists])
273
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
274

    
275
  try:
276
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
277
                   "oob_program=%s" % oob_path_exists])
278

    
279
    AssertCommand(_CLUSTER_VERIFY, fail=True)
280

    
281
    AssertCommand(["chmod", "0500", oob_path_exists])
282
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
283

    
284
    AssertCommand(_CLUSTER_VERIFY)
285
  finally:
286
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
287

    
288
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
289
                 "oob_program="])
290

    
291

    
292
def TestClusterEpo():
293
  """gnt-cluster epo"""
294
  master = qa_config.GetMasterNode()
295

    
296
  # Assert that OOB is unavailable for all nodes
297
  result_output = GetCommandOutput(master["primary"],
298
                                   "gnt-node list --verbose --no-headers -o"
299
                                   " powered")
300
  AssertEqual(compat.all(powered == "(unavail)"
301
                         for powered in result_output.splitlines()), True)
302

    
303
  # Conflicting
304
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
305
  # --all doesn't expect arguments
306
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
307

    
308
  # Unless --all is given master is not allowed to be in the list
309
  AssertCommand(["gnt-cluster", "epo", "-f", master["primary"]], fail=True)
310

    
311
  # This shouldn't fail
312
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
313

    
314
  # All instances should have been stopped now
315
  result_output = GetCommandOutput(master["primary"],
316
                                   "gnt-instance list --no-headers -o status")
317
  # ERROR_down because the instance is stopped but not recorded as such
318
  AssertEqual(compat.all(status == "ERROR_down"
319
                         for status in result_output.splitlines()), True)
320

    
321
  # Now start everything again
322
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
323

    
324
  # All instances should have been started now
325
  result_output = GetCommandOutput(master["primary"],
326
                                   "gnt-instance list --no-headers -o status")
327
  AssertEqual(compat.all(status == "running"
328
                         for status in result_output.splitlines()), True)
329

    
330

    
331
def TestClusterVerify():
332
  """gnt-cluster verify"""
333
  AssertCommand(_CLUSTER_VERIFY)
334
  AssertCommand(["gnt-cluster", "verify-disks"])
335

    
336

    
337
def TestJobqueue():
338
  """gnt-debug test-jobqueue"""
339
  AssertCommand(["gnt-debug", "test-jobqueue"])
340

    
341

    
342
def TestDelay(node):
343
  """gnt-debug delay"""
344
  AssertCommand(["gnt-debug", "delay", "1"])
345
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
346
  AssertCommand(["gnt-debug", "delay", "--no-master",
347
                 "-n", node["primary"], "1"])
348

    
349

    
350
def TestClusterReservedLvs():
351
  """gnt-cluster reserved lvs"""
352
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
353
  lvname = _QA_LV_PREFIX + "test"
354
  lvfullname = "/".join([vgname, lvname])
355
  for fail, cmd in [
356
    (False, _CLUSTER_VERIFY),
357
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
358
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
359
    (True, _CLUSTER_VERIFY),
360
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
361
             "%s,.*/other-test" % lvfullname]),
362
    (False, _CLUSTER_VERIFY),
363
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
364
             ".*/%s.*" % _QA_LV_PREFIX]),
365
    (False, _CLUSTER_VERIFY),
366
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
367
    (True, _CLUSTER_VERIFY),
368
    (False, ["lvremove", "-f", lvfullname]),
369
    (False, _CLUSTER_VERIFY),
370
    ]:
371
    AssertCommand(cmd, fail=fail)
372

    
373

    
374
def TestClusterModifyEmpty():
375
  """gnt-cluster modify"""
376
  AssertCommand(["gnt-cluster", "modify"], fail=True)
377

    
378

    
379
def TestClusterModifyDisk():
380
  """gnt-cluster modify -D"""
381
  for param in _FAIL_PARAMS:
382
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
383

    
384

    
385
def TestClusterModifyBe():
386
  """gnt-cluster modify -B"""
387
  for fail, cmd in [
388
    # max/min mem
389
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
390
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
391
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
392
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
393
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
394
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
395
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
396
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
397
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
398
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
399
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
400
    # vcpus
401
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
402
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
403
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
404
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
405
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
406
    # auto_balance
407
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
408
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
409
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
410
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
411
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
412
    ]:
413
    AssertCommand(cmd, fail=fail)
414

    
415
  # redo the original-requested BE parameters, if any
416
  bep = qa_config.get("backend-parameters", "")
417
  if bep:
418
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
419

    
420

    
421
def TestClusterInfo():
422
  """gnt-cluster info"""
423
  AssertCommand(["gnt-cluster", "info"])
424

    
425

    
426
def TestClusterRedistConf():
427
  """gnt-cluster redist-conf"""
428
  AssertCommand(["gnt-cluster", "redist-conf"])
429

    
430

    
431
def TestClusterGetmaster():
432
  """gnt-cluster getmaster"""
433
  AssertCommand(["gnt-cluster", "getmaster"])
434

    
435

    
436
def TestClusterVersion():
437
  """gnt-cluster version"""
438
  AssertCommand(["gnt-cluster", "version"])
439

    
440

    
441
def TestClusterRenewCrypto():
442
  """gnt-cluster renew-crypto"""
443
  master = qa_config.GetMasterNode()
444

    
445
  # Conflicting options
446
  cmd = ["gnt-cluster", "renew-crypto", "--force",
447
         "--new-cluster-certificate", "--new-confd-hmac-key"]
448
  conflicting = [
449
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
450
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
451
    ]
452
  for i in conflicting:
453
    AssertCommand(cmd + i, fail=True)
454

    
455
  # Invalid RAPI certificate
456
  cmd = ["gnt-cluster", "renew-crypto", "--force",
457
         "--rapi-certificate=/dev/null"]
458
  AssertCommand(cmd, fail=True)
459

    
460
  rapi_cert_backup = qa_utils.BackupFile(master["primary"],
461
                                         pathutils.RAPI_CERT_FILE)
462
  try:
463
    # Custom RAPI certificate
464
    fh = tempfile.NamedTemporaryFile()
465

    
466
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
467
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
468

    
469
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
470

    
471
    tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
472
    try:
473
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
474
                     "--rapi-certificate=%s" % tmpcert])
475
    finally:
476
      AssertCommand(["rm", "-f", tmpcert])
477

    
478
    # Custom cluster domain secret
479
    cds_fh = tempfile.NamedTemporaryFile()
480
    cds_fh.write(utils.GenerateSecret())
481
    cds_fh.write("\n")
482
    cds_fh.flush()
483

    
484
    tmpcds = qa_utils.UploadFile(master["primary"], cds_fh.name)
485
    try:
486
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
487
                     "--cluster-domain-secret=%s" % tmpcds])
488
    finally:
489
      AssertCommand(["rm", "-f", tmpcds])
490

    
491
    # Normal case
492
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
493
                   "--new-cluster-certificate", "--new-confd-hmac-key",
494
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
495

    
496
    # Restore RAPI certificate
497
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
498
                   "--rapi-certificate=%s" % rapi_cert_backup])
499
  finally:
500
    AssertCommand(["rm", "-f", rapi_cert_backup])
501

    
502

    
503
def TestClusterBurnin():
504
  """Burnin"""
505
  master = qa_config.GetMasterNode()
506

    
507
  options = qa_config.get("options", {})
508
  disk_template = options.get("burnin-disk-template", "drbd")
509
  parallel = options.get("burnin-in-parallel", False)
510
  check_inst = options.get("burnin-check-instances", False)
511
  do_rename = options.get("burnin-rename", "")
512
  do_reboot = options.get("burnin-reboot", True)
513
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
514

    
515
  # Get as many instances as we need
516
  instances = []
517
  try:
518
    try:
519
      num = qa_config.get("options", {}).get("burnin-instances", 1)
520
      for _ in range(0, num):
521
        instances.append(qa_config.AcquireInstance())
522
    except qa_error.OutOfInstancesError:
523
      print "Not enough instances, continuing anyway."
524

    
525
    if len(instances) < 1:
526
      raise qa_error.Error("Burnin needs at least one instance")
527

    
528
    script = qa_utils.UploadFile(master["primary"], "../tools/burnin")
529
    try:
530
      # Run burnin
531
      cmd = [script,
532
             "--os=%s" % qa_config.get("os"),
533
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
534
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
535
             "--disk-size=%s" % ",".join(qa_config.get("disk")),
536
             "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
537
             "--disk-template=%s" % disk_template]
538
      if parallel:
539
        cmd.append("--parallel")
540
        cmd.append("--early-release")
541
      if check_inst:
542
        cmd.append("--http-check")
543
      if do_rename:
544
        cmd.append("--rename=%s" % do_rename)
545
      if not do_reboot:
546
        cmd.append("--no-reboot")
547
      else:
548
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
549
      cmd += [inst["name"] for inst in instances]
550
      AssertCommand(cmd)
551
    finally:
552
      AssertCommand(["rm", "-f", script])
553

    
554
  finally:
555
    for inst in instances:
556
      inst.Release()
557

    
558

    
559
def TestClusterMasterFailover():
560
  """gnt-cluster master-failover"""
561
  master = qa_config.GetMasterNode()
562
  failovermaster = qa_config.AcquireNode(exclude=master)
563

    
564
  cmd = ["gnt-cluster", "master-failover"]
565
  try:
566
    AssertCommand(cmd, node=failovermaster)
567
    # Back to original master node
568
    AssertCommand(cmd, node=master)
569
  finally:
570
    qa_config.ReleaseNode(failovermaster)
571

    
572

    
573
def TestClusterMasterFailoverWithDrainedQueue():
574
  """gnt-cluster master-failover with drained queue"""
575
  drain_check = ["test", "-f", pathutils.JOB_QUEUE_DRAIN_FILE]
576

    
577
  master = qa_config.GetMasterNode()
578
  failovermaster = qa_config.AcquireNode(exclude=master)
579

    
580
  # Ensure queue is not drained
581
  for node in [master, failovermaster]:
582
    AssertCommand(drain_check, node=node, fail=True)
583

    
584
  # Drain queue on failover master
585
  AssertCommand(["touch", pathutils.JOB_QUEUE_DRAIN_FILE], node=failovermaster)
586

    
587
  cmd = ["gnt-cluster", "master-failover"]
588
  try:
589
    AssertCommand(drain_check, node=failovermaster)
590
    AssertCommand(cmd, node=failovermaster)
591
    AssertCommand(drain_check, fail=True)
592
    AssertCommand(drain_check, node=failovermaster, fail=True)
593

    
594
    # Back to original master node
595
    AssertCommand(cmd, node=master)
596
  finally:
597
    qa_config.ReleaseNode(failovermaster)
598

    
599
  AssertCommand(drain_check, fail=True)
600
  AssertCommand(drain_check, node=failovermaster, fail=True)
601

    
602

    
603
def TestClusterCopyfile():
604
  """gnt-cluster copyfile"""
605
  master = qa_config.GetMasterNode()
606

    
607
  uniqueid = utils.NewUUID()
608

    
609
  # Create temporary file
610
  f = tempfile.NamedTemporaryFile()
611
  f.write(uniqueid)
612
  f.flush()
613
  f.seek(0)
614

    
615
  # Upload file to master node
616
  testname = qa_utils.UploadFile(master["primary"], f.name)
617
  try:
618
    # Copy file to all nodes
619
    AssertCommand(["gnt-cluster", "copyfile", testname])
620
    _CheckFileOnAllNodes(testname, uniqueid)
621
  finally:
622
    _RemoveFileFromAllNodes(testname)
623

    
624

    
625
def TestClusterCommand():
626
  """gnt-cluster command"""
627
  uniqueid = utils.NewUUID()
628
  rfile = "/tmp/gnt%s" % utils.NewUUID()
629
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
630
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
631
                              "%s >%s" % (rcmd, rfile)])
632

    
633
  try:
634
    AssertCommand(cmd)
635
    _CheckFileOnAllNodes(rfile, uniqueid)
636
  finally:
637
    _RemoveFileFromAllNodes(rfile)
638

    
639

    
640
def TestClusterDestroy():
641
  """gnt-cluster destroy"""
642
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
643

    
644

    
645
def TestClusterRepairDiskSizes():
646
  """gnt-cluster repair-disk-sizes"""
647
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
648

    
649

    
650
def TestSetExclStorCluster(newvalue):
651
  """Set the exclusive_storage node parameter at the cluster level.
652

653
  @type newvalue: bool
654
  @param newvalue: New value of exclusive_storage
655
  @rtype: bool
656
  @return: The old value of exclusive_storage
657

658
  """
659
  oldvalue = _GetBoolClusterField("exclusive_storage")
660
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
661
                 "exclusive_storage=%s" % newvalue])
662
  effvalue = _GetBoolClusterField("exclusive_storage")
663
  if effvalue != newvalue:
664
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
665
                         " of %s" % (effvalue, newvalue))
666
  qa_config.SetExclusiveStorage(newvalue)
667
  return oldvalue
668

    
669

    
670
def _BuildSetESCmd(value, node_name):
671
  return ["gnt-node", "modify", "--node-parameters",
672
          "exclusive_storage=%s" % value, node_name]
673

    
674

    
675
def TestExclStorSingleNode(node):
676
  """cluster-verify reports exclusive_storage set only on one node.
677

678
  """
679
  node_name = node["primary"]
680
  es_val = _GetBoolClusterField("exclusive_storage")
681
  assert not es_val
682
  AssertCommand(_BuildSetESCmd(True, node_name))
683
  AssertClusterVerify(fail=True, errors=[constants.CV_EGROUPMIXEDESFLAG])
684
  AssertCommand(_BuildSetESCmd("default", node_name))
685
  AssertClusterVerify()
686

    
687

    
688
def TestExclStorSharedPv(node):
689
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
690

691
  """
692
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
693
  lvname1 = _QA_LV_PREFIX + "vol1"
694
  lvname2 = _QA_LV_PREFIX + "vol2"
695
  node_name = node["primary"]
696
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
697
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
698
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
699
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
700
                                         constants.CV_ENODEORPHANLV])
701
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
702
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
703
  AssertClusterVerify()