Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ e0e44476

History | View | Annotate | Download (41.7 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import constants
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34

    
35
import qa_config
36
import qa_daemon
37
import qa_utils
38
import qa_error
39
import qa_instance
40

    
41
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
42

    
43

    
44
# Prefix for LVM volumes created by QA code during tests
45
_QA_LV_PREFIX = "qa-"
46

    
47
#: cluster verify command
48
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
49

    
50

    
51
def _RemoveFileFromAllNodes(filename):
52
  """Removes a file from all nodes.
53

54
  """
55
  for node in qa_config.get("nodes"):
56
    AssertCommand(["rm", "-f", filename], node=node)
57

    
58

    
59
def _CheckFileOnAllNodes(filename, content):
60
  """Verifies the content of the given file on all nodes.
61

62
  """
63
  cmd = utils.ShellQuoteArgs(["cat", filename])
64
  for node in qa_config.get("nodes"):
65
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
66

    
67

    
68
def _GetClusterField(field_path):
69
  """Get the value of a cluster field.
70

71
  @type field_path: list of strings
72
  @param field_path: Names of the groups/fields to navigate to get the desired
73
      value, e.g. C{["Default node parameters", "oob_program"]}
74
  @return: The effective value of the field (the actual type depends on the
75
      chosen field)
76

77
  """
78
  assert isinstance(field_path, list)
79
  assert field_path
80
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
81
  for key in field_path:
82
    ret = ret[key]
83
  return ret
84

    
85

    
86
# Cluster-verify errors (date, "ERROR", then error code)
87
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
88

    
89

    
90
def _GetCVErrorCodes(cvout):
91
  errs = set()
92
  warns = set()
93
  for l in cvout.splitlines():
94
    m = _CVERROR_RE.match(l)
95
    if m:
96
      etype = m.group(1)
97
      ecode = m.group(2)
98
      if etype == "ERROR":
99
        errs.add(ecode)
100
      elif etype == "WARNING":
101
        warns.add(ecode)
102
  return (errs, warns)
103

    
104

    
105
def _CheckVerifyErrors(actual, expected, etype):
106
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
107
  if not actual.issuperset(exp_codes):
108
    missing = exp_codes.difference(actual)
109
    raise qa_error.Error("Cluster-verify didn't return these expected"
110
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))
111

    
112

    
113
def AssertClusterVerify(fail=False, errors=None, warnings=None):
114
  """Run cluster-verify and check the result
115

116
  @type fail: bool
117
  @param fail: if cluster-verify is expected to fail instead of succeeding
118
  @type errors: list of tuples
119
  @param errors: List of CV_XXX errors that are expected; if specified, all the
120
      errors listed must appear in cluster-verify output. A non-empty value
121
      implies C{fail=True}.
122
  @type warnings: list of tuples
123
  @param warnings: Same as C{errors} but for warnings.
124

125
  """
126
  cvcmd = "gnt-cluster verify"
127
  mnode = qa_config.GetMasterNode()
128
  if errors or warnings:
129
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
130
                             fail=(fail or errors))
131
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
132
    if errors:
133
      _CheckVerifyErrors(act_errs, errors, "error")
134
    if warnings:
135
      _CheckVerifyErrors(act_warns, warnings, "warning")
136
  else:
137
    AssertCommand(cvcmd, fail=fail, node=mnode)
138

    
139

    
140
# data for testing failures due to bad keys/values for disk parameters
141
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
142
                "drbd:nonexistent=1",
143
                "drbd:resync-rate=invalid",
144
                ]
145

    
146

    
147
def TestClusterInitDisk():
148
  """gnt-cluster init -D"""
149
  name = qa_config.get("name")
150
  for param in _FAIL_PARAMS:
151
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
152

    
153

    
154
def TestClusterInit(rapi_user, rapi_secret):
155
  """gnt-cluster init"""
156
  master = qa_config.GetMasterNode()
157

    
158
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
159
  rapi_dir = os.path.dirname(rapi_users_path)
160

    
161
  # First create the RAPI credentials
162
  fh = tempfile.NamedTemporaryFile()
163
  try:
164
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
165
    fh.flush()
166

    
167
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
168
    try:
169
      AssertCommand(["mkdir", "-p", rapi_dir])
170
      AssertCommand(["mv", tmpru, rapi_users_path])
171
    finally:
172
      AssertCommand(["rm", "-f", tmpru])
173
  finally:
174
    fh.close()
175

    
176
  # Initialize cluster
177
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
178
  cmd = [
179
    "gnt-cluster", "init",
180
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
181
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
182
    "--enabled-disk-templates=%s" %
183
      ",".join(enabled_disk_templates),
184
    ]
185
  if constants.DT_FILE in enabled_disk_templates:
186
    cmd.append(
187
        "--file-storage-dir=%s" %
188
        qa_config.get("default-file-storage-dir",
189
                      pathutils.DEFAULT_FILE_STORAGE_DIR))
190

    
191
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
192
                    "nic-count"):
193
    for spec_val in ("min", "max", "std"):
194
      spec = qa_config.get("ispec_%s_%s" %
195
                           (spec_type.replace("-", "_"), spec_val), None)
196
      if spec is not None:
197
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
198

    
199
  if master.secondary:
200
    cmd.append("--secondary-ip=%s" % master.secondary)
201

    
202
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
203
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
204
    if vgname:
205
      cmd.append("--vg-name=%s" % vgname)
206
    else:
207
      raise qa_error.Error("Please specify a volume group if you enable"
208
                           " lvm-based disk templates in the QA.")
209

    
210
  master_netdev = qa_config.get("master-netdev", None)
211
  if master_netdev:
212
    cmd.append("--master-netdev=%s" % master_netdev)
213

    
214
  nicparams = qa_config.get("default-nicparams", None)
215
  if nicparams:
216
    cmd.append("--nic-parameters=%s" %
217
               ",".join(utils.FormatKeyValue(nicparams)))
218

    
219
  # Cluster value of the exclusive-storage node parameter
220
  e_s = qa_config.get("exclusive-storage")
221
  if e_s is not None:
222
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
223
  else:
224
    e_s = False
225
  qa_config.SetExclusiveStorage(e_s)
226

    
227
  extra_args = qa_config.get("cluster-init-args")
228
  if extra_args:
229
    cmd.extend(extra_args)
230

    
231
  cmd.append(qa_config.get("name"))
232

    
233
  AssertCommand(cmd)
234

    
235
  cmd = ["gnt-cluster", "modify"]
236

    
237
  # hypervisor parameter modifications
238
  hvp = qa_config.get("hypervisor-parameters", {})
239
  for k, v in hvp.items():
240
    cmd.extend(["-H", "%s:%s" % (k, v)])
241
  # backend parameter modifications
242
  bep = qa_config.get("backend-parameters", "")
243
  if bep:
244
    cmd.extend(["-B", bep])
245

    
246
  if len(cmd) > 2:
247
    AssertCommand(cmd)
248

    
249
  # OS parameters
250
  osp = qa_config.get("os-parameters", {})
251
  for k, v in osp.items():
252
    AssertCommand(["gnt-os", "modify", "-O", v, k])
253

    
254
  # OS hypervisor parameters
255
  os_hvp = qa_config.get("os-hvp", {})
256
  for os_name in os_hvp:
257
    for hv, hvp in os_hvp[os_name].items():
258
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
259

    
260

    
261
def TestClusterRename():
262
  """gnt-cluster rename"""
263
  cmd = ["gnt-cluster", "rename", "-f"]
264

    
265
  original_name = qa_config.get("name")
266
  rename_target = qa_config.get("rename", None)
267
  if rename_target is None:
268
    print qa_utils.FormatError('"rename" entry is missing')
269
    return
270

    
271
  for data in [
272
    cmd + [rename_target],
273
    _CLUSTER_VERIFY,
274
    cmd + [original_name],
275
    _CLUSTER_VERIFY,
276
    ]:
277
    AssertCommand(data)
278

    
279

    
280
def TestClusterOob():
281
  """out-of-band framework"""
282
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
283

    
284
  AssertCommand(_CLUSTER_VERIFY)
285
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
286
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
287
                 utils.NewUUID()])
288

    
289
  AssertCommand(_CLUSTER_VERIFY, fail=True)
290

    
291
  AssertCommand(["touch", oob_path_exists])
292
  AssertCommand(["chmod", "0400", oob_path_exists])
293
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
294

    
295
  try:
296
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
297
                   "oob_program=%s" % oob_path_exists])
298

    
299
    AssertCommand(_CLUSTER_VERIFY, fail=True)
300

    
301
    AssertCommand(["chmod", "0500", oob_path_exists])
302
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
303

    
304
    AssertCommand(_CLUSTER_VERIFY)
305
  finally:
306
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
307

    
308
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
309
                 "oob_program="])
310

    
311

    
312
def TestClusterEpo():
313
  """gnt-cluster epo"""
314
  master = qa_config.GetMasterNode()
315

    
316
  # Assert that OOB is unavailable for all nodes
317
  result_output = GetCommandOutput(master.primary,
318
                                   "gnt-node list --verbose --no-headers -o"
319
                                   " powered")
320
  AssertEqual(compat.all(powered == "(unavail)"
321
                         for powered in result_output.splitlines()), True)
322

    
323
  # Conflicting
324
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
325
  # --all doesn't expect arguments
326
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
327

    
328
  # Unless --all is given master is not allowed to be in the list
329
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
330

    
331
  # This shouldn't fail
332
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
333

    
334
  # All instances should have been stopped now
335
  result_output = GetCommandOutput(master.primary,
336
                                   "gnt-instance list --no-headers -o status")
337
  # ERROR_down because the instance is stopped but not recorded as such
338
  AssertEqual(compat.all(status == "ERROR_down"
339
                         for status in result_output.splitlines()), True)
340

    
341
  # Now start everything again
342
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
343

    
344
  # All instances should have been started now
345
  result_output = GetCommandOutput(master.primary,
346
                                   "gnt-instance list --no-headers -o status")
347
  AssertEqual(compat.all(status == "running"
348
                         for status in result_output.splitlines()), True)
349

    
350

    
351
def TestClusterVerify():
352
  """gnt-cluster verify"""
353
  AssertCommand(_CLUSTER_VERIFY)
354
  AssertCommand(["gnt-cluster", "verify-disks"])
355

    
356

    
357
def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes):
358
  """gnt-cluster verify-disks with broken DRBD"""
359
  qa_daemon.TestPauseWatcher()
360

    
361
  try:
362
    info = qa_instance.GetInstanceInfo(instance.name)
363
    snode = inst_nodes[1]
364
    for idx, minor in enumerate(info["drbd-minors"][snode.primary]):
365
      if idx % 2 == 0:
366
        break_drbd_cmd = \
367
          "(drbdsetup %d down >/dev/null 2>&1;" \
368
          " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \
369
          (minor, minor)
370
      else:
371
        break_drbd_cmd = \
372
          "(drbdsetup %d detach >/dev/null 2>&1;" \
373
          " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \
374
          (minor, minor)
375
      AssertCommand(break_drbd_cmd, node=snode)
376

    
377
    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
378
                                     "gnt-cluster verify-disks")
379
    activation_msg = "Activating disks for instance '%s'" % instance.name
380
    if activation_msg not in verify_output:
381
      raise qa_error.Error("gnt-cluster verify-disks did not activate broken"
382
                           " DRBD disks:\n%s" % verify_output)
383

    
384
    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
385
                                     "gnt-cluster verify-disks")
386
    if activation_msg in verify_output:
387
      raise qa_error.Error("gnt-cluster verify-disks wants to activate broken"
388
                           " DRBD disks on second attempt:\n%s" % verify_output)
389

    
390
    AssertCommand(_CLUSTER_VERIFY)
391
  finally:
392
    qa_daemon.TestResumeWatcher()
393

    
394

    
395
def TestJobqueue():
396
  """gnt-debug test-jobqueue"""
397
  AssertCommand(["gnt-debug", "test-jobqueue"])
398

    
399

    
400
def TestDelay(node):
401
  """gnt-debug delay"""
402
  AssertCommand(["gnt-debug", "delay", "1"])
403
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
404
  AssertCommand(["gnt-debug", "delay", "--no-master",
405
                 "-n", node.primary, "1"])
406

    
407

    
408
def TestClusterReservedLvs():
409
  """gnt-cluster reserved lvs"""
410
  # if no lvm-based templates are supported, skip the test
411
  if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG):
412
    return
413
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
414
  lvname = _QA_LV_PREFIX + "test"
415
  lvfullname = "/".join([vgname, lvname])
416
  for fail, cmd in [
417
    (False, _CLUSTER_VERIFY),
418
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
419
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
420
    (True, _CLUSTER_VERIFY),
421
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
422
             "%s,.*/other-test" % lvfullname]),
423
    (False, _CLUSTER_VERIFY),
424
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
425
             ".*/%s.*" % _QA_LV_PREFIX]),
426
    (False, _CLUSTER_VERIFY),
427
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
428
    (True, _CLUSTER_VERIFY),
429
    (False, ["lvremove", "-f", lvfullname]),
430
    (False, _CLUSTER_VERIFY),
431
    ]:
432
    AssertCommand(cmd, fail=fail)
433

    
434

    
435
def TestClusterModifyEmpty():
436
  """gnt-cluster modify"""
437
  AssertCommand(["gnt-cluster", "modify"], fail=True)
438

    
439

    
440
def TestClusterModifyDisk():
441
  """gnt-cluster modify -D"""
442
  for param in _FAIL_PARAMS:
443
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
444

    
445

    
446
def _GetOtherEnabledDiskTemplate(undesired_disk_templates,
447
                                 enabled_disk_templates):
448
  """Returns one template that is not in the undesired set.
449

450
  @type undesired_disk_templates: list of string
451
  @param undesired_disk_templates: a list of disk templates that we want to
452
      exclude when drawing one disk template from the list of enabled
453
      disk templates
454
  @type enabled_disk_templates: list of string
455
  @param enabled_disk_templates: list of enabled disk templates (in QA)
456

457
  """
458
  desired_templates = list(set(enabled_disk_templates)
459
                                - set(undesired_disk_templates))
460
  if desired_templates:
461
    template = desired_templates[0]
462
  else:
463
    # If no desired disk template is available for QA, choose 'diskless' and
464
    # hope for the best.
465
    template = constants.ST_DISKLESS
466

    
467
  return template
468

    
469

    
470
def TestClusterModifyFileBasedStorageDir(
471
    file_disk_template, dir_config_key, default_dir, option_name):
472
  """Tests gnt-cluster modify wrt to file-based directory options.
473

474
  @type file_disk_template: string
475
  @param file_disk_template: file-based disk template
476
  @type dir_config_key: string
477
  @param dir_config_key: key for the QA config to retrieve the default
478
     directory value
479
  @type default_dir: string
480
  @param default_dir: default directory, if the QA config does not specify
481
     it
482
  @type option_name: string
483
  @param option_name: name of the option of 'gnt-cluster modify' to
484
     change the directory
485

486
  """
487
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
488
  assert file_disk_template in [constants.DT_FILE, constants.DT_SHARED_FILE]
489
  if not qa_config.IsTemplateSupported(file_disk_template):
490
    return
491

    
492
  # Get some non-file-based disk template to disable file storage
493
  other_disk_template = _GetOtherEnabledDiskTemplate(
494
      utils.storage.GetDiskTemplatesOfStorageType(constants.ST_FILE),
495
      enabled_disk_templates)
496

    
497
  file_storage_dir = qa_config.get(dir_config_key, default_dir)
498
  invalid_file_storage_dir = "/boot/"
499

    
500
  for fail, cmd in [
501
    (False, ["gnt-cluster", "modify",
502
             "--enabled-disk-templates=%s" % file_disk_template,
503
             "--ipolicy-disk-templates=%s" % file_disk_template]),
504
    (False, ["gnt-cluster", "modify",
505
             "--%s=%s" % (option_name, file_storage_dir)]),
506
    (False, ["gnt-cluster", "modify",
507
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
508
    # file storage dir is set to an inacceptable path, thus verify
509
    # should fail
510
    (True, ["gnt-cluster", "verify"]),
511
    # unsetting the storage dir while file storage is enabled
512
    # should fail
513
    (True, ["gnt-cluster", "modify",
514
            "--%s=" % option_name]),
515
    (False, ["gnt-cluster", "modify",
516
             "--%s=%s" % (option_name, file_storage_dir)]),
517
    (False, ["gnt-cluster", "modify",
518
             "--enabled-disk-templates=%s" % other_disk_template,
519
             "--ipolicy-disk-templates=%s" % other_disk_template]),
520
    (False, ["gnt-cluster", "modify",
521
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
522
    # file storage is set to an inacceptable path, but file storage
523
    # is disabled, thus verify should not fail
524
    (False, ["gnt-cluster", "verify"]),
525
    # unsetting the file storage dir while file storage is not enabled
526
    # should be fine
527
    (False, ["gnt-cluster", "modify",
528
             "--%s=" % option_name]),
529
    # resetting everything to sane values
530
    (False, ["gnt-cluster", "modify",
531
             "--%s=%s" % (option_name, file_storage_dir),
532
             "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
533
             "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)])
534
    ]:
535
    AssertCommand(cmd, fail=fail)
536

    
537

    
538
def TestClusterModifyFileStorageDir():
539
  """gnt-cluster modify --file-storage-dir=..."""
540
  TestClusterModifyFileBasedStorageDir(
541
      constants.DT_FILE, "default-file-storage-dir",
542
      pathutils.DEFAULT_FILE_STORAGE_DIR,
543
      "file-storage-dir")
544

    
545

    
546
def TestClusterModifySharedFileStorageDir():
547
  """gnt-cluster modify --shared-file-storage-dir=..."""
548
  TestClusterModifyFileBasedStorageDir(
549
      constants.DT_SHARED_FILE, "default-shared-file-storage-dir",
550
      pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR,
551
      "shared-file-storage-dir")
552

    
553

    
554
def TestClusterModifyDiskTemplates():
555
  """gnt-cluster modify --enabled-disk-templates=..."""
556
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
557
  default_disk_template = qa_config.GetDefaultDiskTemplate()
558

    
559
  _TestClusterModifyDiskTemplatesArguments(default_disk_template,
560
                                           enabled_disk_templates)
561
  _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates)
562

    
563
  _RestoreEnabledDiskTemplates()
564
  nodes = qa_config.AcquireManyNodes(2)
565

    
566
  instance_template = enabled_disk_templates[0]
567
  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)
568

    
569
  _TestClusterModifyUnusedDiskTemplate(instance_template)
570
  _TestClusterModifyUsedDiskTemplate(instance_template,
571
                                     enabled_disk_templates)
572

    
573
  qa_instance.TestInstanceRemove(instance)
574
  _RestoreEnabledDiskTemplates()
575

    
576

    
577
def _RestoreEnabledDiskTemplates():
578
  """Sets the list of enabled disk templates back to the list of enabled disk
579
     templates from the QA configuration. This can be used to make sure that
580
     the tests that modify the list of disk templates do not interfere with
581
     other tests.
582

583
  """
584
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
585
  cmd = ["gnt-cluster", "modify",
586
         "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
587
         "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates),
588
         ]
589

    
590
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
591
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
592
    cmd.append("--vg-name=%s" % vgname)
593

    
594
  AssertCommand(cmd, fail=False)
595

    
596

    
597
def _TestClusterModifyDiskTemplatesArguments(default_disk_template,
598
                                             enabled_disk_templates):
599
  """Tests argument handling of 'gnt-cluster modify' with respect to
600
     the parameter '--enabled-disk-templates'. This test is independent
601
     of instances.
602

603
  """
604
  _RestoreEnabledDiskTemplates()
605

    
606
  # bogus templates
607
  AssertCommand(["gnt-cluster", "modify",
608
                 "--enabled-disk-templates=pinkbunny"],
609
                fail=True)
610

    
611
  # duplicate entries do no harm
612
  AssertCommand(
613
    ["gnt-cluster", "modify",
614
     "--enabled-disk-templates=%s,%s" %
615
      (default_disk_template, default_disk_template),
616
     "--ipolicy-disk-templates=%s" % default_disk_template],
617
    fail=False)
618

    
619
  if constants.DT_DRBD8 in enabled_disk_templates:
620
    # interaction with --drbd-usermode-helper option
621
    drbd_usermode_helper = qa_config.get("drbd-usermode-helper", None)
622
    if not drbd_usermode_helper:
623
      drbd_usermode_helper = "/bin/true"
624
    # specifying a helper when drbd gets disabled is ok. Note that drbd still
625
    # has to be installed on the nodes in this case
626
    AssertCommand(["gnt-cluster", "modify",
627
                   "--drbd-usermode-helper=%s" % drbd_usermode_helper,
628
                   "--enabled-disk-templates=%s" % constants.DT_DISKLESS,
629
                   "--ipolicy-disk-templates=%s" % constants.DT_DISKLESS],
630
                   fail=False)
631
    # specifying a helper when drbd is re-enabled
632
    AssertCommand(["gnt-cluster", "modify",
633
                   "--drbd-usermode-helper=%s" % drbd_usermode_helper,
634
                   "--enabled-disk-templates=%s" %
635
                     ",".join(enabled_disk_templates),
636
                   "--ipolicy-disk-templates=%s" %
637
                     ",".join(enabled_disk_templates)],
638
                  fail=False)
639

    
640

    
641
def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates):
642
  """Tests argument handling of 'gnt-cluster modify' with respect to
643
     the parameter '--enabled-disk-templates' and '--vg-name'. This test is
644
     independent of instances.
645

646
  """
647
  if not utils.IsLvmEnabled(enabled_disk_templates):
648
    # These tests only make sense if lvm is enabled for QA
649
    return
650

    
651
  # determine an LVM and a non-LVM disk template for the tests
652
  non_lvm_template = _GetOtherEnabledDiskTemplate(utils.GetLvmDiskTemplates(),
653
                                                  enabled_disk_templates)
654
  lvm_template = list(set(enabled_disk_templates)
655
                      .intersection(set(utils.GetLvmDiskTemplates())))[0]
656

    
657
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
658

    
659
  # Clean start: unset volume group name, disable lvm storage
660
  AssertCommand(
661
    ["gnt-cluster", "modify",
662
     "--enabled-disk-templates=%s" % non_lvm_template,
663
     "--ipolicy-disk-templates=%s" % non_lvm_template,
664
     "--vg-name="],
665
    fail=False)
666

    
667
  # Try to enable lvm, when no volume group is given
668
  AssertCommand(
669
    ["gnt-cluster", "modify",
670
     "--enabled-disk-templates=%s" % lvm_template,
671
     "--ipolicy-disk-templates=%s" % lvm_template],
672
    fail=True)
673

    
674
  # Set volume group, with lvm still disabled: just a warning
675
  AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False)
676

    
677
  # Try unsetting vg name and enabling lvm at the same time
678
  AssertCommand(
679
    ["gnt-cluster", "modify",
680
     "--enabled-disk-templates=%s" % lvm_template,
681
     "--ipolicy-disk-templates=%s" % lvm_template,
682
     "--vg-name="],
683
    fail=True)
684

    
685
  # Enable lvm with vg name present
686
  AssertCommand(
687
    ["gnt-cluster", "modify",
688
     "--enabled-disk-templates=%s" % lvm_template,
689
     "--ipolicy-disk-templates=%s" % lvm_template],
690
    fail=False)
691

    
692
  # Try unsetting vg name with lvm still enabled
693
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True)
694

    
695
  # Disable lvm with vg name still set
696
  AssertCommand(
697
    ["gnt-cluster", "modify",
698
     "--enabled-disk-templates=%s" % non_lvm_template,
699
     "--ipolicy-disk-templates=%s" % non_lvm_template,
700
     ],
701
    fail=False)
702

    
703
  # Try unsetting vg name with lvm disabled
704
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False)
705

    
706
  # Set vg name and enable lvm at the same time
707
  AssertCommand(
708
    ["gnt-cluster", "modify",
709
     "--enabled-disk-templates=%s" % lvm_template,
710
     "--ipolicy-disk-templates=%s" % lvm_template,
711
     "--vg-name=%s" % vgname],
712
    fail=False)
713

    
714
  # Unset vg name and disable lvm at the same time
715
  AssertCommand(
716
    ["gnt-cluster", "modify",
717
     "--enabled-disk-templates=%s" % non_lvm_template,
718
     "--ipolicy-disk-templates=%s" % non_lvm_template,
719
     "--vg-name="],
720
    fail=False)
721

    
722
  _RestoreEnabledDiskTemplates()
723

    
724

    
725
def _TestClusterModifyUsedDiskTemplate(instance_template,
726
                                       enabled_disk_templates):
727
  """Tests that disk templates that are currently in use by instances cannot
728
     be disabled on the cluster.
729

730
  """
731
  # If the list of enabled disk templates contains only one template
732
  # we need to add some other templates, because the list of enabled disk
733
  # templates can only be set to a non-empty list.
734
  new_disk_templates = list(set(enabled_disk_templates)
735
                              - set([instance_template]))
736
  if not new_disk_templates:
737
    new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK])
738
                                - set([instance_template]))
739
  AssertCommand(
740
    ["gnt-cluster", "modify",
741
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
742
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
743
    fail=True)
744

    
745

    
746
def _TestClusterModifyUnusedDiskTemplate(instance_template):
747
  """Tests that unused disk templates can be disabled safely."""
748
  all_disk_templates = constants.DISK_TEMPLATES
749
  if not utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
750
    all_disk_templates = list(set(all_disk_templates) -
751
                              set(utils.GetLvmDiskTemplates()))
752

    
753
  AssertCommand(
754
    ["gnt-cluster", "modify",
755
     "--enabled-disk-templates=%s" % ",".join(all_disk_templates),
756
     "--ipolicy-disk-templates=%s" % ",".join(all_disk_templates)],
757
    fail=False)
758
  new_disk_templates = [instance_template]
759
  AssertCommand(
760
    ["gnt-cluster", "modify",
761
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
762
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
763
    fail=False)
764

    
765

    
766
def TestClusterModifyBe():
767
  """gnt-cluster modify -B"""
768
  for fail, cmd in [
769
    # max/min mem
770
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
771
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
772
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
773
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
774
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
775
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
776
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
777
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
778
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
779
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
780
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
781
    # vcpus
782
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
783
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
784
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
785
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
786
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
787
    # auto_balance
788
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
789
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
790
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
791
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
792
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
793
    ]:
794
    AssertCommand(cmd, fail=fail)
795

    
796
  # redo the original-requested BE parameters, if any
797
  bep = qa_config.get("backend-parameters", "")
798
  if bep:
799
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
800

    
801

    
802
def _GetClusterIPolicy():
803
  """Return the run-time values of the cluster-level instance policy.
804

805
  @rtype: tuple
806
  @return: (policy, specs), where:
807
      - policy is a dictionary of the policy values, instance specs excluded
808
      - specs is a dictionary containing only the specs, using the internal
809
        format (see L{constants.IPOLICY_DEFAULTS} for an example)
810

811
  """
812
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
813
  policy = info["Instance policy - limits for instances"]
814
  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
815

    
816
  # Sanity checks
817
  assert "minmax" in ret_specs and "std" in ret_specs
818
  assert len(ret_specs["minmax"]) > 0
819
  assert len(ret_policy) > 0
820
  return (ret_policy, ret_specs)
821

    
822

    
823
def TestClusterModifyIPolicy():
824
  """gnt-cluster modify --ipolicy-*"""
825
  basecmd = ["gnt-cluster", "modify"]
826
  (old_policy, old_specs) = _GetClusterIPolicy()
827
  for par in ["vcpu-ratio", "spindle-ratio"]:
828
    curr_val = float(old_policy[par])
829
    test_values = [
830
      (True, 1.0),
831
      (True, 1.5),
832
      (True, 2),
833
      (False, "a"),
834
      # Restore the old value
835
      (True, curr_val),
836
      ]
837
    for (good, val) in test_values:
838
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
839
      AssertCommand(cmd, fail=not good)
840
      if good:
841
        curr_val = val
842
      # Check the affected parameter
843
      (eff_policy, eff_specs) = _GetClusterIPolicy()
844
      AssertEqual(float(eff_policy[par]), curr_val)
845
      # Check everything else
846
      AssertEqual(eff_specs, old_specs)
847
      for p in eff_policy.keys():
848
        if p == par:
849
          continue
850
        AssertEqual(eff_policy[p], old_policy[p])
851

    
852
  # Allowing disk templates via ipolicy requires them to be
853
  # enabled on the cluster.
854
  if not (qa_config.IsTemplateSupported(constants.DT_PLAIN)
855
          and qa_config.IsTemplateSupported(constants.DT_DRBD8)):
856
    return
857
  # Disk templates are treated slightly differently
858
  par = "disk-templates"
859
  disp_str = "allowed disk templates"
860
  curr_val = old_policy[disp_str]
861
  test_values = [
862
    (True, constants.DT_PLAIN),
863
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
864
    (False, "thisisnotadisktemplate"),
865
    (False, ""),
866
    # Restore the old value
867
    (True, curr_val.replace(" ", "")),
868
    ]
869
  for (good, val) in test_values:
870
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
871
    AssertCommand(cmd, fail=not good)
872
    if good:
873
      curr_val = val
874
    # Check the affected parameter
875
    (eff_policy, eff_specs) = _GetClusterIPolicy()
876
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
877
    # Check everything else
878
    AssertEqual(eff_specs, old_specs)
879
    for p in eff_policy.keys():
880
      if p == disp_str:
881
        continue
882
      AssertEqual(eff_policy[p], old_policy[p])
883

    
884

    
885
def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False,
886
                         old_values=None):
887
  """Change instance specs.
888

889
  At most one of new_specs or diff_specs can be specified.
890

891
  @type new_specs: dict
892
  @param new_specs: new complete specs, in the same format returned by
893
      L{_GetClusterIPolicy}
894
  @type diff_specs: dict
895
  @param diff_specs: partial specs, it can be an incomplete specifications, but
896
      if min/max specs are specified, their number must match the number of the
897
      existing specs
898
  @type fail: bool
899
  @param fail: if the change is expected to fail
900
  @type old_values: tuple
901
  @param old_values: (old_policy, old_specs), as returned by
902
      L{_GetClusterIPolicy}
903
  @return: same as L{_GetClusterIPolicy}
904

905
  """
906
  build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
907
  return qa_utils.TestSetISpecs(
908
    new_specs=new_specs, diff_specs=diff_specs,
909
    get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd,
910
    fail=fail, old_values=old_values)
911

    
912

    
913
def TestClusterModifyISpecs():
914
  """gnt-cluster modify --specs-*"""
915
  params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
916
  (cur_policy, cur_specs) = _GetClusterIPolicy()
917
  # This test assumes that there is only one min/max bound
918
  assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
919
  for par in params:
920
    test_values = [
921
      (True, 0, 4, 12),
922
      (True, 4, 4, 12),
923
      (True, 4, 12, 12),
924
      (True, 4, 4, 4),
925
      (False, 4, 0, 12),
926
      (False, 4, 16, 12),
927
      (False, 4, 4, 0),
928
      (False, 12, 4, 4),
929
      (False, 12, 4, 0),
930
      (False, "a", 4, 12),
931
      (False, 0, "a", 12),
932
      (False, 0, 4, "a"),
933
      # This is to restore the old values
934
      (True,
935
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
936
       cur_specs[constants.ISPECS_STD][par],
937
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
938
      ]
939
    for (good, mn, st, mx) in test_values:
940
      new_vals = {
941
        constants.ISPECS_MINMAX: [{
942
          constants.ISPECS_MIN: {par: mn},
943
          constants.ISPECS_MAX: {par: mx}
944
          }],
945
        constants.ISPECS_STD: {par: st}
946
        }
947
      cur_state = (cur_policy, cur_specs)
948
      # We update cur_specs, as we've copied the values to restore already
949
      (cur_policy, cur_specs) = TestClusterSetISpecs(
950
        diff_specs=new_vals, fail=not good, old_values=cur_state)
951

    
952
    # Get the ipolicy command
953
    mnode = qa_config.GetMasterNode()
954
    initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
955
    modcmd = ["gnt-cluster", "modify"]
956
    opts = initcmd.split()
957
    assert opts[0:2] == ["gnt-cluster", "init"]
958
    for k in range(2, len(opts) - 1):
959
      if opts[k].startswith("--ipolicy-"):
960
        assert k + 2 <= len(opts)
961
        modcmd.extend(opts[k:k + 2])
962
    # Re-apply the ipolicy (this should be a no-op)
963
    AssertCommand(modcmd)
964
    new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
965
    AssertEqual(initcmd, new_initcmd)
966

    
967

    
968
def TestClusterInfo():
969
  """gnt-cluster info"""
970
  AssertCommand(["gnt-cluster", "info"])
971

    
972

    
973
def TestClusterRedistConf():
974
  """gnt-cluster redist-conf"""
975
  AssertCommand(["gnt-cluster", "redist-conf"])
976

    
977

    
978
def TestClusterGetmaster():
979
  """gnt-cluster getmaster"""
980
  AssertCommand(["gnt-cluster", "getmaster"])
981

    
982

    
983
def TestClusterVersion():
984
  """gnt-cluster version"""
985
  AssertCommand(["gnt-cluster", "version"])
986

    
987

    
988
def TestClusterRenewCrypto():
989
  """gnt-cluster renew-crypto"""
990
  master = qa_config.GetMasterNode()
991

    
992
  # Conflicting options
993
  cmd = ["gnt-cluster", "renew-crypto", "--force",
994
         "--new-cluster-certificate", "--new-confd-hmac-key"]
995
  conflicting = [
996
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
997
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
998
    ]
999
  for i in conflicting:
1000
    AssertCommand(cmd + i, fail=True)
1001

    
1002
  # Invalid RAPI certificate
1003
  cmd = ["gnt-cluster", "renew-crypto", "--force",
1004
         "--rapi-certificate=/dev/null"]
1005
  AssertCommand(cmd, fail=True)
1006

    
1007
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
1008
                                         pathutils.RAPI_CERT_FILE)
1009
  try:
1010
    # Custom RAPI certificate
1011
    fh = tempfile.NamedTemporaryFile()
1012

    
1013
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
1014
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
1015

    
1016
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
1017

    
1018
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
1019
    try:
1020
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1021
                     "--rapi-certificate=%s" % tmpcert])
1022
    finally:
1023
      AssertCommand(["rm", "-f", tmpcert])
1024

    
1025
    # Custom cluster domain secret
1026
    cds_fh = tempfile.NamedTemporaryFile()
1027
    cds_fh.write(utils.GenerateSecret())
1028
    cds_fh.write("\n")
1029
    cds_fh.flush()
1030

    
1031
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
1032
    try:
1033
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1034
                     "--cluster-domain-secret=%s" % tmpcds])
1035
    finally:
1036
      AssertCommand(["rm", "-f", tmpcds])
1037

    
1038
    # Normal case
1039
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1040
                   "--new-cluster-certificate", "--new-confd-hmac-key",
1041
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
1042

    
1043
    # Restore RAPI certificate
1044
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1045
                   "--rapi-certificate=%s" % rapi_cert_backup])
1046
  finally:
1047
    AssertCommand(["rm", "-f", rapi_cert_backup])
1048

    
1049

    
1050
def TestClusterBurnin():
1051
  """Burnin"""
1052
  master = qa_config.GetMasterNode()
1053

    
1054
  options = qa_config.get("options", {})
1055
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
1056
  parallel = options.get("burnin-in-parallel", False)
1057
  check_inst = options.get("burnin-check-instances", False)
1058
  do_rename = options.get("burnin-rename", "")
1059
  do_reboot = options.get("burnin-reboot", True)
1060
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
1061

    
1062
  # Get as many instances as we need
1063
  instances = []
1064
  try:
1065
    try:
1066
      num = qa_config.get("options", {}).get("burnin-instances", 1)
1067
      for _ in range(0, num):
1068
        instances.append(qa_config.AcquireInstance())
1069
    except qa_error.OutOfInstancesError:
1070
      print "Not enough instances, continuing anyway."
1071

    
1072
    if len(instances) < 1:
1073
      raise qa_error.Error("Burnin needs at least one instance")
1074

    
1075
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
1076
    try:
1077
      disks = qa_config.GetDiskOptions()
1078
      # Run burnin
1079
      cmd = [script,
1080
             "--os=%s" % qa_config.get("os"),
1081
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
1082
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
1083
             "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
1084
             "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
1085
             "--disk-template=%s" % disk_template]
1086
      if parallel:
1087
        cmd.append("--parallel")
1088
        cmd.append("--early-release")
1089
      if check_inst:
1090
        cmd.append("--http-check")
1091
      if do_rename:
1092
        cmd.append("--rename=%s" % do_rename)
1093
      if not do_reboot:
1094
        cmd.append("--no-reboot")
1095
      else:
1096
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
1097
      cmd += [inst.name for inst in instances]
1098
      AssertCommand(cmd)
1099
    finally:
1100
      AssertCommand(["rm", "-f", script])
1101

    
1102
  finally:
1103
    for inst in instances:
1104
      inst.Release()
1105

    
1106

    
1107
def TestClusterMasterFailover():
1108
  """gnt-cluster master-failover"""
1109
  master = qa_config.GetMasterNode()
1110
  failovermaster = qa_config.AcquireNode(exclude=master)
1111

    
1112
  cmd = ["gnt-cluster", "master-failover"]
1113
  try:
1114
    AssertCommand(cmd, node=failovermaster)
1115
    # Back to original master node
1116
    AssertCommand(cmd, node=master)
1117
  finally:
1118
    failovermaster.Release()
1119

    
1120

    
1121
def _NodeQueueDrainFile(node):
1122
  """Returns path to queue drain file for a node.
1123

1124
  """
1125
  return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE)
1126

    
1127

    
1128
def _AssertDrainFile(node, **kwargs):
1129
  """Checks for the queue drain file.
1130

1131
  """
1132
  AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs)
1133

    
1134

    
1135
def TestClusterMasterFailoverWithDrainedQueue():
1136
  """gnt-cluster master-failover with drained queue"""
1137
  master = qa_config.GetMasterNode()
1138
  failovermaster = qa_config.AcquireNode(exclude=master)
1139

    
1140
  # Ensure queue is not drained
1141
  for node in [master, failovermaster]:
1142
    _AssertDrainFile(node, fail=True)
1143

    
1144
  # Drain queue on failover master
1145
  AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)],
1146
                node=failovermaster)
1147

    
1148
  cmd = ["gnt-cluster", "master-failover"]
1149
  try:
1150
    _AssertDrainFile(failovermaster)
1151
    AssertCommand(cmd, node=failovermaster)
1152
    _AssertDrainFile(master, fail=True)
1153
    _AssertDrainFile(failovermaster, fail=True)
1154

    
1155
    # Back to original master node
1156
    AssertCommand(cmd, node=master)
1157
  finally:
1158
    failovermaster.Release()
1159

    
1160
  # Ensure queue is not drained
1161
  for node in [master, failovermaster]:
1162
    _AssertDrainFile(node, fail=True)
1163

    
1164

    
1165
def TestClusterCopyfile():
1166
  """gnt-cluster copyfile"""
1167
  master = qa_config.GetMasterNode()
1168

    
1169
  uniqueid = utils.NewUUID()
1170

    
1171
  # Create temporary file
1172
  f = tempfile.NamedTemporaryFile()
1173
  f.write(uniqueid)
1174
  f.flush()
1175
  f.seek(0)
1176

    
1177
  # Upload file to master node
1178
  testname = qa_utils.UploadFile(master.primary, f.name)
1179
  try:
1180
    # Copy file to all nodes
1181
    AssertCommand(["gnt-cluster", "copyfile", testname])
1182
    _CheckFileOnAllNodes(testname, uniqueid)
1183
  finally:
1184
    _RemoveFileFromAllNodes(testname)
1185

    
1186

    
1187
def TestClusterCommand():
1188
  """gnt-cluster command"""
1189
  uniqueid = utils.NewUUID()
1190
  rfile = "/tmp/gnt%s" % utils.NewUUID()
1191
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
1192
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
1193
                              "%s >%s" % (rcmd, rfile)])
1194

    
1195
  try:
1196
    AssertCommand(cmd)
1197
    _CheckFileOnAllNodes(rfile, uniqueid)
1198
  finally:
1199
    _RemoveFileFromAllNodes(rfile)
1200

    
1201

    
1202
def TestClusterDestroy():
1203
  """gnt-cluster destroy"""
1204
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
1205

    
1206

    
1207
def TestClusterRepairDiskSizes():
1208
  """gnt-cluster repair-disk-sizes"""
1209
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
1210

    
1211

    
1212
def TestSetExclStorCluster(newvalue):
1213
  """Set the exclusive_storage node parameter at the cluster level.
1214

1215
  @type newvalue: bool
1216
  @param newvalue: New value of exclusive_storage
1217
  @rtype: bool
1218
  @return: The old value of exclusive_storage
1219

1220
  """
1221
  es_path = ["Default node parameters", "exclusive_storage"]
1222
  oldvalue = _GetClusterField(es_path)
1223
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
1224
                 "exclusive_storage=%s" % newvalue])
1225
  effvalue = _GetClusterField(es_path)
1226
  if effvalue != newvalue:
1227
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
1228
                         " of %s" % (effvalue, newvalue))
1229
  qa_config.SetExclusiveStorage(newvalue)
1230
  return oldvalue
1231

    
1232

    
1233
def TestExclStorSharedPv(node):
1234
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
1235

1236
  """
1237
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
1238
  lvname1 = _QA_LV_PREFIX + "vol1"
1239
  lvname2 = _QA_LV_PREFIX + "vol2"
1240
  node_name = node.primary
1241
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
1242
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
1243
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
1244
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
1245
                                         constants.CV_ENODEORPHANLV])
1246
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
1247
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
1248
  AssertClusterVerify()