Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ 847a7b02

History | View | Annotate | Download (42.1 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import _constants
31
from ganeti import constants
32
from ganeti import compat
33
from ganeti import utils
34
from ganeti import pathutils
35

    
36
import qa_config
37
import qa_daemon
38
import qa_utils
39
import qa_error
40
import qa_instance
41

    
42
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
43

    
44

    
45
# Prefix for LVM volumes created by QA code during tests
46
_QA_LV_PREFIX = "qa-"
47

    
48
#: cluster verify command
49
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
50

    
51

    
52
def _RemoveFileFromAllNodes(filename):
53
  """Removes a file from all nodes.
54

55
  """
56
  for node in qa_config.get("nodes"):
57
    AssertCommand(["rm", "-f", filename], node=node)
58

    
59

    
60
def _CheckFileOnAllNodes(filename, content):
61
  """Verifies the content of the given file on all nodes.
62

63
  """
64
  cmd = utils.ShellQuoteArgs(["cat", filename])
65
  for node in qa_config.get("nodes"):
66
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
67

    
68

    
69
def _GetClusterField(field_path):
70
  """Get the value of a cluster field.
71

72
  @type field_path: list of strings
73
  @param field_path: Names of the groups/fields to navigate to get the desired
74
      value, e.g. C{["Default node parameters", "oob_program"]}
75
  @return: The effective value of the field (the actual type depends on the
76
      chosen field)
77

78
  """
79
  assert isinstance(field_path, list)
80
  assert field_path
81
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
82
  for key in field_path:
83
    ret = ret[key]
84
  return ret
85

    
86

    
87
# Cluster-verify errors (date, "ERROR", then error code)
88
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
89

    
90

    
91
def _GetCVErrorCodes(cvout):
92
  errs = set()
93
  warns = set()
94
  for l in cvout.splitlines():
95
    m = _CVERROR_RE.match(l)
96
    if m:
97
      etype = m.group(1)
98
      ecode = m.group(2)
99
      if etype == "ERROR":
100
        errs.add(ecode)
101
      elif etype == "WARNING":
102
        warns.add(ecode)
103
  return (errs, warns)
104

    
105

    
106
def _CheckVerifyErrors(actual, expected, etype):
107
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
108
  if not actual.issuperset(exp_codes):
109
    missing = exp_codes.difference(actual)
110
    raise qa_error.Error("Cluster-verify didn't return these expected"
111
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))
112

    
113

    
114
def AssertClusterVerify(fail=False, errors=None, warnings=None):
115
  """Run cluster-verify and check the result
116

117
  @type fail: bool
118
  @param fail: if cluster-verify is expected to fail instead of succeeding
119
  @type errors: list of tuples
120
  @param errors: List of CV_XXX errors that are expected; if specified, all the
121
      errors listed must appear in cluster-verify output. A non-empty value
122
      implies C{fail=True}.
123
  @type warnings: list of tuples
124
  @param warnings: Same as C{errors} but for warnings.
125

126
  """
127
  cvcmd = "gnt-cluster verify"
128
  mnode = qa_config.GetMasterNode()
129
  if errors or warnings:
130
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
131
                             fail=(fail or errors))
132
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
133
    if errors:
134
      _CheckVerifyErrors(act_errs, errors, "error")
135
    if warnings:
136
      _CheckVerifyErrors(act_warns, warnings, "warning")
137
  else:
138
    AssertCommand(cvcmd, fail=fail, node=mnode)
139

    
140

    
141
# data for testing failures due to bad keys/values for disk parameters
142
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
143
                "drbd:nonexistent=1",
144
                "drbd:resync-rate=invalid",
145
                ]
146

    
147

    
148
def TestClusterInitDisk():
149
  """gnt-cluster init -D"""
150
  name = qa_config.get("name")
151
  for param in _FAIL_PARAMS:
152
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
153

    
154

    
155
def TestClusterInit(rapi_user, rapi_secret):
156
  """gnt-cluster init"""
157
  master = qa_config.GetMasterNode()
158

    
159
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
160
  rapi_dir = os.path.dirname(rapi_users_path)
161

    
162
  # First create the RAPI credentials
163
  fh = tempfile.NamedTemporaryFile()
164
  try:
165
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
166
    fh.flush()
167

    
168
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
169
    try:
170
      AssertCommand(["mkdir", "-p", rapi_dir])
171
      AssertCommand(["mv", tmpru, rapi_users_path])
172
    finally:
173
      AssertCommand(["rm", "-f", tmpru])
174
  finally:
175
    fh.close()
176

    
177
  # Initialize cluster
178
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
179
  cmd = [
180
    "gnt-cluster", "init",
181
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
182
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
183
    "--enabled-disk-templates=%s" %
184
      ",".join(enabled_disk_templates),
185
    ]
186
  if constants.DT_FILE in enabled_disk_templates:
187
    cmd.append(
188
        "--file-storage-dir=%s" %
189
        qa_config.get("default-file-storage-dir",
190
                      pathutils.DEFAULT_FILE_STORAGE_DIR))
191

    
192
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
193
                    "nic-count"):
194
    for spec_val in ("min", "max", "std"):
195
      spec = qa_config.get("ispec_%s_%s" %
196
                           (spec_type.replace("-", "_"), spec_val), None)
197
      if spec is not None:
198
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
199

    
200
  if master.secondary:
201
    cmd.append("--secondary-ip=%s" % master.secondary)
202

    
203
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
204
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
205
    if vgname:
206
      cmd.append("--vg-name=%s" % vgname)
207
    else:
208
      raise qa_error.Error("Please specify a volume group if you enable"
209
                           " lvm-based disk templates in the QA.")
210

    
211
  master_netdev = qa_config.get("master-netdev", None)
212
  if master_netdev:
213
    cmd.append("--master-netdev=%s" % master_netdev)
214

    
215
  nicparams = qa_config.get("default-nicparams", None)
216
  if nicparams:
217
    cmd.append("--nic-parameters=%s" %
218
               ",".join(utils.FormatKeyValue(nicparams)))
219

    
220
  # Cluster value of the exclusive-storage node parameter
221
  e_s = qa_config.get("exclusive-storage")
222
  if e_s is not None:
223
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
224
  else:
225
    e_s = False
226
  qa_config.SetExclusiveStorage(e_s)
227

    
228
  extra_args = qa_config.get("cluster-init-args")
229

    
230
  if extra_args:
231
    # This option was removed in 2.10, but in order to not break QA of older
232
    # branches we remove it from the extra_args if it is in there.
233
    opt_drbd_storage = "--no-drbd-storage"
234
    if opt_drbd_storage in extra_args:
235
      extra_args.remove(opt_drbd_storage)
236
    cmd.extend(extra_args)
237

    
238
  cmd.append(qa_config.get("name"))
239

    
240
  AssertCommand(cmd)
241

    
242
  cmd = ["gnt-cluster", "modify"]
243

    
244
  # hypervisor parameter modifications
245
  hvp = qa_config.get("hypervisor-parameters", {})
246
  for k, v in hvp.items():
247
    cmd.extend(["-H", "%s:%s" % (k, v)])
248
  # backend parameter modifications
249
  bep = qa_config.get("backend-parameters", "")
250
  if bep:
251
    cmd.extend(["-B", bep])
252

    
253
  if len(cmd) > 2:
254
    AssertCommand(cmd)
255

    
256
  # OS parameters
257
  osp = qa_config.get("os-parameters", {})
258
  for k, v in osp.items():
259
    AssertCommand(["gnt-os", "modify", "-O", v, k])
260

    
261
  # OS hypervisor parameters
262
  os_hvp = qa_config.get("os-hvp", {})
263
  for os_name in os_hvp:
264
    for hv, hvp in os_hvp[os_name].items():
265
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
266

    
267

    
268
def TestClusterRename():
269
  """gnt-cluster rename"""
270
  cmd = ["gnt-cluster", "rename", "-f"]
271

    
272
  original_name = qa_config.get("name")
273
  rename_target = qa_config.get("rename", None)
274
  if rename_target is None:
275
    print qa_utils.FormatError('"rename" entry is missing')
276
    return
277

    
278
  for data in [
279
    cmd + [rename_target],
280
    _CLUSTER_VERIFY,
281
    cmd + [original_name],
282
    _CLUSTER_VERIFY,
283
    ]:
284
    AssertCommand(data)
285

    
286

    
287
def TestClusterOob():
288
  """out-of-band framework"""
289
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
290

    
291
  AssertCommand(_CLUSTER_VERIFY)
292
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
293
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
294
                 utils.NewUUID()])
295

    
296
  AssertCommand(_CLUSTER_VERIFY, fail=True)
297

    
298
  AssertCommand(["touch", oob_path_exists])
299
  AssertCommand(["chmod", "0400", oob_path_exists])
300
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
301

    
302
  try:
303
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
304
                   "oob_program=%s" % oob_path_exists])
305

    
306
    AssertCommand(_CLUSTER_VERIFY, fail=True)
307

    
308
    AssertCommand(["chmod", "0500", oob_path_exists])
309
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
310

    
311
    AssertCommand(_CLUSTER_VERIFY)
312
  finally:
313
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
314

    
315
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
316
                 "oob_program="])
317

    
318

    
319
def TestClusterEpo():
320
  """gnt-cluster epo"""
321
  master = qa_config.GetMasterNode()
322

    
323
  # Assert that OOB is unavailable for all nodes
324
  result_output = GetCommandOutput(master.primary,
325
                                   "gnt-node list --verbose --no-headers -o"
326
                                   " powered")
327
  AssertEqual(compat.all(powered == "(unavail)"
328
                         for powered in result_output.splitlines()), True)
329

    
330
  # Conflicting
331
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
332
  # --all doesn't expect arguments
333
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
334

    
335
  # Unless --all is given master is not allowed to be in the list
336
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
337

    
338
  # This shouldn't fail
339
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
340

    
341
  # All instances should have been stopped now
342
  result_output = GetCommandOutput(master.primary,
343
                                   "gnt-instance list --no-headers -o status")
344
  # ERROR_down because the instance is stopped but not recorded as such
345
  AssertEqual(compat.all(status == "ERROR_down"
346
                         for status in result_output.splitlines()), True)
347

    
348
  # Now start everything again
349
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
350

    
351
  # All instances should have been started now
352
  result_output = GetCommandOutput(master.primary,
353
                                   "gnt-instance list --no-headers -o status")
354
  AssertEqual(compat.all(status == "running"
355
                         for status in result_output.splitlines()), True)
356

    
357

    
358
def TestClusterVerify():
359
  """gnt-cluster verify"""
360
  AssertCommand(_CLUSTER_VERIFY)
361
  AssertCommand(["gnt-cluster", "verify-disks"])
362

    
363

    
364
def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes):
365
  """gnt-cluster verify-disks with broken DRBD"""
366
  qa_daemon.TestPauseWatcher()
367

    
368
  try:
369
    info = qa_instance.GetInstanceInfo(instance.name)
370
    snode = inst_nodes[1]
371
    for idx, minor in enumerate(info["drbd-minors"][snode.primary]):
372
      if idx % 2 == 0:
373
        break_drbd_cmd = \
374
          "(drbdsetup %d down >/dev/null 2>&1;" \
375
          " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \
376
          (minor, minor)
377
      else:
378
        break_drbd_cmd = \
379
          "(drbdsetup %d detach >/dev/null 2>&1;" \
380
          " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \
381
          (minor, minor)
382
      AssertCommand(break_drbd_cmd, node=snode)
383

    
384
    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
385
                                     "gnt-cluster verify-disks")
386
    activation_msg = "Activating disks for instance '%s'" % instance.name
387
    if activation_msg not in verify_output:
388
      raise qa_error.Error("gnt-cluster verify-disks did not activate broken"
389
                           " DRBD disks:\n%s" % verify_output)
390

    
391
    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
392
                                     "gnt-cluster verify-disks")
393
    if activation_msg in verify_output:
394
      raise qa_error.Error("gnt-cluster verify-disks wants to activate broken"
395
                           " DRBD disks on second attempt:\n%s" % verify_output)
396

    
397
    AssertCommand(_CLUSTER_VERIFY)
398
  finally:
399
    qa_daemon.TestResumeWatcher()
400

    
401

    
402
def TestJobqueue():
403
  """gnt-debug test-jobqueue"""
404
  AssertCommand(["gnt-debug", "test-jobqueue"])
405

    
406

    
407
def TestDelay(node):
408
  """gnt-debug delay"""
409
  AssertCommand(["gnt-debug", "delay", "1"])
410
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
411
  AssertCommand(["gnt-debug", "delay", "--no-master",
412
                 "-n", node.primary, "1"])
413

    
414

    
415
def TestClusterReservedLvs():
416
  """gnt-cluster reserved lvs"""
417
  # if no lvm-based templates are supported, skip the test
418
  if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG):
419
    return
420
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
421
  lvname = _QA_LV_PREFIX + "test"
422
  lvfullname = "/".join([vgname, lvname])
423
  for fail, cmd in [
424
    (False, _CLUSTER_VERIFY),
425
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
426
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
427
    (True, _CLUSTER_VERIFY),
428
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
429
             "%s,.*/other-test" % lvfullname]),
430
    (False, _CLUSTER_VERIFY),
431
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
432
             ".*/%s.*" % _QA_LV_PREFIX]),
433
    (False, _CLUSTER_VERIFY),
434
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
435
    (True, _CLUSTER_VERIFY),
436
    (False, ["lvremove", "-f", lvfullname]),
437
    (False, _CLUSTER_VERIFY),
438
    ]:
439
    AssertCommand(cmd, fail=fail)
440

    
441

    
442
def TestClusterModifyEmpty():
443
  """gnt-cluster modify"""
444
  AssertCommand(["gnt-cluster", "modify"], fail=True)
445

    
446

    
447
def TestClusterModifyDisk():
448
  """gnt-cluster modify -D"""
449
  for param in _FAIL_PARAMS:
450
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
451

    
452

    
453
def _GetOtherEnabledDiskTemplate(undesired_disk_templates,
454
                                 enabled_disk_templates):
455
  """Returns one template that is not in the undesired set.
456

457
  @type undesired_disk_templates: list of string
458
  @param undesired_disk_templates: a list of disk templates that we want to
459
      exclude when drawing one disk template from the list of enabled
460
      disk templates
461
  @type enabled_disk_templates: list of string
462
  @param enabled_disk_templates: list of enabled disk templates (in QA)
463

464
  """
465
  desired_templates = list(set(enabled_disk_templates)
466
                                - set(undesired_disk_templates))
467
  if desired_templates:
468
    template = desired_templates[0]
469
  else:
470
    # If no desired disk template is available for QA, choose 'diskless' and
471
    # hope for the best.
472
    template = constants.ST_DISKLESS
473

    
474
  return template
475

    
476

    
477
def TestClusterModifyFileBasedStorageDir(
478
    file_disk_template, dir_config_key, default_dir, option_name):
479
  """Tests gnt-cluster modify wrt to file-based directory options.
480

481
  @type file_disk_template: string
482
  @param file_disk_template: file-based disk template
483
  @type dir_config_key: string
484
  @param dir_config_key: key for the QA config to retrieve the default
485
     directory value
486
  @type default_dir: string
487
  @param default_dir: default directory, if the QA config does not specify
488
     it
489
  @type option_name: string
490
  @param option_name: name of the option of 'gnt-cluster modify' to
491
     change the directory
492

493
  """
494
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
495
  assert file_disk_template in [constants.DT_FILE, constants.DT_SHARED_FILE]
496
  if not qa_config.IsTemplateSupported(file_disk_template):
497
    return
498

    
499
  # Get some non-file-based disk template to disable file storage
500
  other_disk_template = _GetOtherEnabledDiskTemplate(
501
      utils.storage.GetDiskTemplatesOfStorageType(constants.ST_FILE),
502
      enabled_disk_templates)
503

    
504
  file_storage_dir = qa_config.get(dir_config_key, default_dir)
505
  invalid_file_storage_dir = "/boot/"
506

    
507
  for fail, cmd in [
508
    (False, ["gnt-cluster", "modify",
509
             "--enabled-disk-templates=%s" % file_disk_template,
510
             "--ipolicy-disk-templates=%s" % file_disk_template]),
511
    (False, ["gnt-cluster", "modify",
512
             "--%s=%s" % (option_name, file_storage_dir)]),
513
    (False, ["gnt-cluster", "modify",
514
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
515
    # file storage dir is set to an inacceptable path, thus verify
516
    # should fail
517
    (True, ["gnt-cluster", "verify"]),
518
    # unsetting the storage dir while file storage is enabled
519
    # should fail
520
    (True, ["gnt-cluster", "modify",
521
            "--%s=" % option_name]),
522
    (False, ["gnt-cluster", "modify",
523
             "--%s=%s" % (option_name, file_storage_dir)]),
524
    (False, ["gnt-cluster", "modify",
525
             "--enabled-disk-templates=%s" % other_disk_template,
526
             "--ipolicy-disk-templates=%s" % other_disk_template]),
527
    (False, ["gnt-cluster", "modify",
528
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
529
    # file storage is set to an inacceptable path, but file storage
530
    # is disabled, thus verify should not fail
531
    (False, ["gnt-cluster", "verify"]),
532
    # unsetting the file storage dir while file storage is not enabled
533
    # should be fine
534
    (False, ["gnt-cluster", "modify",
535
             "--%s=" % option_name]),
536
    # resetting everything to sane values
537
    (False, ["gnt-cluster", "modify",
538
             "--%s=%s" % (option_name, file_storage_dir),
539
             "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
540
             "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)])
541
    ]:
542
    AssertCommand(cmd, fail=fail)
543

    
544

    
545
def TestClusterModifyFileStorageDir():
546
  """gnt-cluster modify --file-storage-dir=..."""
547
  TestClusterModifyFileBasedStorageDir(
548
      constants.DT_FILE, "default-file-storage-dir",
549
      pathutils.DEFAULT_FILE_STORAGE_DIR,
550
      "file-storage-dir")
551

    
552

    
553
def TestClusterModifySharedFileStorageDir():
554
  """gnt-cluster modify --shared-file-storage-dir=..."""
555
  TestClusterModifyFileBasedStorageDir(
556
      constants.DT_SHARED_FILE, "default-shared-file-storage-dir",
557
      pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR,
558
      "shared-file-storage-dir")
559

    
560

    
561
def TestClusterModifyDiskTemplates():
562
  """gnt-cluster modify --enabled-disk-templates=..."""
563
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
564
  default_disk_template = qa_config.GetDefaultDiskTemplate()
565

    
566
  _TestClusterModifyDiskTemplatesArguments(default_disk_template)
567
  _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates)
568
  _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates)
569

    
570
  _RestoreEnabledDiskTemplates()
571
  nodes = qa_config.AcquireManyNodes(2)
572

    
573
  instance_template = enabled_disk_templates[0]
574
  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)
575

    
576
  _TestClusterModifyUsedDiskTemplate(instance_template,
577
                                     enabled_disk_templates)
578

    
579
  qa_instance.TestInstanceRemove(instance)
580
  _RestoreEnabledDiskTemplates()
581

    
582

    
583
def _RestoreEnabledDiskTemplates():
584
  """Sets the list of enabled disk templates back to the list of enabled disk
585
     templates from the QA configuration. This can be used to make sure that
586
     the tests that modify the list of disk templates do not interfere with
587
     other tests.
588

589
  """
590
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
591
  cmd = ["gnt-cluster", "modify",
592
         "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
593
         "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates),
594
         ]
595

    
596
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
597
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
598
    cmd.append("--vg-name=%s" % vgname)
599

    
600
  AssertCommand(cmd, fail=False)
601

    
602

    
603
def _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates):
604
  """Tests argument handling of 'gnt-cluster modify' with respect to
605
     the parameter '--drbd-usermode-helper'. This test is independent
606
     of instances.
607

608
  """
609
  _RestoreEnabledDiskTemplates()
610

    
611
  if constants.DT_DRBD8 not in enabled_disk_templates:
612
    return
613
  if constants.DT_PLAIN not in enabled_disk_templates:
614
    return
615

    
616
  drbd_usermode_helper = qa_config.get("drbd-usermode-helper", "/bin/true")
617
  bogus_usermode_helper = "/tmp/pinkbunny"
618
  for command, fail in [
619
    (["gnt-cluster", "modify",
620
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
621
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
622
    (["gnt-cluster", "modify",
623
      "--drbd-usermode-helper=%s" % drbd_usermode_helper], False),
624
    (["gnt-cluster", "modify",
625
      "--drbd-usermode-helper=%s" % bogus_usermode_helper], True),
626
    # unsetting helper when DRBD is enabled should not work
627
    (["gnt-cluster", "modify",
628
      "--drbd-usermode-helper="], True),
629
    (["gnt-cluster", "modify",
630
      "--enabled-disk-templates=%s" % constants.DT_PLAIN,
631
      "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False),
632
    (["gnt-cluster", "modify",
633
      "--drbd-usermode-helper="], False),
634
    (["gnt-cluster", "modify",
635
      "--drbd-usermode-helper=%s" % drbd_usermode_helper], False),
636
    (["gnt-cluster", "modify",
637
      "--drbd-usermode-helper=%s" % drbd_usermode_helper,
638
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
639
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
640
    (["gnt-cluster", "modify",
641
      "--drbd-usermode-helper=",
642
      "--enabled-disk-templates=%s" % constants.DT_PLAIN,
643
      "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False),
644
    (["gnt-cluster", "modify",
645
      "--drbd-usermode-helper=%s" % drbd_usermode_helper,
646
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
647
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
648
    ]:
649
    AssertCommand(command, fail=fail)
650
  _RestoreEnabledDiskTemplates()
651

    
652

    
653
def _TestClusterModifyDiskTemplatesArguments(default_disk_template):
654
  """Tests argument handling of 'gnt-cluster modify' with respect to
655
     the parameter '--enabled-disk-templates'. This test is independent
656
     of instances.
657

658
  """
659
  _RestoreEnabledDiskTemplates()
660

    
661
  # bogus templates
662
  AssertCommand(["gnt-cluster", "modify",
663
                 "--enabled-disk-templates=pinkbunny"],
664
                fail=True)
665

    
666
  # duplicate entries do no harm
667
  AssertCommand(
668
    ["gnt-cluster", "modify",
669
     "--enabled-disk-templates=%s,%s" %
670
      (default_disk_template, default_disk_template),
671
     "--ipolicy-disk-templates=%s" % default_disk_template],
672
    fail=False)
673

    
674

    
675
def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates):
676
  """Tests argument handling of 'gnt-cluster modify' with respect to
677
     the parameter '--enabled-disk-templates' and '--vg-name'. This test is
678
     independent of instances.
679

680
  """
681
  if not utils.IsLvmEnabled(enabled_disk_templates):
682
    # These tests only make sense if lvm is enabled for QA
683
    return
684

    
685
  # determine an LVM and a non-LVM disk template for the tests
686
  non_lvm_template = _GetOtherEnabledDiskTemplate(constants.DTS_LVM,
687
                                                  enabled_disk_templates)
688
  lvm_template = list(set(enabled_disk_templates) & constants.DTS_LVM)[0]
689

    
690
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
691

    
692
  # Clean start: unset volume group name, disable lvm storage
693
  AssertCommand(
694
    ["gnt-cluster", "modify",
695
     "--enabled-disk-templates=%s" % non_lvm_template,
696
     "--ipolicy-disk-templates=%s" % non_lvm_template,
697
     "--vg-name="],
698
    fail=False)
699

    
700
  # Try to enable lvm, when no volume group is given
701
  AssertCommand(
702
    ["gnt-cluster", "modify",
703
     "--enabled-disk-templates=%s" % lvm_template,
704
     "--ipolicy-disk-templates=%s" % lvm_template],
705
    fail=True)
706

    
707
  # Set volume group, with lvm still disabled: just a warning
708
  AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False)
709

    
710
  # Try unsetting vg name and enabling lvm at the same time
711
  AssertCommand(
712
    ["gnt-cluster", "modify",
713
     "--enabled-disk-templates=%s" % lvm_template,
714
     "--ipolicy-disk-templates=%s" % lvm_template,
715
     "--vg-name="],
716
    fail=True)
717

    
718
  # Enable lvm with vg name present
719
  AssertCommand(
720
    ["gnt-cluster", "modify",
721
     "--enabled-disk-templates=%s" % lvm_template,
722
     "--ipolicy-disk-templates=%s" % lvm_template],
723
    fail=False)
724

    
725
  # Try unsetting vg name with lvm still enabled
726
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True)
727

    
728
  # Disable lvm with vg name still set
729
  AssertCommand(
730
    ["gnt-cluster", "modify",
731
     "--enabled-disk-templates=%s" % non_lvm_template,
732
     "--ipolicy-disk-templates=%s" % non_lvm_template,
733
     ],
734
    fail=False)
735

    
736
  # Try unsetting vg name with lvm disabled
737
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False)
738

    
739
  # Set vg name and enable lvm at the same time
740
  AssertCommand(
741
    ["gnt-cluster", "modify",
742
     "--enabled-disk-templates=%s" % lvm_template,
743
     "--ipolicy-disk-templates=%s" % lvm_template,
744
     "--vg-name=%s" % vgname],
745
    fail=False)
746

    
747
  # Unset vg name and disable lvm at the same time
748
  AssertCommand(
749
    ["gnt-cluster", "modify",
750
     "--enabled-disk-templates=%s" % non_lvm_template,
751
     "--ipolicy-disk-templates=%s" % non_lvm_template,
752
     "--vg-name="],
753
    fail=False)
754

    
755
  _RestoreEnabledDiskTemplates()
756

    
757

    
758
def _TestClusterModifyUsedDiskTemplate(instance_template,
759
                                       enabled_disk_templates):
760
  """Tests that disk templates that are currently in use by instances cannot
761
     be disabled on the cluster.
762

763
  """
764
  # If the list of enabled disk templates contains only one template
765
  # we need to add some other templates, because the list of enabled disk
766
  # templates can only be set to a non-empty list.
767
  new_disk_templates = list(set(enabled_disk_templates)
768
                              - set([instance_template]))
769
  if not new_disk_templates:
770
    new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK])
771
                                - set([instance_template]))
772
  AssertCommand(
773
    ["gnt-cluster", "modify",
774
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
775
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
776
    fail=True)
777

    
778

    
779
def TestClusterModifyBe():
780
  """gnt-cluster modify -B"""
781
  for fail, cmd in [
782
    # max/min mem
783
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
784
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
785
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
786
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
787
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
788
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
789
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
790
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
791
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
792
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
793
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
794
    # vcpus
795
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
796
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
797
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
798
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
799
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
800
    # auto_balance
801
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
802
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
803
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
804
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
805
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
806
    ]:
807
    AssertCommand(cmd, fail=fail)
808

    
809
  # redo the original-requested BE parameters, if any
810
  bep = qa_config.get("backend-parameters", "")
811
  if bep:
812
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
813

    
814

    
815
def _GetClusterIPolicy():
816
  """Return the run-time values of the cluster-level instance policy.
817

818
  @rtype: tuple
819
  @return: (policy, specs), where:
820
      - policy is a dictionary of the policy values, instance specs excluded
821
      - specs is a dictionary containing only the specs, using the internal
822
        format (see L{constants.IPOLICY_DEFAULTS} for an example)
823

824
  """
825
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
826
  policy = info["Instance policy - limits for instances"]
827
  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
828

    
829
  # Sanity checks
830
  assert "minmax" in ret_specs and "std" in ret_specs
831
  assert len(ret_specs["minmax"]) > 0
832
  assert len(ret_policy) > 0
833
  return (ret_policy, ret_specs)
834

    
835

    
836
def TestClusterModifyIPolicy():
837
  """gnt-cluster modify --ipolicy-*"""
838
  basecmd = ["gnt-cluster", "modify"]
839
  (old_policy, old_specs) = _GetClusterIPolicy()
840
  for par in ["vcpu-ratio", "spindle-ratio"]:
841
    curr_val = float(old_policy[par])
842
    test_values = [
843
      (True, 1.0),
844
      (True, 1.5),
845
      (True, 2),
846
      (False, "a"),
847
      # Restore the old value
848
      (True, curr_val),
849
      ]
850
    for (good, val) in test_values:
851
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
852
      AssertCommand(cmd, fail=not good)
853
      if good:
854
        curr_val = val
855
      # Check the affected parameter
856
      (eff_policy, eff_specs) = _GetClusterIPolicy()
857
      AssertEqual(float(eff_policy[par]), curr_val)
858
      # Check everything else
859
      AssertEqual(eff_specs, old_specs)
860
      for p in eff_policy.keys():
861
        if p == par:
862
          continue
863
        AssertEqual(eff_policy[p], old_policy[p])
864

    
865
  # Allowing disk templates via ipolicy requires them to be
866
  # enabled on the cluster.
867
  if not (qa_config.IsTemplateSupported(constants.DT_PLAIN)
868
          and qa_config.IsTemplateSupported(constants.DT_DRBD8)):
869
    return
870
  # Disk templates are treated slightly differently
871
  par = "disk-templates"
872
  disp_str = "allowed disk templates"
873
  curr_val = old_policy[disp_str]
874
  test_values = [
875
    (True, constants.DT_PLAIN),
876
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
877
    (False, "thisisnotadisktemplate"),
878
    (False, ""),
879
    # Restore the old value
880
    (True, curr_val.replace(" ", "")),
881
    ]
882
  for (good, val) in test_values:
883
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
884
    AssertCommand(cmd, fail=not good)
885
    if good:
886
      curr_val = val
887
    # Check the affected parameter
888
    (eff_policy, eff_specs) = _GetClusterIPolicy()
889
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
890
    # Check everything else
891
    AssertEqual(eff_specs, old_specs)
892
    for p in eff_policy.keys():
893
      if p == disp_str:
894
        continue
895
      AssertEqual(eff_policy[p], old_policy[p])
896

    
897

    
898
def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False,
899
                         old_values=None):
900
  """Change instance specs.
901

902
  At most one of new_specs or diff_specs can be specified.
903

904
  @type new_specs: dict
905
  @param new_specs: new complete specs, in the same format returned by
906
      L{_GetClusterIPolicy}
907
  @type diff_specs: dict
908
  @param diff_specs: partial specs, it can be an incomplete specifications, but
909
      if min/max specs are specified, their number must match the number of the
910
      existing specs
911
  @type fail: bool
912
  @param fail: if the change is expected to fail
913
  @type old_values: tuple
914
  @param old_values: (old_policy, old_specs), as returned by
915
      L{_GetClusterIPolicy}
916
  @return: same as L{_GetClusterIPolicy}
917

918
  """
919
  build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
920
  return qa_utils.TestSetISpecs(
921
    new_specs=new_specs, diff_specs=diff_specs,
922
    get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd,
923
    fail=fail, old_values=old_values)
924

    
925

    
926
def TestClusterModifyISpecs():
927
  """gnt-cluster modify --specs-*"""
928
  params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
929
  (cur_policy, cur_specs) = _GetClusterIPolicy()
930
  # This test assumes that there is only one min/max bound
931
  assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
932
  for par in params:
933
    test_values = [
934
      (True, 0, 4, 12),
935
      (True, 4, 4, 12),
936
      (True, 4, 12, 12),
937
      (True, 4, 4, 4),
938
      (False, 4, 0, 12),
939
      (False, 4, 16, 12),
940
      (False, 4, 4, 0),
941
      (False, 12, 4, 4),
942
      (False, 12, 4, 0),
943
      (False, "a", 4, 12),
944
      (False, 0, "a", 12),
945
      (False, 0, 4, "a"),
946
      # This is to restore the old values
947
      (True,
948
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
949
       cur_specs[constants.ISPECS_STD][par],
950
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
951
      ]
952
    for (good, mn, st, mx) in test_values:
953
      new_vals = {
954
        constants.ISPECS_MINMAX: [{
955
          constants.ISPECS_MIN: {par: mn},
956
          constants.ISPECS_MAX: {par: mx}
957
          }],
958
        constants.ISPECS_STD: {par: st}
959
        }
960
      cur_state = (cur_policy, cur_specs)
961
      # We update cur_specs, as we've copied the values to restore already
962
      (cur_policy, cur_specs) = TestClusterSetISpecs(
963
        diff_specs=new_vals, fail=not good, old_values=cur_state)
964

    
965
    # Get the ipolicy command
966
    mnode = qa_config.GetMasterNode()
967
    initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
968
    modcmd = ["gnt-cluster", "modify"]
969
    opts = initcmd.split()
970
    assert opts[0:2] == ["gnt-cluster", "init"]
971
    for k in range(2, len(opts) - 1):
972
      if opts[k].startswith("--ipolicy-"):
973
        assert k + 2 <= len(opts)
974
        modcmd.extend(opts[k:k + 2])
975
    # Re-apply the ipolicy (this should be a no-op)
976
    AssertCommand(modcmd)
977
    new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
978
    AssertEqual(initcmd, new_initcmd)
979

    
980

    
981
def TestClusterInfo():
982
  """gnt-cluster info"""
983
  AssertCommand(["gnt-cluster", "info"])
984

    
985

    
986
def TestClusterRedistConf():
987
  """gnt-cluster redist-conf"""
988
  AssertCommand(["gnt-cluster", "redist-conf"])
989

    
990

    
991
def TestClusterGetmaster():
992
  """gnt-cluster getmaster"""
993
  AssertCommand(["gnt-cluster", "getmaster"])
994

    
995

    
996
def TestClusterVersion():
997
  """gnt-cluster version"""
998
  AssertCommand(["gnt-cluster", "version"])
999

    
1000

    
1001
def TestClusterRenewCrypto():
1002
  """gnt-cluster renew-crypto"""
1003
  master = qa_config.GetMasterNode()
1004

    
1005
  # Conflicting options
1006
  cmd = ["gnt-cluster", "renew-crypto", "--force",
1007
         "--new-cluster-certificate", "--new-confd-hmac-key"]
1008
  conflicting = [
1009
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
1010
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
1011
    ]
1012
  for i in conflicting:
1013
    AssertCommand(cmd + i, fail=True)
1014

    
1015
  # Invalid RAPI certificate
1016
  cmd = ["gnt-cluster", "renew-crypto", "--force",
1017
         "--rapi-certificate=/dev/null"]
1018
  AssertCommand(cmd, fail=True)
1019

    
1020
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
1021
                                         pathutils.RAPI_CERT_FILE)
1022
  try:
1023
    # Custom RAPI certificate
1024
    fh = tempfile.NamedTemporaryFile()
1025

    
1026
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
1027
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
1028

    
1029
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
1030

    
1031
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
1032
    try:
1033
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1034
                     "--rapi-certificate=%s" % tmpcert])
1035
    finally:
1036
      AssertCommand(["rm", "-f", tmpcert])
1037

    
1038
    # Custom cluster domain secret
1039
    cds_fh = tempfile.NamedTemporaryFile()
1040
    cds_fh.write(utils.GenerateSecret())
1041
    cds_fh.write("\n")
1042
    cds_fh.flush()
1043

    
1044
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
1045
    try:
1046
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1047
                     "--cluster-domain-secret=%s" % tmpcds])
1048
    finally:
1049
      AssertCommand(["rm", "-f", tmpcds])
1050

    
1051
    # Normal case
1052
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1053
                   "--new-cluster-certificate", "--new-confd-hmac-key",
1054
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
1055

    
1056
    # Restore RAPI certificate
1057
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1058
                   "--rapi-certificate=%s" % rapi_cert_backup])
1059
  finally:
1060
    AssertCommand(["rm", "-f", rapi_cert_backup])
1061

    
1062

    
1063
def TestClusterBurnin():
1064
  """Burnin"""
1065
  master = qa_config.GetMasterNode()
1066

    
1067
  options = qa_config.get("options", {})
1068
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
1069
  parallel = options.get("burnin-in-parallel", False)
1070
  check_inst = options.get("burnin-check-instances", False)
1071
  do_rename = options.get("burnin-rename", "")
1072
  do_reboot = options.get("burnin-reboot", True)
1073
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
1074

    
1075
  # Get as many instances as we need
1076
  instances = []
1077
  try:
1078
    try:
1079
      num = qa_config.get("options", {}).get("burnin-instances", 1)
1080
      for _ in range(0, num):
1081
        instances.append(qa_config.AcquireInstance())
1082
    except qa_error.OutOfInstancesError:
1083
      print "Not enough instances, continuing anyway."
1084

    
1085
    if len(instances) < 1:
1086
      raise qa_error.Error("Burnin needs at least one instance")
1087

    
1088
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
1089
    try:
1090
      disks = qa_config.GetDiskOptions()
1091
      # Run burnin
1092
      cmd = ["env",
1093
             "PYTHONPATH=%s" % _constants.VERSIONEDSHAREDIR,
1094
             script,
1095
             "--os=%s" % qa_config.get("os"),
1096
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
1097
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
1098
             "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
1099
             "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
1100
             "--disk-template=%s" % disk_template]
1101
      if parallel:
1102
        cmd.append("--parallel")
1103
        cmd.append("--early-release")
1104
      if check_inst:
1105
        cmd.append("--http-check")
1106
      if do_rename:
1107
        cmd.append("--rename=%s" % do_rename)
1108
      if not do_reboot:
1109
        cmd.append("--no-reboot")
1110
      else:
1111
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
1112
      cmd += [inst.name for inst in instances]
1113
      AssertCommand(cmd)
1114
    finally:
1115
      AssertCommand(["rm", "-f", script])
1116

    
1117
  finally:
1118
    for inst in instances:
1119
      inst.Release()
1120

    
1121

    
1122
def TestClusterMasterFailover():
1123
  """gnt-cluster master-failover"""
1124
  master = qa_config.GetMasterNode()
1125
  failovermaster = qa_config.AcquireNode(exclude=master)
1126

    
1127
  cmd = ["gnt-cluster", "master-failover"]
1128
  try:
1129
    AssertCommand(cmd, node=failovermaster)
1130
    # Back to original master node
1131
    AssertCommand(cmd, node=master)
1132
  finally:
1133
    failovermaster.Release()
1134

    
1135

    
1136
def _NodeQueueDrainFile(node):
1137
  """Returns path to queue drain file for a node.
1138

1139
  """
1140
  return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE)
1141

    
1142

    
1143
def _AssertDrainFile(node, **kwargs):
1144
  """Checks for the queue drain file.
1145

1146
  """
1147
  AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs)
1148

    
1149

    
1150
def TestClusterMasterFailoverWithDrainedQueue():
1151
  """gnt-cluster master-failover with drained queue"""
1152
  master = qa_config.GetMasterNode()
1153
  failovermaster = qa_config.AcquireNode(exclude=master)
1154

    
1155
  # Ensure queue is not drained
1156
  for node in [master, failovermaster]:
1157
    _AssertDrainFile(node, fail=True)
1158

    
1159
  # Drain queue on failover master
1160
  AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)],
1161
                node=failovermaster)
1162

    
1163
  cmd = ["gnt-cluster", "master-failover"]
1164
  try:
1165
    _AssertDrainFile(failovermaster)
1166
    AssertCommand(cmd, node=failovermaster)
1167
    _AssertDrainFile(master, fail=True)
1168
    _AssertDrainFile(failovermaster, fail=True)
1169

    
1170
    # Back to original master node
1171
    AssertCommand(cmd, node=master)
1172
  finally:
1173
    failovermaster.Release()
1174

    
1175
  # Ensure queue is not drained
1176
  for node in [master, failovermaster]:
1177
    _AssertDrainFile(node, fail=True)
1178

    
1179

    
1180
def TestClusterCopyfile():
1181
  """gnt-cluster copyfile"""
1182
  master = qa_config.GetMasterNode()
1183

    
1184
  uniqueid = utils.NewUUID()
1185

    
1186
  # Create temporary file
1187
  f = tempfile.NamedTemporaryFile()
1188
  f.write(uniqueid)
1189
  f.flush()
1190
  f.seek(0)
1191

    
1192
  # Upload file to master node
1193
  testname = qa_utils.UploadFile(master.primary, f.name)
1194
  try:
1195
    # Copy file to all nodes
1196
    AssertCommand(["gnt-cluster", "copyfile", testname])
1197
    _CheckFileOnAllNodes(testname, uniqueid)
1198
  finally:
1199
    _RemoveFileFromAllNodes(testname)
1200

    
1201

    
1202
def TestClusterCommand():
1203
  """gnt-cluster command"""
1204
  uniqueid = utils.NewUUID()
1205
  rfile = "/tmp/gnt%s" % utils.NewUUID()
1206
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
1207
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
1208
                              "%s >%s" % (rcmd, rfile)])
1209

    
1210
  try:
1211
    AssertCommand(cmd)
1212
    _CheckFileOnAllNodes(rfile, uniqueid)
1213
  finally:
1214
    _RemoveFileFromAllNodes(rfile)
1215

    
1216

    
1217
def TestClusterDestroy():
1218
  """gnt-cluster destroy"""
1219
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
1220

    
1221

    
1222
def TestClusterRepairDiskSizes():
1223
  """gnt-cluster repair-disk-sizes"""
1224
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
1225

    
1226

    
1227
def TestSetExclStorCluster(newvalue):
1228
  """Set the exclusive_storage node parameter at the cluster level.
1229

1230
  @type newvalue: bool
1231
  @param newvalue: New value of exclusive_storage
1232
  @rtype: bool
1233
  @return: The old value of exclusive_storage
1234

1235
  """
1236
  es_path = ["Default node parameters", "exclusive_storage"]
1237
  oldvalue = _GetClusterField(es_path)
1238
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
1239
                 "exclusive_storage=%s" % newvalue])
1240
  effvalue = _GetClusterField(es_path)
1241
  if effvalue != newvalue:
1242
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
1243
                         " of %s" % (effvalue, newvalue))
1244
  qa_config.SetExclusiveStorage(newvalue)
1245
  return oldvalue
1246

    
1247

    
1248
def TestExclStorSharedPv(node):
1249
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
1250

1251
  """
1252
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
1253
  lvname1 = _QA_LV_PREFIX + "vol1"
1254
  lvname2 = _QA_LV_PREFIX + "vol2"
1255
  node_name = node.primary
1256
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
1257
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
1258
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
1259
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
1260
                                         constants.CV_ENODEORPHANLV])
1261
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
1262
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
1263
  AssertClusterVerify()