Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ d48c944b

History | View | Annotate | Download (42.9 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import _constants
31
from ganeti import constants
32
from ganeti import compat
33
from ganeti import utils
34
from ganeti import pathutils
35

    
36
import qa_config
37
import qa_daemon
38
import qa_utils
39
import qa_error
40
import qa_instance
41

    
42
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
43

    
44

    
45
# Prefix for LVM volumes created by QA code during tests
46
_QA_LV_PREFIX = "qa-"
47

    
48
#: cluster verify command
49
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
50

    
51

    
52
def _RemoveFileFromAllNodes(filename):
53
  """Removes a file from all nodes.
54

55
  """
56
  for node in qa_config.get("nodes"):
57
    AssertCommand(["rm", "-f", filename], node=node)
58

    
59

    
60
def _CheckFileOnAllNodes(filename, content):
61
  """Verifies the content of the given file on all nodes.
62

63
  """
64
  cmd = utils.ShellQuoteArgs(["cat", filename])
65
  for node in qa_config.get("nodes"):
66
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
67

    
68

    
69
def _GetClusterField(field_path):
70
  """Get the value of a cluster field.
71

72
  @type field_path: list of strings
73
  @param field_path: Names of the groups/fields to navigate to get the desired
74
      value, e.g. C{["Default node parameters", "oob_program"]}
75
  @return: The effective value of the field (the actual type depends on the
76
      chosen field)
77

78
  """
79
  assert isinstance(field_path, list)
80
  assert field_path
81
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
82
  for key in field_path:
83
    ret = ret[key]
84
  return ret
85

    
86

    
87
# Cluster-verify errors (date, "ERROR", then error code)
88
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
89

    
90

    
91
def _GetCVErrorCodes(cvout):
92
  errs = set()
93
  warns = set()
94
  for l in cvout.splitlines():
95
    m = _CVERROR_RE.match(l)
96
    if m:
97
      etype = m.group(1)
98
      ecode = m.group(2)
99
      if etype == "ERROR":
100
        errs.add(ecode)
101
      elif etype == "WARNING":
102
        warns.add(ecode)
103
  return (errs, warns)
104

    
105

    
106
def _CheckVerifyErrors(actual, expected, etype):
107
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
108
  if not actual.issuperset(exp_codes):
109
    missing = exp_codes.difference(actual)
110
    raise qa_error.Error("Cluster-verify didn't return these expected"
111
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))
112

    
113

    
114
def AssertClusterVerify(fail=False, errors=None, warnings=None):
115
  """Run cluster-verify and check the result
116

117
  @type fail: bool
118
  @param fail: if cluster-verify is expected to fail instead of succeeding
119
  @type errors: list of tuples
120
  @param errors: List of CV_XXX errors that are expected; if specified, all the
121
      errors listed must appear in cluster-verify output. A non-empty value
122
      implies C{fail=True}.
123
  @type warnings: list of tuples
124
  @param warnings: Same as C{errors} but for warnings.
125

126
  """
127
  cvcmd = "gnt-cluster verify"
128
  mnode = qa_config.GetMasterNode()
129
  if errors or warnings:
130
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
131
                             fail=(fail or errors))
132
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
133
    if errors:
134
      _CheckVerifyErrors(act_errs, errors, "error")
135
    if warnings:
136
      _CheckVerifyErrors(act_warns, warnings, "warning")
137
  else:
138
    AssertCommand(cvcmd, fail=fail, node=mnode)
139

    
140

    
141
# data for testing failures due to bad keys/values for disk parameters
142
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
143
                "drbd:nonexistent=1",
144
                "drbd:resync-rate=invalid",
145
                ]
146

    
147

    
148
def TestClusterInitDisk():
149
  """gnt-cluster init -D"""
150
  name = qa_config.get("name")
151
  for param in _FAIL_PARAMS:
152
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
153

    
154

    
155
def TestClusterInit(rapi_user, rapi_secret):
156
  """gnt-cluster init"""
157
  master = qa_config.GetMasterNode()
158

    
159
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
160
  rapi_dir = os.path.dirname(rapi_users_path)
161

    
162
  # First create the RAPI credentials
163
  fh = tempfile.NamedTemporaryFile()
164
  try:
165
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
166
    fh.flush()
167

    
168
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
169
    try:
170
      AssertCommand(["mkdir", "-p", rapi_dir])
171
      AssertCommand(["mv", tmpru, rapi_users_path])
172
    finally:
173
      AssertCommand(["rm", "-f", tmpru])
174
  finally:
175
    fh.close()
176

    
177
  # Initialize cluster
178
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
179
  cmd = [
180
    "gnt-cluster", "init",
181
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
182
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
183
    "--enabled-disk-templates=%s" %
184
      ",".join(enabled_disk_templates),
185
    ]
186
  if constants.DT_FILE in enabled_disk_templates:
187
    cmd.append(
188
        "--file-storage-dir=%s" %
189
        qa_config.get("default-file-storage-dir",
190
                      pathutils.DEFAULT_FILE_STORAGE_DIR))
191

    
192
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
193
                    "nic-count"):
194
    for spec_val in ("min", "max", "std"):
195
      spec = qa_config.get("ispec_%s_%s" %
196
                           (spec_type.replace("-", "_"), spec_val), None)
197
      if spec is not None:
198
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
199

    
200
  if master.secondary:
201
    cmd.append("--secondary-ip=%s" % master.secondary)
202

    
203
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
204
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
205
    if vgname:
206
      cmd.append("--vg-name=%s" % vgname)
207
    else:
208
      raise qa_error.Error("Please specify a volume group if you enable"
209
                           " lvm-based disk templates in the QA.")
210

    
211
  master_netdev = qa_config.get("master-netdev", None)
212
  if master_netdev:
213
    cmd.append("--master-netdev=%s" % master_netdev)
214

    
215
  nicparams = qa_config.get("default-nicparams", None)
216
  if nicparams:
217
    cmd.append("--nic-parameters=%s" %
218
               ",".join(utils.FormatKeyValue(nicparams)))
219

    
220
  # Cluster value of the exclusive-storage node parameter
221
  e_s = qa_config.get("exclusive-storage")
222
  if e_s is not None:
223
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
224
  else:
225
    e_s = False
226
  qa_config.SetExclusiveStorage(e_s)
227

    
228
  extra_args = qa_config.get("cluster-init-args")
229

    
230
  if extra_args:
231
    # This option was removed in 2.10, but in order to not break QA of older
232
    # branches we remove it from the extra_args if it is in there.
233
    opt_drbd_storage = "--no-drbd-storage"
234
    if opt_drbd_storage in extra_args:
235
      extra_args.remove(opt_drbd_storage)
236
    cmd.extend(extra_args)
237

    
238
  cmd.append(qa_config.get("name"))
239

    
240
  AssertCommand(cmd)
241

    
242
  cmd = ["gnt-cluster", "modify"]
243

    
244
  # hypervisor parameter modifications
245
  hvp = qa_config.get("hypervisor-parameters", {})
246
  for k, v in hvp.items():
247
    cmd.extend(["-H", "%s:%s" % (k, v)])
248
  # backend parameter modifications
249
  bep = qa_config.get("backend-parameters", "")
250
  if bep:
251
    cmd.extend(["-B", bep])
252

    
253
  if len(cmd) > 2:
254
    AssertCommand(cmd)
255

    
256
  # OS parameters
257
  osp = qa_config.get("os-parameters", {})
258
  for k, v in osp.items():
259
    AssertCommand(["gnt-os", "modify", "-O", v, k])
260

    
261
  # OS hypervisor parameters
262
  os_hvp = qa_config.get("os-hvp", {})
263
  for os_name in os_hvp:
264
    for hv, hvp in os_hvp[os_name].items():
265
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
266

    
267

    
268
def TestClusterRename():
269
  """gnt-cluster rename"""
270
  cmd = ["gnt-cluster", "rename", "-f"]
271

    
272
  original_name = qa_config.get("name")
273
  rename_target = qa_config.get("rename", None)
274
  if rename_target is None:
275
    print qa_utils.FormatError('"rename" entry is missing')
276
    return
277

    
278
  for data in [
279
    cmd + [rename_target],
280
    _CLUSTER_VERIFY,
281
    cmd + [original_name],
282
    _CLUSTER_VERIFY,
283
    ]:
284
    AssertCommand(data)
285

    
286

    
287
def TestClusterOob():
288
  """out-of-band framework"""
289
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
290

    
291
  AssertCommand(_CLUSTER_VERIFY)
292
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
293
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
294
                 utils.NewUUID()])
295

    
296
  AssertCommand(_CLUSTER_VERIFY, fail=True)
297

    
298
  AssertCommand(["touch", oob_path_exists])
299
  AssertCommand(["chmod", "0400", oob_path_exists])
300
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
301

    
302
  try:
303
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
304
                   "oob_program=%s" % oob_path_exists])
305

    
306
    AssertCommand(_CLUSTER_VERIFY, fail=True)
307

    
308
    AssertCommand(["chmod", "0500", oob_path_exists])
309
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
310

    
311
    AssertCommand(_CLUSTER_VERIFY)
312
  finally:
313
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
314

    
315
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
316
                 "oob_program="])
317

    
318

    
319
def TestClusterEpo():
320
  """gnt-cluster epo"""
321
  master = qa_config.GetMasterNode()
322

    
323
  # Assert that OOB is unavailable for all nodes
324
  result_output = GetCommandOutput(master.primary,
325
                                   "gnt-node list --verbose --no-headers -o"
326
                                   " powered")
327
  AssertEqual(compat.all(powered == "(unavail)"
328
                         for powered in result_output.splitlines()), True)
329

    
330
  # Conflicting
331
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
332
  # --all doesn't expect arguments
333
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
334

    
335
  # Unless --all is given master is not allowed to be in the list
336
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
337

    
338
  # This shouldn't fail
339
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
340

    
341
  # All instances should have been stopped now
342
  result_output = GetCommandOutput(master.primary,
343
                                   "gnt-instance list --no-headers -o status")
344
  # ERROR_down because the instance is stopped but not recorded as such
345
  AssertEqual(compat.all(status == "ERROR_down"
346
                         for status in result_output.splitlines()), True)
347

    
348
  # Now start everything again
349
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
350

    
351
  # All instances should have been started now
352
  result_output = GetCommandOutput(master.primary,
353
                                   "gnt-instance list --no-headers -o status")
354
  AssertEqual(compat.all(status == "running"
355
                         for status in result_output.splitlines()), True)
356

    
357

    
358
def TestClusterVerify():
359
  """gnt-cluster verify"""
360
  AssertCommand(_CLUSTER_VERIFY)
361
  AssertCommand(["gnt-cluster", "verify-disks"])
362

    
363

    
364
def TestClusterVerifyDisksBrokenDRBD(instance, inst_nodes):
365
  """gnt-cluster verify-disks with broken DRBD"""
366
  qa_daemon.TestPauseWatcher()
367

    
368
  try:
369
    info = qa_instance.GetInstanceInfo(instance.name)
370
    snode = inst_nodes[1]
371
    for idx, minor in enumerate(info["drbd-minors"][snode.primary]):
372
      if idx % 2 == 0:
373
        break_drbd_cmd = \
374
          "(drbdsetup %d down >/dev/null 2>&1;" \
375
          " drbdsetup down resource%d >/dev/null 2>&1) || /bin/true" % \
376
          (minor, minor)
377
      else:
378
        break_drbd_cmd = \
379
          "(drbdsetup %d detach >/dev/null 2>&1;" \
380
          " drbdsetup detach %d >/dev/null 2>&1) || /bin/true" % \
381
          (minor, minor)
382
      AssertCommand(break_drbd_cmd, node=snode)
383

    
384
    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
385
                                     "gnt-cluster verify-disks")
386
    activation_msg = "Activating disks for instance '%s'" % instance.name
387
    if activation_msg not in verify_output:
388
      raise qa_error.Error("gnt-cluster verify-disks did not activate broken"
389
                           " DRBD disks:\n%s" % verify_output)
390

    
391
    verify_output = GetCommandOutput(qa_config.GetMasterNode().primary,
392
                                     "gnt-cluster verify-disks")
393
    if activation_msg in verify_output:
394
      raise qa_error.Error("gnt-cluster verify-disks wants to activate broken"
395
                           " DRBD disks on second attempt:\n%s" % verify_output)
396

    
397
    AssertCommand(_CLUSTER_VERIFY)
398
  finally:
399
    qa_daemon.TestResumeWatcher()
400

    
401

    
402
def TestJobqueue():
403
  """gnt-debug test-jobqueue"""
404
  AssertCommand(["gnt-debug", "test-jobqueue"])
405

    
406

    
407
def TestDelay(node):
408
  """gnt-debug delay"""
409
  AssertCommand(["gnt-debug", "delay", "1"])
410
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
411
  AssertCommand(["gnt-debug", "delay", "--no-master",
412
                 "-n", node.primary, "1"])
413

    
414

    
415
def TestClusterReservedLvs():
416
  """gnt-cluster reserved lvs"""
417
  # if no lvm-based templates are supported, skip the test
418
  if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG):
419
    return
420
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
421
  lvname = _QA_LV_PREFIX + "test"
422
  lvfullname = "/".join([vgname, lvname])
423
  for fail, cmd in [
424
    (False, _CLUSTER_VERIFY),
425
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
426
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
427
    (True, _CLUSTER_VERIFY),
428
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
429
             "%s,.*/other-test" % lvfullname]),
430
    (False, _CLUSTER_VERIFY),
431
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
432
             ".*/%s.*" % _QA_LV_PREFIX]),
433
    (False, _CLUSTER_VERIFY),
434
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
435
    (True, _CLUSTER_VERIFY),
436
    (False, ["lvremove", "-f", lvfullname]),
437
    (False, _CLUSTER_VERIFY),
438
    ]:
439
    AssertCommand(cmd, fail=fail)
440

    
441

    
442
def TestClusterModifyEmpty():
443
  """gnt-cluster modify"""
444
  AssertCommand(["gnt-cluster", "modify"], fail=True)
445

    
446

    
447
def TestClusterModifyDisk():
448
  """gnt-cluster modify -D"""
449
  for param in _FAIL_PARAMS:
450
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
451

    
452

    
453
def _GetOtherEnabledDiskTemplate(undesired_disk_templates,
454
                                 enabled_disk_templates):
455
  """Returns one template that is not in the undesired set.
456

457
  @type undesired_disk_templates: list of string
458
  @param undesired_disk_templates: a list of disk templates that we want to
459
      exclude when drawing one disk template from the list of enabled
460
      disk templates
461
  @type enabled_disk_templates: list of string
462
  @param enabled_disk_templates: list of enabled disk templates (in QA)
463

464
  """
465
  desired_templates = list(set(enabled_disk_templates)
466
                                - set(undesired_disk_templates))
467
  if desired_templates:
468
    template = desired_templates[0]
469
  else:
470
    # If no desired disk template is available for QA, choose 'diskless' and
471
    # hope for the best.
472
    template = constants.ST_DISKLESS
473

    
474
  return template
475

    
476

    
477
def TestClusterModifyFileBasedStorageDir(
478
    file_disk_template, dir_config_key, default_dir, option_name):
479
  """Tests gnt-cluster modify wrt to file-based directory options.
480

481
  @type file_disk_template: string
482
  @param file_disk_template: file-based disk template
483
  @type dir_config_key: string
484
  @param dir_config_key: key for the QA config to retrieve the default
485
     directory value
486
  @type default_dir: string
487
  @param default_dir: default directory, if the QA config does not specify
488
     it
489
  @type option_name: string
490
  @param option_name: name of the option of 'gnt-cluster modify' to
491
     change the directory
492

493
  """
494
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
495
  assert file_disk_template in [constants.DT_FILE, constants.DT_SHARED_FILE]
496
  if not qa_config.IsTemplateSupported(file_disk_template):
497
    return
498

    
499
  # Get some non-file-based disk template to disable file storage
500
  other_disk_template = _GetOtherEnabledDiskTemplate(
501
      utils.storage.GetDiskTemplatesOfStorageType(constants.ST_FILE),
502
      enabled_disk_templates)
503

    
504
  file_storage_dir = qa_config.get(dir_config_key, default_dir)
505
  invalid_file_storage_dir = "/boot/"
506

    
507
  for fail, cmd in [
508
    (False, ["gnt-cluster", "modify",
509
             "--enabled-disk-templates=%s" % file_disk_template,
510
             "--ipolicy-disk-templates=%s" % file_disk_template]),
511
    (False, ["gnt-cluster", "modify",
512
             "--%s=%s" % (option_name, file_storage_dir)]),
513
    (False, ["gnt-cluster", "modify",
514
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
515
    # file storage dir is set to an inacceptable path, thus verify
516
    # should fail
517
    (True, ["gnt-cluster", "verify"]),
518
    # unsetting the storage dir while file storage is enabled
519
    # should fail
520
    (True, ["gnt-cluster", "modify",
521
            "--%s=" % option_name]),
522
    (False, ["gnt-cluster", "modify",
523
             "--%s=%s" % (option_name, file_storage_dir)]),
524
    (False, ["gnt-cluster", "modify",
525
             "--enabled-disk-templates=%s" % other_disk_template,
526
             "--ipolicy-disk-templates=%s" % other_disk_template]),
527
    (False, ["gnt-cluster", "modify",
528
             "--%s=%s" % (option_name, invalid_file_storage_dir)]),
529
    # file storage is set to an inacceptable path, but file storage
530
    # is disabled, thus verify should not fail
531
    (False, ["gnt-cluster", "verify"]),
532
    # unsetting the file storage dir while file storage is not enabled
533
    # should be fine
534
    (False, ["gnt-cluster", "modify",
535
             "--%s=" % option_name]),
536
    # resetting everything to sane values
537
    (False, ["gnt-cluster", "modify",
538
             "--%s=%s" % (option_name, file_storage_dir),
539
             "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
540
             "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates)])
541
    ]:
542
    AssertCommand(cmd, fail=fail)
543

    
544

    
545
def TestClusterModifyFileStorageDir():
546
  """gnt-cluster modify --file-storage-dir=..."""
547
  TestClusterModifyFileBasedStorageDir(
548
      constants.DT_FILE, "default-file-storage-dir",
549
      pathutils.DEFAULT_FILE_STORAGE_DIR,
550
      "file-storage-dir")
551

    
552

    
553
def TestClusterModifySharedFileStorageDir():
554
  """gnt-cluster modify --shared-file-storage-dir=..."""
555
  TestClusterModifyFileBasedStorageDir(
556
      constants.DT_SHARED_FILE, "default-shared-file-storage-dir",
557
      pathutils.DEFAULT_SHARED_FILE_STORAGE_DIR,
558
      "shared-file-storage-dir")
559

    
560

    
561
def TestClusterModifyDiskTemplates():
562
  """gnt-cluster modify --enabled-disk-templates=..."""
563
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
564
  default_disk_template = qa_config.GetDefaultDiskTemplate()
565

    
566
  _TestClusterModifyDiskTemplatesArguments(default_disk_template)
567
  _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates)
568
  _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates)
569

    
570
  _RestoreEnabledDiskTemplates()
571
  nodes = qa_config.AcquireManyNodes(2)
572

    
573
  instance_template = enabled_disk_templates[0]
574
  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)
575

    
576
  _TestClusterModifyUnusedDiskTemplate(instance_template)
577
  _TestClusterModifyUsedDiskTemplate(instance_template,
578
                                     enabled_disk_templates)
579

    
580
  qa_instance.TestInstanceRemove(instance)
581
  _RestoreEnabledDiskTemplates()
582

    
583

    
584
def _RestoreEnabledDiskTemplates():
585
  """Sets the list of enabled disk templates back to the list of enabled disk
586
     templates from the QA configuration. This can be used to make sure that
587
     the tests that modify the list of disk templates do not interfere with
588
     other tests.
589

590
  """
591
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
592
  cmd = ["gnt-cluster", "modify",
593
         "--enabled-disk-templates=%s" % ",".join(enabled_disk_templates),
594
         "--ipolicy-disk-templates=%s" % ",".join(enabled_disk_templates),
595
         ]
596

    
597
  if utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
598
    vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
599
    cmd.append("--vg-name=%s" % vgname)
600

    
601
  AssertCommand(cmd, fail=False)
602

    
603

    
604
def _TestClusterModifyDiskTemplatesDrbdHelper(enabled_disk_templates):
605
  """Tests argument handling of 'gnt-cluster modify' with respect to
606
     the parameter '--drbd-usermode-helper'. This test is independent
607
     of instances.
608

609
  """
610
  _RestoreEnabledDiskTemplates()
611

    
612
  if constants.DT_DRBD8 not in enabled_disk_templates:
613
    return
614
  if constants.DT_PLAIN not in enabled_disk_templates:
615
    return
616

    
617
  drbd_usermode_helper = qa_config.get("drbd-usermode-helper", "/bin/true")
618
  bogus_usermode_helper = "/tmp/pinkbunny"
619
  for command, fail in [
620
    (["gnt-cluster", "modify",
621
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
622
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
623
    (["gnt-cluster", "modify",
624
      "--drbd-usermode-helper=%s" % drbd_usermode_helper], False),
625
    (["gnt-cluster", "modify",
626
      "--drbd-usermode-helper=%s" % bogus_usermode_helper], True),
627
    # unsetting helper when DRBD is enabled should not work
628
    (["gnt-cluster", "modify",
629
      "--drbd-usermode-helper="], True),
630
    (["gnt-cluster", "modify",
631
      "--enabled-disk-templates=%s" % constants.DT_PLAIN,
632
      "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False),
633
    (["gnt-cluster", "modify",
634
      "--drbd-usermode-helper="], False),
635
    (["gnt-cluster", "modify",
636
      "--drbd-usermode-helper=%s" % drbd_usermode_helper], False),
637
    (["gnt-cluster", "modify",
638
      "--drbd-usermode-helper=%s" % drbd_usermode_helper,
639
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
640
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
641
    (["gnt-cluster", "modify",
642
      "--drbd-usermode-helper=",
643
      "--enabled-disk-templates=%s" % constants.DT_PLAIN,
644
      "--ipolicy-disk-templates=%s" % constants.DT_PLAIN], False),
645
    (["gnt-cluster", "modify",
646
      "--drbd-usermode-helper=%s" % drbd_usermode_helper,
647
      "--enabled-disk-templates=%s" % constants.DT_DRBD8,
648
      "--ipolicy-disk-templates=%s" % constants.DT_DRBD8], False),
649
    ]:
650
    AssertCommand(command, fail=fail)
651
  _RestoreEnabledDiskTemplates()
652

    
653

    
654
def _TestClusterModifyDiskTemplatesArguments(default_disk_template):
655
  """Tests argument handling of 'gnt-cluster modify' with respect to
656
     the parameter '--enabled-disk-templates'. This test is independent
657
     of instances.
658

659
  """
660
  _RestoreEnabledDiskTemplates()
661

    
662
  # bogus templates
663
  AssertCommand(["gnt-cluster", "modify",
664
                 "--enabled-disk-templates=pinkbunny"],
665
                fail=True)
666

    
667
  # duplicate entries do no harm
668
  AssertCommand(
669
    ["gnt-cluster", "modify",
670
     "--enabled-disk-templates=%s,%s" %
671
      (default_disk_template, default_disk_template),
672
     "--ipolicy-disk-templates=%s" % default_disk_template],
673
    fail=False)
674

    
675

    
676
def _TestClusterModifyDiskTemplatesVgName(enabled_disk_templates):
677
  """Tests argument handling of 'gnt-cluster modify' with respect to
678
     the parameter '--enabled-disk-templates' and '--vg-name'. This test is
679
     independent of instances.
680

681
  """
682
  if not utils.IsLvmEnabled(enabled_disk_templates):
683
    # These tests only make sense if lvm is enabled for QA
684
    return
685

    
686
  # determine an LVM and a non-LVM disk template for the tests
687
  non_lvm_template = _GetOtherEnabledDiskTemplate(constants.DTS_LVM,
688
                                                  enabled_disk_templates)
689
  lvm_template = list(set(enabled_disk_templates) & constants.DTS_LVM)[0]
690

    
691
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
692

    
693
  # Clean start: unset volume group name, disable lvm storage
694
  AssertCommand(
695
    ["gnt-cluster", "modify",
696
     "--enabled-disk-templates=%s" % non_lvm_template,
697
     "--ipolicy-disk-templates=%s" % non_lvm_template,
698
     "--vg-name="],
699
    fail=False)
700

    
701
  # Try to enable lvm, when no volume group is given
702
  AssertCommand(
703
    ["gnt-cluster", "modify",
704
     "--enabled-disk-templates=%s" % lvm_template,
705
     "--ipolicy-disk-templates=%s" % lvm_template],
706
    fail=True)
707

    
708
  # Set volume group, with lvm still disabled: just a warning
709
  AssertCommand(["gnt-cluster", "modify", "--vg-name=%s" % vgname], fail=False)
710

    
711
  # Try unsetting vg name and enabling lvm at the same time
712
  AssertCommand(
713
    ["gnt-cluster", "modify",
714
     "--enabled-disk-templates=%s" % lvm_template,
715
     "--ipolicy-disk-templates=%s" % lvm_template,
716
     "--vg-name="],
717
    fail=True)
718

    
719
  # Enable lvm with vg name present
720
  AssertCommand(
721
    ["gnt-cluster", "modify",
722
     "--enabled-disk-templates=%s" % lvm_template,
723
     "--ipolicy-disk-templates=%s" % lvm_template],
724
    fail=False)
725

    
726
  # Try unsetting vg name with lvm still enabled
727
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=True)
728

    
729
  # Disable lvm with vg name still set
730
  AssertCommand(
731
    ["gnt-cluster", "modify",
732
     "--enabled-disk-templates=%s" % non_lvm_template,
733
     "--ipolicy-disk-templates=%s" % non_lvm_template,
734
     ],
735
    fail=False)
736

    
737
  # Try unsetting vg name with lvm disabled
738
  AssertCommand(["gnt-cluster", "modify", "--vg-name="], fail=False)
739

    
740
  # Set vg name and enable lvm at the same time
741
  AssertCommand(
742
    ["gnt-cluster", "modify",
743
     "--enabled-disk-templates=%s" % lvm_template,
744
     "--ipolicy-disk-templates=%s" % lvm_template,
745
     "--vg-name=%s" % vgname],
746
    fail=False)
747

    
748
  # Unset vg name and disable lvm at the same time
749
  AssertCommand(
750
    ["gnt-cluster", "modify",
751
     "--enabled-disk-templates=%s" % non_lvm_template,
752
     "--ipolicy-disk-templates=%s" % non_lvm_template,
753
     "--vg-name="],
754
    fail=False)
755

    
756
  _RestoreEnabledDiskTemplates()
757

    
758

    
759
def _TestClusterModifyUsedDiskTemplate(instance_template,
760
                                       enabled_disk_templates):
761
  """Tests that disk templates that are currently in use by instances cannot
762
     be disabled on the cluster.
763

764
  """
765
  # If the list of enabled disk templates contains only one template
766
  # we need to add some other templates, because the list of enabled disk
767
  # templates can only be set to a non-empty list.
768
  new_disk_templates = list(set(enabled_disk_templates)
769
                              - set([instance_template]))
770
  if not new_disk_templates:
771
    new_disk_templates = list(set([constants.DT_DISKLESS, constants.DT_BLOCK])
772
                                - set([instance_template]))
773
  AssertCommand(
774
    ["gnt-cluster", "modify",
775
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
776
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
777
    fail=True)
778

    
779

    
780
def _TestClusterModifyUnusedDiskTemplate(instance_template):
781
  """Tests that unused disk templates can be disabled safely."""
782
  all_disk_templates = constants.DISK_TEMPLATES
783
  if not utils.IsLvmEnabled(qa_config.GetEnabledDiskTemplates()):
784
    all_disk_templates = list(set(all_disk_templates) -
785
                              constants.DTS_LVM)
786

    
787
  AssertCommand(
788
    ["gnt-cluster", "modify",
789
     "--enabled-disk-templates=%s" % ",".join(all_disk_templates),
790
     "--ipolicy-disk-templates=%s" % ",".join(all_disk_templates)],
791
    fail=False)
792
  new_disk_templates = [instance_template]
793
  AssertCommand(
794
    ["gnt-cluster", "modify",
795
     "--enabled-disk-templates=%s" % ",".join(new_disk_templates),
796
     "--ipolicy-disk-templates=%s" % ",".join(new_disk_templates)],
797
    fail=False)
798

    
799

    
800
def TestClusterModifyBe():
801
  """gnt-cluster modify -B"""
802
  for fail, cmd in [
803
    # max/min mem
804
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
805
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
806
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
807
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
808
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
809
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
810
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
811
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
812
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
813
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
814
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
815
    # vcpus
816
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
817
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
818
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
819
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
820
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
821
    # auto_balance
822
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
823
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
824
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
825
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
826
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
827
    ]:
828
    AssertCommand(cmd, fail=fail)
829

    
830
  # redo the original-requested BE parameters, if any
831
  bep = qa_config.get("backend-parameters", "")
832
  if bep:
833
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
834

    
835

    
836
def _GetClusterIPolicy():
837
  """Return the run-time values of the cluster-level instance policy.
838

839
  @rtype: tuple
840
  @return: (policy, specs), where:
841
      - policy is a dictionary of the policy values, instance specs excluded
842
      - specs is a dictionary containing only the specs, using the internal
843
        format (see L{constants.IPOLICY_DEFAULTS} for an example)
844

845
  """
846
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
847
  policy = info["Instance policy - limits for instances"]
848
  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
849

    
850
  # Sanity checks
851
  assert "minmax" in ret_specs and "std" in ret_specs
852
  assert len(ret_specs["minmax"]) > 0
853
  assert len(ret_policy) > 0
854
  return (ret_policy, ret_specs)
855

    
856

    
857
def TestClusterModifyIPolicy():
858
  """gnt-cluster modify --ipolicy-*"""
859
  basecmd = ["gnt-cluster", "modify"]
860
  (old_policy, old_specs) = _GetClusterIPolicy()
861
  for par in ["vcpu-ratio", "spindle-ratio"]:
862
    curr_val = float(old_policy[par])
863
    test_values = [
864
      (True, 1.0),
865
      (True, 1.5),
866
      (True, 2),
867
      (False, "a"),
868
      # Restore the old value
869
      (True, curr_val),
870
      ]
871
    for (good, val) in test_values:
872
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
873
      AssertCommand(cmd, fail=not good)
874
      if good:
875
        curr_val = val
876
      # Check the affected parameter
877
      (eff_policy, eff_specs) = _GetClusterIPolicy()
878
      AssertEqual(float(eff_policy[par]), curr_val)
879
      # Check everything else
880
      AssertEqual(eff_specs, old_specs)
881
      for p in eff_policy.keys():
882
        if p == par:
883
          continue
884
        AssertEqual(eff_policy[p], old_policy[p])
885

    
886
  # Allowing disk templates via ipolicy requires them to be
887
  # enabled on the cluster.
888
  if not (qa_config.IsTemplateSupported(constants.DT_PLAIN)
889
          and qa_config.IsTemplateSupported(constants.DT_DRBD8)):
890
    return
891
  # Disk templates are treated slightly differently
892
  par = "disk-templates"
893
  disp_str = "allowed disk templates"
894
  curr_val = old_policy[disp_str]
895
  test_values = [
896
    (True, constants.DT_PLAIN),
897
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
898
    (False, "thisisnotadisktemplate"),
899
    (False, ""),
900
    # Restore the old value
901
    (True, curr_val.replace(" ", "")),
902
    ]
903
  for (good, val) in test_values:
904
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
905
    AssertCommand(cmd, fail=not good)
906
    if good:
907
      curr_val = val
908
    # Check the affected parameter
909
    (eff_policy, eff_specs) = _GetClusterIPolicy()
910
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
911
    # Check everything else
912
    AssertEqual(eff_specs, old_specs)
913
    for p in eff_policy.keys():
914
      if p == disp_str:
915
        continue
916
      AssertEqual(eff_policy[p], old_policy[p])
917

    
918

    
919
def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False,
920
                         old_values=None):
921
  """Change instance specs.
922

923
  At most one of new_specs or diff_specs can be specified.
924

925
  @type new_specs: dict
926
  @param new_specs: new complete specs, in the same format returned by
927
      L{_GetClusterIPolicy}
928
  @type diff_specs: dict
929
  @param diff_specs: partial specs, it can be an incomplete specifications, but
930
      if min/max specs are specified, their number must match the number of the
931
      existing specs
932
  @type fail: bool
933
  @param fail: if the change is expected to fail
934
  @type old_values: tuple
935
  @param old_values: (old_policy, old_specs), as returned by
936
      L{_GetClusterIPolicy}
937
  @return: same as L{_GetClusterIPolicy}
938

939
  """
940
  build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
941
  return qa_utils.TestSetISpecs(
942
    new_specs=new_specs, diff_specs=diff_specs,
943
    get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd,
944
    fail=fail, old_values=old_values)
945

    
946

    
947
def TestClusterModifyISpecs():
948
  """gnt-cluster modify --specs-*"""
949
  params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
950
  (cur_policy, cur_specs) = _GetClusterIPolicy()
951
  # This test assumes that there is only one min/max bound
952
  assert len(cur_specs[constants.ISPECS_MINMAX]) == 1
953
  for par in params:
954
    test_values = [
955
      (True, 0, 4, 12),
956
      (True, 4, 4, 12),
957
      (True, 4, 12, 12),
958
      (True, 4, 4, 4),
959
      (False, 4, 0, 12),
960
      (False, 4, 16, 12),
961
      (False, 4, 4, 0),
962
      (False, 12, 4, 4),
963
      (False, 12, 4, 0),
964
      (False, "a", 4, 12),
965
      (False, 0, "a", 12),
966
      (False, 0, 4, "a"),
967
      # This is to restore the old values
968
      (True,
969
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MIN][par],
970
       cur_specs[constants.ISPECS_STD][par],
971
       cur_specs[constants.ISPECS_MINMAX][0][constants.ISPECS_MAX][par])
972
      ]
973
    for (good, mn, st, mx) in test_values:
974
      new_vals = {
975
        constants.ISPECS_MINMAX: [{
976
          constants.ISPECS_MIN: {par: mn},
977
          constants.ISPECS_MAX: {par: mx}
978
          }],
979
        constants.ISPECS_STD: {par: st}
980
        }
981
      cur_state = (cur_policy, cur_specs)
982
      # We update cur_specs, as we've copied the values to restore already
983
      (cur_policy, cur_specs) = TestClusterSetISpecs(
984
        diff_specs=new_vals, fail=not good, old_values=cur_state)
985

    
986
    # Get the ipolicy command
987
    mnode = qa_config.GetMasterNode()
988
    initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
989
    modcmd = ["gnt-cluster", "modify"]
990
    opts = initcmd.split()
991
    assert opts[0:2] == ["gnt-cluster", "init"]
992
    for k in range(2, len(opts) - 1):
993
      if opts[k].startswith("--ipolicy-"):
994
        assert k + 2 <= len(opts)
995
        modcmd.extend(opts[k:k + 2])
996
    # Re-apply the ipolicy (this should be a no-op)
997
    AssertCommand(modcmd)
998
    new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
999
    AssertEqual(initcmd, new_initcmd)
1000

    
1001

    
1002
def TestClusterInfo():
1003
  """gnt-cluster info"""
1004
  AssertCommand(["gnt-cluster", "info"])
1005

    
1006

    
1007
def TestClusterRedistConf():
1008
  """gnt-cluster redist-conf"""
1009
  AssertCommand(["gnt-cluster", "redist-conf"])
1010

    
1011

    
1012
def TestClusterGetmaster():
1013
  """gnt-cluster getmaster"""
1014
  AssertCommand(["gnt-cluster", "getmaster"])
1015

    
1016

    
1017
def TestClusterVersion():
1018
  """gnt-cluster version"""
1019
  AssertCommand(["gnt-cluster", "version"])
1020

    
1021

    
1022
def TestClusterRenewCrypto():
1023
  """gnt-cluster renew-crypto"""
1024
  master = qa_config.GetMasterNode()
1025

    
1026
  # Conflicting options
1027
  cmd = ["gnt-cluster", "renew-crypto", "--force",
1028
         "--new-cluster-certificate", "--new-confd-hmac-key"]
1029
  conflicting = [
1030
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
1031
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
1032
    ]
1033
  for i in conflicting:
1034
    AssertCommand(cmd + i, fail=True)
1035

    
1036
  # Invalid RAPI certificate
1037
  cmd = ["gnt-cluster", "renew-crypto", "--force",
1038
         "--rapi-certificate=/dev/null"]
1039
  AssertCommand(cmd, fail=True)
1040

    
1041
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
1042
                                         pathutils.RAPI_CERT_FILE)
1043
  try:
1044
    # Custom RAPI certificate
1045
    fh = tempfile.NamedTemporaryFile()
1046

    
1047
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
1048
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
1049

    
1050
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
1051

    
1052
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
1053
    try:
1054
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1055
                     "--rapi-certificate=%s" % tmpcert])
1056
    finally:
1057
      AssertCommand(["rm", "-f", tmpcert])
1058

    
1059
    # Custom cluster domain secret
1060
    cds_fh = tempfile.NamedTemporaryFile()
1061
    cds_fh.write(utils.GenerateSecret())
1062
    cds_fh.write("\n")
1063
    cds_fh.flush()
1064

    
1065
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
1066
    try:
1067
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1068
                     "--cluster-domain-secret=%s" % tmpcds])
1069
    finally:
1070
      AssertCommand(["rm", "-f", tmpcds])
1071

    
1072
    # Normal case
1073
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1074
                   "--new-cluster-certificate", "--new-confd-hmac-key",
1075
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
1076

    
1077
    # Restore RAPI certificate
1078
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
1079
                   "--rapi-certificate=%s" % rapi_cert_backup])
1080
  finally:
1081
    AssertCommand(["rm", "-f", rapi_cert_backup])
1082

    
1083

    
1084
def TestClusterBurnin():
1085
  """Burnin"""
1086
  master = qa_config.GetMasterNode()
1087

    
1088
  options = qa_config.get("options", {})
1089
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
1090
  parallel = options.get("burnin-in-parallel", False)
1091
  check_inst = options.get("burnin-check-instances", False)
1092
  do_rename = options.get("burnin-rename", "")
1093
  do_reboot = options.get("burnin-reboot", True)
1094
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
1095

    
1096
  # Get as many instances as we need
1097
  instances = []
1098
  try:
1099
    try:
1100
      num = qa_config.get("options", {}).get("burnin-instances", 1)
1101
      for _ in range(0, num):
1102
        instances.append(qa_config.AcquireInstance())
1103
    except qa_error.OutOfInstancesError:
1104
      print "Not enough instances, continuing anyway."
1105

    
1106
    if len(instances) < 1:
1107
      raise qa_error.Error("Burnin needs at least one instance")
1108

    
1109
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
1110
    try:
1111
      disks = qa_config.GetDiskOptions()
1112
      # Run burnin
1113
      cmd = ["env",
1114
             "PYTHONPATH=%s" % _constants.VERSIONEDSHAREDIR,
1115
             script,
1116
             "--os=%s" % qa_config.get("os"),
1117
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
1118
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
1119
             "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
1120
             "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
1121
             "--disk-template=%s" % disk_template]
1122
      if parallel:
1123
        cmd.append("--parallel")
1124
        cmd.append("--early-release")
1125
      if check_inst:
1126
        cmd.append("--http-check")
1127
      if do_rename:
1128
        cmd.append("--rename=%s" % do_rename)
1129
      if not do_reboot:
1130
        cmd.append("--no-reboot")
1131
      else:
1132
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
1133
      cmd += [inst.name for inst in instances]
1134
      AssertCommand(cmd)
1135
    finally:
1136
      AssertCommand(["rm", "-f", script])
1137

    
1138
  finally:
1139
    for inst in instances:
1140
      inst.Release()
1141

    
1142

    
1143
def TestClusterMasterFailover():
1144
  """gnt-cluster master-failover"""
1145
  master = qa_config.GetMasterNode()
1146
  failovermaster = qa_config.AcquireNode(exclude=master)
1147

    
1148
  cmd = ["gnt-cluster", "master-failover"]
1149
  try:
1150
    AssertCommand(cmd, node=failovermaster)
1151
    # Back to original master node
1152
    AssertCommand(cmd, node=master)
1153
  finally:
1154
    failovermaster.Release()
1155

    
1156

    
1157
def _NodeQueueDrainFile(node):
1158
  """Returns path to queue drain file for a node.
1159

1160
  """
1161
  return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE)
1162

    
1163

    
1164
def _AssertDrainFile(node, **kwargs):
1165
  """Checks for the queue drain file.
1166

1167
  """
1168
  AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs)
1169

    
1170

    
1171
def TestClusterMasterFailoverWithDrainedQueue():
1172
  """gnt-cluster master-failover with drained queue"""
1173
  master = qa_config.GetMasterNode()
1174
  failovermaster = qa_config.AcquireNode(exclude=master)
1175

    
1176
  # Ensure queue is not drained
1177
  for node in [master, failovermaster]:
1178
    _AssertDrainFile(node, fail=True)
1179

    
1180
  # Drain queue on failover master
1181
  AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)],
1182
                node=failovermaster)
1183

    
1184
  cmd = ["gnt-cluster", "master-failover"]
1185
  try:
1186
    _AssertDrainFile(failovermaster)
1187
    AssertCommand(cmd, node=failovermaster)
1188
    _AssertDrainFile(master, fail=True)
1189
    _AssertDrainFile(failovermaster, fail=True)
1190

    
1191
    # Back to original master node
1192
    AssertCommand(cmd, node=master)
1193
  finally:
1194
    failovermaster.Release()
1195

    
1196
  # Ensure queue is not drained
1197
  for node in [master, failovermaster]:
1198
    _AssertDrainFile(node, fail=True)
1199

    
1200

    
1201
def TestClusterCopyfile():
1202
  """gnt-cluster copyfile"""
1203
  master = qa_config.GetMasterNode()
1204

    
1205
  uniqueid = utils.NewUUID()
1206

    
1207
  # Create temporary file
1208
  f = tempfile.NamedTemporaryFile()
1209
  f.write(uniqueid)
1210
  f.flush()
1211
  f.seek(0)
1212

    
1213
  # Upload file to master node
1214
  testname = qa_utils.UploadFile(master.primary, f.name)
1215
  try:
1216
    # Copy file to all nodes
1217
    AssertCommand(["gnt-cluster", "copyfile", testname])
1218
    _CheckFileOnAllNodes(testname, uniqueid)
1219
  finally:
1220
    _RemoveFileFromAllNodes(testname)
1221

    
1222

    
1223
def TestClusterCommand():
1224
  """gnt-cluster command"""
1225
  uniqueid = utils.NewUUID()
1226
  rfile = "/tmp/gnt%s" % utils.NewUUID()
1227
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
1228
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
1229
                              "%s >%s" % (rcmd, rfile)])
1230

    
1231
  try:
1232
    AssertCommand(cmd)
1233
    _CheckFileOnAllNodes(rfile, uniqueid)
1234
  finally:
1235
    _RemoveFileFromAllNodes(rfile)
1236

    
1237

    
1238
def TestClusterDestroy():
1239
  """gnt-cluster destroy"""
1240
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
1241

    
1242

    
1243
def TestClusterRepairDiskSizes():
1244
  """gnt-cluster repair-disk-sizes"""
1245
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
1246

    
1247

    
1248
def TestSetExclStorCluster(newvalue):
1249
  """Set the exclusive_storage node parameter at the cluster level.
1250

1251
  @type newvalue: bool
1252
  @param newvalue: New value of exclusive_storage
1253
  @rtype: bool
1254
  @return: The old value of exclusive_storage
1255

1256
  """
1257
  es_path = ["Default node parameters", "exclusive_storage"]
1258
  oldvalue = _GetClusterField(es_path)
1259
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
1260
                 "exclusive_storage=%s" % newvalue])
1261
  effvalue = _GetClusterField(es_path)
1262
  if effvalue != newvalue:
1263
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
1264
                         " of %s" % (effvalue, newvalue))
1265
  qa_config.SetExclusiveStorage(newvalue)
1266
  return oldvalue
1267

    
1268

    
1269
def TestExclStorSharedPv(node):
1270
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
1271

1272
  """
1273
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
1274
  lvname1 = _QA_LV_PREFIX + "vol1"
1275
  lvname2 = _QA_LV_PREFIX + "vol2"
1276
  node_name = node.primary
1277
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
1278
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
1279
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
1280
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
1281
                                         constants.CV_ENODEORPHANLV])
1282
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
1283
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
1284
  AssertClusterVerify()