Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ 63e08b25

History | View | Annotate | Download (30.1 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import constants
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34

    
35
import qa_config
36
import qa_utils
37
import qa_error
38
import qa_instance
39

    
40
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
41

    
42

    
43
# Prefix for LVM volumes created by QA code during tests
44
_QA_LV_PREFIX = "qa-"
45

    
46
#: cluster verify command
47
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
48

    
49

    
50
def _RemoveFileFromAllNodes(filename):
51
  """Removes a file from all nodes.
52

53
  """
54
  for node in qa_config.get("nodes"):
55
    AssertCommand(["rm", "-f", filename], node=node)
56

    
57

    
58
def _CheckFileOnAllNodes(filename, content):
59
  """Verifies the content of the given file on all nodes.
60

61
  """
62
  cmd = utils.ShellQuoteArgs(["cat", filename])
63
  for node in qa_config.get("nodes"):
64
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
65

    
66

    
67
def _GetClusterField(field_path):
68
  """Get the value of a cluster field.
69

70
  @type field_path: list of strings
71
  @param field_path: Names of the groups/fields to navigate to get the desired
72
      value, e.g. C{["Default node parameters", "oob_program"]}
73
  @return: The effective value of the field (the actual type depends on the
74
      chosen field)
75

76
  """
77
  assert isinstance(field_path, list)
78
  assert field_path
79
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
80
  for key in field_path:
81
    ret = ret[key]
82
  return ret
83

    
84

    
85
# Cluster-verify errors (date, "ERROR", then error code)
86
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
87

    
88

    
89
def _GetCVErrorCodes(cvout):
90
  errs = set()
91
  warns = set()
92
  for l in cvout.splitlines():
93
    m = _CVERROR_RE.match(l)
94
    if m:
95
      etype = m.group(1)
96
      ecode = m.group(2)
97
      if etype == "ERROR":
98
        errs.add(ecode)
99
      elif etype == "WARNING":
100
        warns.add(ecode)
101
  return (errs, warns)
102

    
103

    
104
def _CheckVerifyErrors(actual, expected, etype):
105
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
106
  if not actual.issuperset(exp_codes):
107
    missing = exp_codes.difference(actual)
108
    raise qa_error.Error("Cluster-verify didn't return these expected"
109
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))
110

    
111

    
112
def AssertClusterVerify(fail=False, errors=None, warnings=None):
113
  """Run cluster-verify and check the result
114

115
  @type fail: bool
116
  @param fail: if cluster-verify is expected to fail instead of succeeding
117
  @type errors: list of tuples
118
  @param errors: List of CV_XXX errors that are expected; if specified, all the
119
      errors listed must appear in cluster-verify output. A non-empty value
120
      implies C{fail=True}.
121
  @type warnings: list of tuples
122
  @param warnings: Same as C{errors} but for warnings.
123

124
  """
125
  cvcmd = "gnt-cluster verify"
126
  mnode = qa_config.GetMasterNode()
127
  if errors or warnings:
128
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
129
                             fail=(fail or errors))
130
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
131
    if errors:
132
      _CheckVerifyErrors(act_errs, errors, "error")
133
    if warnings:
134
      _CheckVerifyErrors(act_warns, warnings, "warning")
135
  else:
136
    AssertCommand(cvcmd, fail=fail, node=mnode)
137

    
138

    
139
# data for testing failures due to bad keys/values for disk parameters
140
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
141
                "drbd:nonexistent=1",
142
                "drbd:resync-rate=invalid",
143
                ]
144

    
145

    
146
def TestClusterInitDisk():
147
  """gnt-cluster init -D"""
148
  name = qa_config.get("name")
149
  for param in _FAIL_PARAMS:
150
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
151

    
152

    
153
def TestClusterInit(rapi_user, rapi_secret):
154
  """gnt-cluster init"""
155
  master = qa_config.GetMasterNode()
156

    
157
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
158
  rapi_dir = os.path.dirname(rapi_users_path)
159

    
160
  # First create the RAPI credentials
161
  fh = tempfile.NamedTemporaryFile()
162
  try:
163
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
164
    fh.flush()
165

    
166
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
167
    try:
168
      AssertCommand(["mkdir", "-p", rapi_dir])
169
      AssertCommand(["mv", tmpru, rapi_users_path])
170
    finally:
171
      AssertCommand(["rm", "-f", tmpru])
172
  finally:
173
    fh.close()
174

    
175
  # Initialize cluster
176
  cmd = [
177
    "gnt-cluster", "init",
178
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
179
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
180
    "--enabled-disk-templates=%s" %
181
      ",".join(qa_config.GetEnabledDiskTemplates())
182
    ]
183

    
184
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
185
                    "nic-count"):
186
    for spec_val in ("min", "max", "std"):
187
      spec = qa_config.get("ispec_%s_%s" %
188
                           (spec_type.replace("-", "_"), spec_val), None)
189
      if spec is not None:
190
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
191

    
192
  if master.secondary:
193
    cmd.append("--secondary-ip=%s" % master.secondary)
194

    
195
  vgname = qa_config.get("vg-name", None)
196
  if vgname:
197
    cmd.append("--vg-name=%s" % vgname)
198

    
199
  master_netdev = qa_config.get("master-netdev", None)
200
  if master_netdev:
201
    cmd.append("--master-netdev=%s" % master_netdev)
202

    
203
  nicparams = qa_config.get("default-nicparams", None)
204
  if nicparams:
205
    cmd.append("--nic-parameters=%s" %
206
               ",".join(utils.FormatKeyValue(nicparams)))
207

    
208
  # Cluster value of the exclusive-storage node parameter
209
  e_s = qa_config.get("exclusive-storage")
210
  if e_s is not None:
211
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
212
  else:
213
    e_s = False
214
  qa_config.SetExclusiveStorage(e_s)
215

    
216
  extra_args = qa_config.get("cluster-init-args")
217
  if extra_args:
218
    cmd.extend(extra_args)
219

    
220
  cmd.append(qa_config.get("name"))
221

    
222
  AssertCommand(cmd)
223

    
224
  cmd = ["gnt-cluster", "modify"]
225

    
226
  # hypervisor parameter modifications
227
  hvp = qa_config.get("hypervisor-parameters", {})
228
  for k, v in hvp.items():
229
    cmd.extend(["-H", "%s:%s" % (k, v)])
230
  # backend parameter modifications
231
  bep = qa_config.get("backend-parameters", "")
232
  if bep:
233
    cmd.extend(["-B", bep])
234

    
235
  if len(cmd) > 2:
236
    AssertCommand(cmd)
237

    
238
  # OS parameters
239
  osp = qa_config.get("os-parameters", {})
240
  for k, v in osp.items():
241
    AssertCommand(["gnt-os", "modify", "-O", v, k])
242

    
243
  # OS hypervisor parameters
244
  os_hvp = qa_config.get("os-hvp", {})
245
  for os_name in os_hvp:
246
    for hv, hvp in os_hvp[os_name].items():
247
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
248

    
249

    
250
def TestClusterRename():
251
  """gnt-cluster rename"""
252
  cmd = ["gnt-cluster", "rename", "-f"]
253

    
254
  original_name = qa_config.get("name")
255
  rename_target = qa_config.get("rename", None)
256
  if rename_target is None:
257
    print qa_utils.FormatError('"rename" entry is missing')
258
    return
259

    
260
  for data in [
261
    cmd + [rename_target],
262
    _CLUSTER_VERIFY,
263
    cmd + [original_name],
264
    _CLUSTER_VERIFY,
265
    ]:
266
    AssertCommand(data)
267

    
268

    
269
def TestClusterOob():
270
  """out-of-band framework"""
271
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
272

    
273
  AssertCommand(_CLUSTER_VERIFY)
274
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
275
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
276
                 utils.NewUUID()])
277

    
278
  AssertCommand(_CLUSTER_VERIFY, fail=True)
279

    
280
  AssertCommand(["touch", oob_path_exists])
281
  AssertCommand(["chmod", "0400", oob_path_exists])
282
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
283

    
284
  try:
285
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
286
                   "oob_program=%s" % oob_path_exists])
287

    
288
    AssertCommand(_CLUSTER_VERIFY, fail=True)
289

    
290
    AssertCommand(["chmod", "0500", oob_path_exists])
291
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
292

    
293
    AssertCommand(_CLUSTER_VERIFY)
294
  finally:
295
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
296

    
297
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
298
                 "oob_program="])
299

    
300

    
301
def TestClusterEpo():
302
  """gnt-cluster epo"""
303
  master = qa_config.GetMasterNode()
304

    
305
  # Assert that OOB is unavailable for all nodes
306
  result_output = GetCommandOutput(master.primary,
307
                                   "gnt-node list --verbose --no-headers -o"
308
                                   " powered")
309
  AssertEqual(compat.all(powered == "(unavail)"
310
                         for powered in result_output.splitlines()), True)
311

    
312
  # Conflicting
313
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
314
  # --all doesn't expect arguments
315
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
316

    
317
  # Unless --all is given master is not allowed to be in the list
318
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
319

    
320
  # This shouldn't fail
321
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
322

    
323
  # All instances should have been stopped now
324
  result_output = GetCommandOutput(master.primary,
325
                                   "gnt-instance list --no-headers -o status")
326
  # ERROR_down because the instance is stopped but not recorded as such
327
  AssertEqual(compat.all(status == "ERROR_down"
328
                         for status in result_output.splitlines()), True)
329

    
330
  # Now start everything again
331
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
332

    
333
  # All instances should have been started now
334
  result_output = GetCommandOutput(master.primary,
335
                                   "gnt-instance list --no-headers -o status")
336
  AssertEqual(compat.all(status == "running"
337
                         for status in result_output.splitlines()), True)
338

    
339

    
340
def TestClusterVerify():
341
  """gnt-cluster verify"""
342
  AssertCommand(_CLUSTER_VERIFY)
343
  AssertCommand(["gnt-cluster", "verify-disks"])
344

    
345

    
346
def TestJobqueue():
347
  """gnt-debug test-jobqueue"""
348
  AssertCommand(["gnt-debug", "test-jobqueue"])
349

    
350

    
351
def TestDelay(node):
352
  """gnt-debug delay"""
353
  AssertCommand(["gnt-debug", "delay", "1"])
354
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
355
  AssertCommand(["gnt-debug", "delay", "--no-master",
356
                 "-n", node.primary, "1"])
357

    
358

    
359
def TestClusterReservedLvs():
360
  """gnt-cluster reserved lvs"""
361
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
362
  lvname = _QA_LV_PREFIX + "test"
363
  lvfullname = "/".join([vgname, lvname])
364
  for fail, cmd in [
365
    (False, _CLUSTER_VERIFY),
366
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
367
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
368
    (True, _CLUSTER_VERIFY),
369
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
370
             "%s,.*/other-test" % lvfullname]),
371
    (False, _CLUSTER_VERIFY),
372
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
373
             ".*/%s.*" % _QA_LV_PREFIX]),
374
    (False, _CLUSTER_VERIFY),
375
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
376
    (True, _CLUSTER_VERIFY),
377
    (False, ["lvremove", "-f", lvfullname]),
378
    (False, _CLUSTER_VERIFY),
379
    ]:
380
    AssertCommand(cmd, fail=fail)
381

    
382

    
383
def TestClusterModifyEmpty():
384
  """gnt-cluster modify"""
385
  AssertCommand(["gnt-cluster", "modify"], fail=True)
386

    
387

    
388
def TestClusterModifyDisk():
389
  """gnt-cluster modify -D"""
390
  for param in _FAIL_PARAMS:
391
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
392

    
393

    
394
def TestClusterModifyDiskTemplates():
395
  """gnt-cluster modify --enabled-disk-templates=..."""
396
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
397
  default_disk_template = qa_config.GetDefaultDiskTemplate()
398

    
399
  _TestClusterModifyDiskTemplatesArguments(default_disk_template,
400
                                           enabled_disk_templates)
401

    
402
  _RestoreEnabledDiskTemplates()
403
  nodes = qa_config.AcquireManyNodes(2)
404

    
405
  instance_template = enabled_disk_templates[0]
406
  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)
407

    
408
  _TestClusterModifyUnusedDiskTemplate(instance_template)
409
  _TestClusterModifyUsedDiskTemplate(instance_template,
410
                                     enabled_disk_templates)
411

    
412
  qa_instance.TestInstanceRemove(instance)
413
  _RestoreEnabledDiskTemplates()
414

    
415

    
416
def _RestoreEnabledDiskTemplates():
417
  """Sets the list of enabled disk templates back to the list of enabled disk
418
     templates from the QA configuration. This can be used to make sure that
419
     the tests that modify the list of disk templates do not interfere with
420
     other tests.
421

422
  """
423
  AssertCommand(
424
    ["gnt-cluster", "modify",
425
     "--enabled-disk-template=%s" %
426
       ",".join(qa_config.GetEnabledDiskTemplates())],
427
    fail=False)
428

    
429

    
430
def _TestClusterModifyDiskTemplatesArguments(default_disk_template,
431
                                             enabled_disk_templates):
432
  """Tests argument handling of 'gnt-cluster modify' with respect to
433
     the parameter '--enabled-disk-templates'. This test is independent
434
     of instances.
435

436
  """
437
  AssertCommand(
438
    ["gnt-cluster", "modify",
439
     "--enabled-disk-template=%s" %
440
       ",".join(enabled_disk_templates)],
441
    fail=False)
442

    
443
  # bogus templates
444
  AssertCommand(["gnt-cluster", "modify",
445
                 "--enabled-disk-templates=pinkbunny"],
446
                fail=True)
447

    
448
  # duplicate entries do no harm
449
  AssertCommand(
450
    ["gnt-cluster", "modify",
451
     "--enabled-disk-templates=%s,%s" %
452
      (default_disk_template, default_disk_template)],
453
    fail=False)
454

    
455

    
456
def _TestClusterModifyUsedDiskTemplate(instance_template,
457
                                       enabled_disk_templates):
458
  """Tests that disk templates that are currently in use by instances cannot
459
     be disabled on the cluster.
460

461
  """
462
  # If the list of enabled disk templates contains only one template
463
  # we need to add some other templates, because the list of enabled disk
464
  # templates can only be set to a non-empty list.
465
  new_disk_templates = list(set(enabled_disk_templates)
466
                              - set([instance_template]))
467
  if not new_disk_templates:
468
    new_disk_templates = list(set(constants.DISK_TEMPLATES)
469
                                - set([instance_template]))
470
  AssertCommand(
471
    ["gnt-cluster", "modify",
472
     "--enabled-disk-templates=%s" %
473
       ",".join(new_disk_templates)],
474
    fail=True)
475

    
476

    
477
def _TestClusterModifyUnusedDiskTemplate(instance_template):
478
  """Tests that unused disk templates can be disabled safely."""
479
  all_disk_templates = constants.DISK_TEMPLATES
480
  AssertCommand(
481
    ["gnt-cluster", "modify",
482
     "--enabled-disk-templates=%s" %
483
       ",".join(all_disk_templates)],
484
    fail=False)
485
  new_disk_templates = [instance_template]
486
  AssertCommand(
487
    ["gnt-cluster", "modify",
488
     "--enabled-disk-templates=%s" %
489
       ",".join(new_disk_templates)],
490
    fail=False)
491

    
492

    
493
def TestClusterModifyBe():
494
  """gnt-cluster modify -B"""
495
  for fail, cmd in [
496
    # max/min mem
497
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
498
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
499
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
500
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
501
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
502
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
503
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
504
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
505
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
506
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
507
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
508
    # vcpus
509
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
510
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
511
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
512
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
513
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
514
    # auto_balance
515
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
516
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
517
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
518
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
519
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
520
    ]:
521
    AssertCommand(cmd, fail=fail)
522

    
523
  # redo the original-requested BE parameters, if any
524
  bep = qa_config.get("backend-parameters", "")
525
  if bep:
526
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
527

    
528

    
529
def _GetClusterIPolicy():
530
  """Return the run-time values of the cluster-level instance policy.
531

532
  @rtype: tuple
533
  @return: (policy, specs), where:
534
      - policy is a dictionary of the policy values, instance specs excluded
535
      - specs is dict of dict, specs[par][key] is a spec value, where key is
536
        "min", "max", or "std"
537

538
  """
539
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
540
  policy = info["Instance policy - limits for instances"]
541
  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
542

    
543
  # Sanity checks
544
  assert len(ret_specs) > 0
545
  good = all("min" in d and "std" in d and "max" in d
546
             for d in ret_specs.values())
547
  assert good, "Missing item in specs: %s" % ret_specs
548
  assert len(ret_policy) > 0
549
  return (ret_policy, ret_specs)
550

    
551

    
552
def TestClusterModifyIPolicy():
553
  """gnt-cluster modify --ipolicy-*"""
554
  basecmd = ["gnt-cluster", "modify"]
555
  (old_policy, old_specs) = _GetClusterIPolicy()
556
  for par in ["vcpu-ratio", "spindle-ratio"]:
557
    curr_val = float(old_policy[par])
558
    test_values = [
559
      (True, 1.0),
560
      (True, 1.5),
561
      (True, 2),
562
      (False, "a"),
563
      # Restore the old value
564
      (True, curr_val),
565
      ]
566
    for (good, val) in test_values:
567
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
568
      AssertCommand(cmd, fail=not good)
569
      if good:
570
        curr_val = val
571
      # Check the affected parameter
572
      (eff_policy, eff_specs) = _GetClusterIPolicy()
573
      AssertEqual(float(eff_policy[par]), curr_val)
574
      # Check everything else
575
      AssertEqual(eff_specs, old_specs)
576
      for p in eff_policy.keys():
577
        if p == par:
578
          continue
579
        AssertEqual(eff_policy[p], old_policy[p])
580

    
581
  # Disk templates are treated slightly differently
582
  par = "disk-templates"
583
  disp_str = "enabled disk templates"
584
  curr_val = old_policy[disp_str]
585
  test_values = [
586
    (True, constants.DT_PLAIN),
587
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
588
    (False, "thisisnotadisktemplate"),
589
    (False, ""),
590
    # Restore the old value
591
    (True, curr_val.replace(" ", "")),
592
    ]
593
  for (good, val) in test_values:
594
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
595
    AssertCommand(cmd, fail=not good)
596
    if good:
597
      curr_val = val
598
    # Check the affected parameter
599
    (eff_policy, eff_specs) = _GetClusterIPolicy()
600
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
601
    # Check everything else
602
    AssertEqual(eff_specs, old_specs)
603
    for p in eff_policy.keys():
604
      if p == disp_str:
605
        continue
606
      AssertEqual(eff_policy[p], old_policy[p])
607

    
608

    
609
def TestClusterSetISpecs(new_specs, fail=False, old_values=None):
610
  """Change instance specs.
611

612
  @type new_specs: dict of dict
613
  @param new_specs: new_specs[par][key], where key is "min", "max", "std". It
614
      can be an empty dictionary.
615
  @type fail: bool
616
  @param fail: if the change is expected to fail
617
  @type old_values: tuple
618
  @param old_values: (old_policy, old_specs), as returned by
619
     L{_GetClusterIPolicy}
620
  @return: same as L{_GetClusterIPolicy}
621

622
  """
623
  build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
624
  return qa_utils.TestSetISpecs(new_specs, get_policy_fn=_GetClusterIPolicy,
625
                                build_cmd_fn=build_cmd, fail=fail,
626
                                old_values=old_values)
627

    
628

    
629
def TestClusterModifyISpecs():
630
  """gnt-cluster modify --specs-*"""
631
  params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
632
  (cur_policy, cur_specs) = _GetClusterIPolicy()
633
  for par in params:
634
    test_values = [
635
      (True, 0, 4, 12),
636
      (True, 4, 4, 12),
637
      (True, 4, 12, 12),
638
      (True, 4, 4, 4),
639
      (False, 4, 0, 12),
640
      (False, 4, 16, 12),
641
      (False, 4, 4, 0),
642
      (False, 12, 4, 4),
643
      (False, 12, 4, 0),
644
      (False, "a", 4, 12),
645
      (False, 0, "a", 12),
646
      (False, 0, 4, "a"),
647
      # This is to restore the old values
648
      (True,
649
       cur_specs[par]["min"], cur_specs[par]["std"], cur_specs[par]["max"])
650
      ]
651
    for (good, mn, st, mx) in test_values:
652
      new_vals = {par: {"min": str(mn), "std": str(st), "max": str(mx)}}
653
      cur_state = (cur_policy, cur_specs)
654
      # We update cur_specs, as we've copied the values to restore already
655
      (cur_policy, cur_specs) = TestClusterSetISpecs(new_vals, fail=not good,
656
                                                     old_values=cur_state)
657

    
658
    # Get the ipolicy command
659
    mnode = qa_config.GetMasterNode()
660
    initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
661
    modcmd = ["gnt-cluster", "modify"]
662
    opts = initcmd.split()
663
    assert opts[0:2] == ["gnt-cluster", "init"]
664
    for k in range(2, len(opts) - 1):
665
      if opts[k].startswith("--ipolicy-"):
666
        assert k + 2 <= len(opts)
667
        modcmd.extend(opts[k:k + 2])
668
    # Re-apply the ipolicy (this should be a no-op)
669
    AssertCommand(modcmd)
670
    new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
671
    AssertEqual(initcmd, new_initcmd)
672

    
673

    
674
def TestClusterInfo():
675
  """gnt-cluster info"""
676
  AssertCommand(["gnt-cluster", "info"])
677

    
678

    
679
def TestClusterRedistConf():
680
  """gnt-cluster redist-conf"""
681
  AssertCommand(["gnt-cluster", "redist-conf"])
682

    
683

    
684
def TestClusterGetmaster():
685
  """gnt-cluster getmaster"""
686
  AssertCommand(["gnt-cluster", "getmaster"])
687

    
688

    
689
def TestClusterVersion():
690
  """gnt-cluster version"""
691
  AssertCommand(["gnt-cluster", "version"])
692

    
693

    
694
def TestClusterRenewCrypto():
695
  """gnt-cluster renew-crypto"""
696
  master = qa_config.GetMasterNode()
697

    
698
  # Conflicting options
699
  cmd = ["gnt-cluster", "renew-crypto", "--force",
700
         "--new-cluster-certificate", "--new-confd-hmac-key"]
701
  conflicting = [
702
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
703
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
704
    ]
705
  for i in conflicting:
706
    AssertCommand(cmd + i, fail=True)
707

    
708
  # Invalid RAPI certificate
709
  cmd = ["gnt-cluster", "renew-crypto", "--force",
710
         "--rapi-certificate=/dev/null"]
711
  AssertCommand(cmd, fail=True)
712

    
713
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
714
                                         pathutils.RAPI_CERT_FILE)
715
  try:
716
    # Custom RAPI certificate
717
    fh = tempfile.NamedTemporaryFile()
718

    
719
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
720
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
721

    
722
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
723

    
724
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
725
    try:
726
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
727
                     "--rapi-certificate=%s" % tmpcert])
728
    finally:
729
      AssertCommand(["rm", "-f", tmpcert])
730

    
731
    # Custom cluster domain secret
732
    cds_fh = tempfile.NamedTemporaryFile()
733
    cds_fh.write(utils.GenerateSecret())
734
    cds_fh.write("\n")
735
    cds_fh.flush()
736

    
737
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
738
    try:
739
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
740
                     "--cluster-domain-secret=%s" % tmpcds])
741
    finally:
742
      AssertCommand(["rm", "-f", tmpcds])
743

    
744
    # Normal case
745
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
746
                   "--new-cluster-certificate", "--new-confd-hmac-key",
747
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
748

    
749
    # Restore RAPI certificate
750
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
751
                   "--rapi-certificate=%s" % rapi_cert_backup])
752
  finally:
753
    AssertCommand(["rm", "-f", rapi_cert_backup])
754

    
755

    
756
def TestClusterBurnin():
757
  """Burnin"""
758
  master = qa_config.GetMasterNode()
759

    
760
  options = qa_config.get("options", {})
761
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
762
  parallel = options.get("burnin-in-parallel", False)
763
  check_inst = options.get("burnin-check-instances", False)
764
  do_rename = options.get("burnin-rename", "")
765
  do_reboot = options.get("burnin-reboot", True)
766
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
767

    
768
  # Get as many instances as we need
769
  instances = []
770
  try:
771
    try:
772
      num = qa_config.get("options", {}).get("burnin-instances", 1)
773
      for _ in range(0, num):
774
        instances.append(qa_config.AcquireInstance())
775
    except qa_error.OutOfInstancesError:
776
      print "Not enough instances, continuing anyway."
777

    
778
    if len(instances) < 1:
779
      raise qa_error.Error("Burnin needs at least one instance")
780

    
781
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
782
    try:
783
      disks = qa_config.GetDiskOptions()
784
      # Run burnin
785
      cmd = [script,
786
             "--os=%s" % qa_config.get("os"),
787
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
788
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
789
             "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
790
             "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
791
             "--disk-template=%s" % disk_template]
792
      if parallel:
793
        cmd.append("--parallel")
794
        cmd.append("--early-release")
795
      if check_inst:
796
        cmd.append("--http-check")
797
      if do_rename:
798
        cmd.append("--rename=%s" % do_rename)
799
      if not do_reboot:
800
        cmd.append("--no-reboot")
801
      else:
802
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
803
      cmd += [inst.name for inst in instances]
804
      AssertCommand(cmd)
805
    finally:
806
      AssertCommand(["rm", "-f", script])
807

    
808
  finally:
809
    for inst in instances:
810
      inst.Release()
811

    
812

    
813
def TestClusterMasterFailover():
814
  """gnt-cluster master-failover"""
815
  master = qa_config.GetMasterNode()
816
  failovermaster = qa_config.AcquireNode(exclude=master)
817

    
818
  cmd = ["gnt-cluster", "master-failover"]
819
  try:
820
    AssertCommand(cmd, node=failovermaster)
821
    # Back to original master node
822
    AssertCommand(cmd, node=master)
823
  finally:
824
    failovermaster.Release()
825

    
826

    
827
def _NodeQueueDrainFile(node):
828
  """Returns path to queue drain file for a node.
829

830
  """
831
  return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE)
832

    
833

    
834
def _AssertDrainFile(node, **kwargs):
835
  """Checks for the queue drain file.
836

837
  """
838
  AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs)
839

    
840

    
841
def TestClusterMasterFailoverWithDrainedQueue():
842
  """gnt-cluster master-failover with drained queue"""
843
  master = qa_config.GetMasterNode()
844
  failovermaster = qa_config.AcquireNode(exclude=master)
845

    
846
  # Ensure queue is not drained
847
  for node in [master, failovermaster]:
848
    _AssertDrainFile(node, fail=True)
849

    
850
  # Drain queue on failover master
851
  AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)],
852
                node=failovermaster)
853

    
854
  cmd = ["gnt-cluster", "master-failover"]
855
  try:
856
    _AssertDrainFile(failovermaster)
857
    AssertCommand(cmd, node=failovermaster)
858
    _AssertDrainFile(master, fail=True)
859
    _AssertDrainFile(failovermaster, fail=True)
860

    
861
    # Back to original master node
862
    AssertCommand(cmd, node=master)
863
  finally:
864
    failovermaster.Release()
865

    
866
  # Ensure queue is not drained
867
  for node in [master, failovermaster]:
868
    _AssertDrainFile(node, fail=True)
869

    
870

    
871
def TestClusterCopyfile():
872
  """gnt-cluster copyfile"""
873
  master = qa_config.GetMasterNode()
874

    
875
  uniqueid = utils.NewUUID()
876

    
877
  # Create temporary file
878
  f = tempfile.NamedTemporaryFile()
879
  f.write(uniqueid)
880
  f.flush()
881
  f.seek(0)
882

    
883
  # Upload file to master node
884
  testname = qa_utils.UploadFile(master.primary, f.name)
885
  try:
886
    # Copy file to all nodes
887
    AssertCommand(["gnt-cluster", "copyfile", testname])
888
    _CheckFileOnAllNodes(testname, uniqueid)
889
  finally:
890
    _RemoveFileFromAllNodes(testname)
891

    
892

    
893
def TestClusterCommand():
894
  """gnt-cluster command"""
895
  uniqueid = utils.NewUUID()
896
  rfile = "/tmp/gnt%s" % utils.NewUUID()
897
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
898
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
899
                              "%s >%s" % (rcmd, rfile)])
900

    
901
  try:
902
    AssertCommand(cmd)
903
    _CheckFileOnAllNodes(rfile, uniqueid)
904
  finally:
905
    _RemoveFileFromAllNodes(rfile)
906

    
907

    
908
def TestClusterDestroy():
909
  """gnt-cluster destroy"""
910
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
911

    
912

    
913
def TestClusterRepairDiskSizes():
914
  """gnt-cluster repair-disk-sizes"""
915
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
916

    
917

    
918
def TestSetExclStorCluster(newvalue):
919
  """Set the exclusive_storage node parameter at the cluster level.
920

921
  @type newvalue: bool
922
  @param newvalue: New value of exclusive_storage
923
  @rtype: bool
924
  @return: The old value of exclusive_storage
925

926
  """
927
  es_path = ["Default node parameters", "exclusive_storage"]
928
  oldvalue = _GetClusterField(es_path)
929
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
930
                 "exclusive_storage=%s" % newvalue])
931
  effvalue = _GetClusterField(es_path)
932
  if effvalue != newvalue:
933
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
934
                         " of %s" % (effvalue, newvalue))
935
  qa_config.SetExclusiveStorage(newvalue)
936
  return oldvalue
937

    
938

    
939
def TestExclStorSharedPv(node):
940
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
941

942
  """
943
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
944
  lvname1 = _QA_LV_PREFIX + "vol1"
945
  lvname2 = _QA_LV_PREFIX + "vol2"
946
  node_name = node.primary
947
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
948
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
949
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
950
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
951
                                         constants.CV_ENODEORPHANLV])
952
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
953
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
954
  AssertClusterVerify()