Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ 2dae8d64

History | View | Annotate | Download (28 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import constants
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34

    
35
import qa_config
36
import qa_utils
37
import qa_error
38

    
39
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
40

    
41

    
42
# Prefix for LVM volumes created by QA code during tests
43
_QA_LV_PREFIX = "qa-"
44

    
45
#: cluster verify command
46
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
47

    
48

    
49
def _RemoveFileFromAllNodes(filename):
50
  """Removes a file from all nodes.
51

52
  """
53
  for node in qa_config.get("nodes"):
54
    AssertCommand(["rm", "-f", filename], node=node)
55

    
56

    
57
def _CheckFileOnAllNodes(filename, content):
58
  """Verifies the content of the given file on all nodes.
59

60
  """
61
  cmd = utils.ShellQuoteArgs(["cat", filename])
62
  for node in qa_config.get("nodes"):
63
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
64

    
65

    
66
def _GetClusterField(field_path):
67
  """Get the value of a cluster field.
68

69
  @type field_path: list of strings
70
  @param field_path: Names of the groups/fields to navigate to get the desired
71
      value, e.g. C{["Default node parameters", "oob_program"]}
72
  @return: The effective value of the field (the actual type depends on the
73
      chosen field)
74

75
  """
76
  assert isinstance(field_path, list)
77
  assert field_path
78
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
79
  for key in field_path:
80
    ret = ret[key]
81
  return ret
82

    
83

    
84
# Cluster-verify errors (date, "ERROR", then error code)
85
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
86

    
87

    
88
def _GetCVErrorCodes(cvout):
89
  errs = set()
90
  warns = set()
91
  for l in cvout.splitlines():
92
    m = _CVERROR_RE.match(l)
93
    if m:
94
      etype = m.group(1)
95
      ecode = m.group(2)
96
      if etype == "ERROR":
97
        errs.add(ecode)
98
      elif etype == "WARNING":
99
        warns.add(ecode)
100
  return (errs, warns)
101

    
102

    
103
def _CheckVerifyErrors(actual, expected, etype):
104
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
105
  if not actual.issuperset(exp_codes):
106
    missing = exp_codes.difference(actual)
107
    raise qa_error.Error("Cluster-verify didn't return these expected"
108
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))
109

    
110

    
111
def AssertClusterVerify(fail=False, errors=None, warnings=None):
112
  """Run cluster-verify and check the result
113

114
  @type fail: bool
115
  @param fail: if cluster-verify is expected to fail instead of succeeding
116
  @type errors: list of tuples
117
  @param errors: List of CV_XXX errors that are expected; if specified, all the
118
      errors listed must appear in cluster-verify output. A non-empty value
119
      implies C{fail=True}.
120
  @type warnings: list of tuples
121
  @param warnings: Same as C{errors} but for warnings.
122

123
  """
124
  cvcmd = "gnt-cluster verify"
125
  mnode = qa_config.GetMasterNode()
126
  if errors or warnings:
127
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
128
                             fail=(fail or errors))
129
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
130
    if errors:
131
      _CheckVerifyErrors(act_errs, errors, "error")
132
    if warnings:
133
      _CheckVerifyErrors(act_warns, warnings, "warning")
134
  else:
135
    AssertCommand(cvcmd, fail=fail, node=mnode)
136

    
137

    
138
# data for testing failures due to bad keys/values for disk parameters
139
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
140
                "drbd:nonexistent=1",
141
                "drbd:resync-rate=invalid",
142
                ]
143

    
144

    
145
def TestClusterInitDisk():
146
  """gnt-cluster init -D"""
147
  name = qa_config.get("name")
148
  for param in _FAIL_PARAMS:
149
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
150

    
151

    
152
def TestClusterInit(rapi_user, rapi_secret):
153
  """gnt-cluster init"""
154
  master = qa_config.GetMasterNode()
155

    
156
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
157
  rapi_dir = os.path.dirname(rapi_users_path)
158

    
159
  # First create the RAPI credentials
160
  fh = tempfile.NamedTemporaryFile()
161
  try:
162
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
163
    fh.flush()
164

    
165
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
166
    try:
167
      AssertCommand(["mkdir", "-p", rapi_dir])
168
      AssertCommand(["mv", tmpru, rapi_users_path])
169
    finally:
170
      AssertCommand(["rm", "-f", tmpru])
171
  finally:
172
    fh.close()
173

    
174
  # Initialize cluster
175
  cmd = [
176
    "gnt-cluster", "init",
177
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
178
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
179
    "--enabled-disk-templates=%s" %
180
      ",".join(qa_config.GetEnabledDiskTemplates())
181
    ]
182

    
183
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
184
                    "nic-count"):
185
    for spec_val in ("min", "max", "std"):
186
      spec = qa_config.get("ispec_%s_%s" %
187
                           (spec_type.replace("-", "_"), spec_val), None)
188
      if spec is not None:
189
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
190

    
191
  if master.secondary:
192
    cmd.append("--secondary-ip=%s" % master.secondary)
193

    
194
  vgname = qa_config.get("vg-name", None)
195
  if vgname:
196
    cmd.append("--vg-name=%s" % vgname)
197

    
198
  master_netdev = qa_config.get("master-netdev", None)
199
  if master_netdev:
200
    cmd.append("--master-netdev=%s" % master_netdev)
201

    
202
  nicparams = qa_config.get("default-nicparams", None)
203
  if nicparams:
204
    cmd.append("--nic-parameters=%s" %
205
               ",".join(utils.FormatKeyValue(nicparams)))
206

    
207
  # Cluster value of the exclusive-storage node parameter
208
  e_s = qa_config.get("exclusive-storage")
209
  if e_s is not None:
210
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
211
  else:
212
    e_s = False
213
  qa_config.SetExclusiveStorage(e_s)
214

    
215
  extra_args = qa_config.get("cluster-init-args")
216
  if extra_args:
217
    cmd.extend(extra_args)
218

    
219
  cmd.append(qa_config.get("name"))
220

    
221
  AssertCommand(cmd)
222

    
223
  cmd = ["gnt-cluster", "modify"]
224

    
225
  # hypervisor parameter modifications
226
  hvp = qa_config.get("hypervisor-parameters", {})
227
  for k, v in hvp.items():
228
    cmd.extend(["-H", "%s:%s" % (k, v)])
229
  # backend parameter modifications
230
  bep = qa_config.get("backend-parameters", "")
231
  if bep:
232
    cmd.extend(["-B", bep])
233

    
234
  if len(cmd) > 2:
235
    AssertCommand(cmd)
236

    
237
  # OS parameters
238
  osp = qa_config.get("os-parameters", {})
239
  for k, v in osp.items():
240
    AssertCommand(["gnt-os", "modify", "-O", v, k])
241

    
242
  # OS hypervisor parameters
243
  os_hvp = qa_config.get("os-hvp", {})
244
  for os_name in os_hvp:
245
    for hv, hvp in os_hvp[os_name].items():
246
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
247

    
248

    
249
def TestClusterRename():
250
  """gnt-cluster rename"""
251
  cmd = ["gnt-cluster", "rename", "-f"]
252

    
253
  original_name = qa_config.get("name")
254
  rename_target = qa_config.get("rename", None)
255
  if rename_target is None:
256
    print qa_utils.FormatError('"rename" entry is missing')
257
    return
258

    
259
  for data in [
260
    cmd + [rename_target],
261
    _CLUSTER_VERIFY,
262
    cmd + [original_name],
263
    _CLUSTER_VERIFY,
264
    ]:
265
    AssertCommand(data)
266

    
267

    
268
def TestClusterOob():
269
  """out-of-band framework"""
270
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
271

    
272
  AssertCommand(_CLUSTER_VERIFY)
273
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
274
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
275
                 utils.NewUUID()])
276

    
277
  AssertCommand(_CLUSTER_VERIFY, fail=True)
278

    
279
  AssertCommand(["touch", oob_path_exists])
280
  AssertCommand(["chmod", "0400", oob_path_exists])
281
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
282

    
283
  try:
284
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
285
                   "oob_program=%s" % oob_path_exists])
286

    
287
    AssertCommand(_CLUSTER_VERIFY, fail=True)
288

    
289
    AssertCommand(["chmod", "0500", oob_path_exists])
290
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
291

    
292
    AssertCommand(_CLUSTER_VERIFY)
293
  finally:
294
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
295

    
296
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
297
                 "oob_program="])
298

    
299

    
300
def TestClusterEpo():
301
  """gnt-cluster epo"""
302
  master = qa_config.GetMasterNode()
303

    
304
  # Assert that OOB is unavailable for all nodes
305
  result_output = GetCommandOutput(master.primary,
306
                                   "gnt-node list --verbose --no-headers -o"
307
                                   " powered")
308
  AssertEqual(compat.all(powered == "(unavail)"
309
                         for powered in result_output.splitlines()), True)
310

    
311
  # Conflicting
312
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
313
  # --all doesn't expect arguments
314
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
315

    
316
  # Unless --all is given master is not allowed to be in the list
317
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
318

    
319
  # This shouldn't fail
320
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
321

    
322
  # All instances should have been stopped now
323
  result_output = GetCommandOutput(master.primary,
324
                                   "gnt-instance list --no-headers -o status")
325
  # ERROR_down because the instance is stopped but not recorded as such
326
  AssertEqual(compat.all(status == "ERROR_down"
327
                         for status in result_output.splitlines()), True)
328

    
329
  # Now start everything again
330
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
331

    
332
  # All instances should have been started now
333
  result_output = GetCommandOutput(master.primary,
334
                                   "gnt-instance list --no-headers -o status")
335
  AssertEqual(compat.all(status == "running"
336
                         for status in result_output.splitlines()), True)
337

    
338

    
339
def TestClusterVerify():
340
  """gnt-cluster verify"""
341
  AssertCommand(_CLUSTER_VERIFY)
342
  AssertCommand(["gnt-cluster", "verify-disks"])
343

    
344

    
345
def TestJobqueue():
346
  """gnt-debug test-jobqueue"""
347
  AssertCommand(["gnt-debug", "test-jobqueue"])
348

    
349

    
350
def TestDelay(node):
351
  """gnt-debug delay"""
352
  AssertCommand(["gnt-debug", "delay", "1"])
353
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
354
  AssertCommand(["gnt-debug", "delay", "--no-master",
355
                 "-n", node.primary, "1"])
356

    
357

    
358
def TestClusterReservedLvs():
359
  """gnt-cluster reserved lvs"""
360
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
361
  lvname = _QA_LV_PREFIX + "test"
362
  lvfullname = "/".join([vgname, lvname])
363
  for fail, cmd in [
364
    (False, _CLUSTER_VERIFY),
365
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
366
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
367
    (True, _CLUSTER_VERIFY),
368
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
369
             "%s,.*/other-test" % lvfullname]),
370
    (False, _CLUSTER_VERIFY),
371
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
372
             ".*/%s.*" % _QA_LV_PREFIX]),
373
    (False, _CLUSTER_VERIFY),
374
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
375
    (True, _CLUSTER_VERIFY),
376
    (False, ["lvremove", "-f", lvfullname]),
377
    (False, _CLUSTER_VERIFY),
378
    ]:
379
    AssertCommand(cmd, fail=fail)
380

    
381

    
382
def TestClusterModifyEmpty():
383
  """gnt-cluster modify"""
384
  AssertCommand(["gnt-cluster", "modify"], fail=True)
385

    
386

    
387
def TestClusterModifyDisk():
388
  """gnt-cluster modify -D"""
389
  for param in _FAIL_PARAMS:
390
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
391

    
392

    
393
def TestClusterModifyDiskTemplates():
394
  """gnt-cluster modify --enabled-disk-templates=..."""
395
  default_disk_template = qa_config.GetDefaultDiskTemplate()
396
  AssertCommand(
397
    ["gnt-cluster", "modify",
398
     "--enabled-disk-templates=%s" % default_disk_template],
399
    fail=False)
400
  AssertCommand(["gnt-cluster", "info"])
401
  AssertCommand(
402
    ["gnt-cluster", "modify",
403
     "--enabled-disk-template=%s" %
404
       ",".join(qa_config.GetEnabledDiskTemplates())],
405
    fail=False)
406
  AssertCommand(["gnt-cluster", "info"])
407
  # bogus templates
408
  AssertCommand(["gnt-cluster", "modify",
409
                 "--enabled-disk-templates=pinkbunny"],
410
                fail=True)
411
  # duplicate entries do no harm
412
  AssertCommand(
413
    ["gnt-cluster", "modify",
414
     "--enabled-disk-templates=%s,%s" %
415
      (default_disk_template, default_disk_template)],
416
    fail=False)
417
  AssertCommand(["gnt-cluster", "info"])
418

    
419

    
420
def TestClusterModifyBe():
421
  """gnt-cluster modify -B"""
422
  for fail, cmd in [
423
    # max/min mem
424
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
425
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
426
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
427
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
428
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
429
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
430
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
431
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
432
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
433
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
434
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
435
    # vcpus
436
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
437
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
438
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
439
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
440
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
441
    # auto_balance
442
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
443
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
444
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
445
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
446
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
447
    ]:
448
    AssertCommand(cmd, fail=fail)
449

    
450
  # redo the original-requested BE parameters, if any
451
  bep = qa_config.get("backend-parameters", "")
452
  if bep:
453
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
454

    
455

    
456
def _GetClusterIPolicy():
457
  """Return the run-time values of the cluster-level instance policy.
458

459
  @rtype: tuple
460
  @return: (policy, specs), where:
461
      - policy is a dictionary of the policy values, instance specs excluded
462
      - specs is dict of dict, specs[par][key] is a spec value, where key is
463
        "min", "max", or "std"
464

465
  """
466
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
467
  policy = info["Instance policy - limits for instances"]
468
  ret_specs = {}
469
  ret_policy = {}
470
  ispec_keys = constants.ISPECS_MINMAX_KEYS | frozenset([constants.ISPECS_STD])
471
  for (key, val) in policy.items():
472
    if key in ispec_keys:
473
      for (par, pval) in val.items():
474
        if par == "memory-size":
475
          par = "mem-size"
476
        d = ret_specs.setdefault(par, {})
477
        d[key] = pval
478
    else:
479
      ret_policy[key] = val
480

    
481
  # Sanity checks
482
  assert len(ret_specs) > 0
483
  good = all("min" in d and "std" in d and "max" in d
484
             for d in ret_specs.values())
485
  assert good, "Missing item in specs: %s" % ret_specs
486
  assert len(ret_policy) > 0
487
  return (ret_policy, ret_specs)
488

    
489

    
490
def TestClusterModifyIPolicy():
491
  """gnt-cluster modify --ipolicy-*"""
492
  basecmd = ["gnt-cluster", "modify"]
493
  (old_policy, old_specs) = _GetClusterIPolicy()
494
  for par in ["vcpu-ratio", "spindle-ratio"]:
495
    curr_val = float(old_policy[par])
496
    test_values = [
497
      (True, 1.0),
498
      (True, 1.5),
499
      (True, 2),
500
      (False, "a"),
501
      # Restore the old value
502
      (True, curr_val),
503
      ]
504
    for (good, val) in test_values:
505
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
506
      AssertCommand(cmd, fail=not good)
507
      if good:
508
        curr_val = val
509
      # Check the affected parameter
510
      (eff_policy, eff_specs) = _GetClusterIPolicy()
511
      AssertEqual(float(eff_policy[par]), curr_val)
512
      # Check everything else
513
      AssertEqual(eff_specs, old_specs)
514
      for p in eff_policy.keys():
515
        if p == par:
516
          continue
517
        AssertEqual(eff_policy[p], old_policy[p])
518

    
519
  # Disk templates are treated slightly differently
520
  par = "disk-templates"
521
  disp_str = "enabled disk templates"
522
  curr_val = old_policy[disp_str]
523
  test_values = [
524
    (True, constants.DT_PLAIN),
525
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
526
    (False, "thisisnotadisktemplate"),
527
    (False, ""),
528
    # Restore the old value
529
    (True, curr_val.replace(" ", "")),
530
    ]
531
  for (good, val) in test_values:
532
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
533
    AssertCommand(cmd, fail=not good)
534
    if good:
535
      curr_val = val
536
    # Check the affected parameter
537
    (eff_policy, eff_specs) = _GetClusterIPolicy()
538
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
539
    # Check everything else
540
    AssertEqual(eff_specs, old_specs)
541
    for p in eff_policy.keys():
542
      if p == disp_str:
543
        continue
544
      AssertEqual(eff_policy[p], old_policy[p])
545

    
546

    
547
def TestClusterSetISpecs(new_specs, fail=False, old_values=None):
548
  """Change instance specs.
549

550
  @type new_specs: dict of dict
551
  @param new_specs: new_specs[par][key], where key is "min", "max", "std". It
552
      can be an empty dictionary.
553
  @type fail: bool
554
  @param fail: if the change is expected to fail
555
  @type old_values: tuple
556
  @param old_values: (old_policy, old_specs), as returned by
557
     L{_GetClusterIPolicy}
558
  @return: same as L{_GetClusterIPolicy}
559

560
  """
561
  if old_values:
562
    (old_policy, old_specs) = old_values
563
  else:
564
    (old_policy, old_specs) = _GetClusterIPolicy()
565
  if new_specs:
566
    cmd = ["gnt-cluster", "modify"]
567
    for (par, keyvals) in new_specs.items():
568
      if par == "spindle-use":
569
        # ignore spindle-use, which is not settable
570
        continue
571
      cmd += [
572
        "--specs-%s" % par,
573
        ",".join(["%s=%s" % (k, v) for (k, v) in keyvals.items()]),
574
        ]
575
    AssertCommand(cmd, fail=fail)
576
  # Check the new state
577
  (eff_policy, eff_specs) = _GetClusterIPolicy()
578
  AssertEqual(eff_policy, old_policy)
579
  if fail:
580
    AssertEqual(eff_specs, old_specs)
581
  else:
582
    for par in eff_specs:
583
      for key in eff_specs[par]:
584
        if par in new_specs and key in new_specs[par]:
585
          AssertEqual(int(eff_specs[par][key]), int(new_specs[par][key]))
586
        else:
587
          AssertEqual(int(eff_specs[par][key]), int(old_specs[par][key]))
588
  return (eff_policy, eff_specs)
589

    
590

    
591
def TestClusterModifyISpecs():
592
  """gnt-cluster modify --specs-*"""
593
  params = ["mem-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
594
  (cur_policy, cur_specs) = _GetClusterIPolicy()
595
  for par in params:
596
    test_values = [
597
      (True, 0, 4, 12),
598
      (True, 4, 4, 12),
599
      (True, 4, 12, 12),
600
      (True, 4, 4, 4),
601
      (False, 4, 0, 12),
602
      (False, 4, 16, 12),
603
      (False, 4, 4, 0),
604
      (False, 12, 4, 4),
605
      (False, 12, 4, 0),
606
      (False, "a", 4, 12),
607
      (False, 0, "a", 12),
608
      (False, 0, 4, "a"),
609
      # This is to restore the old values
610
      (True,
611
       cur_specs[par]["min"], cur_specs[par]["std"], cur_specs[par]["max"])
612
      ]
613
    for (good, mn, st, mx) in test_values:
614
      new_vals = {par: {"min": str(mn), "std": str(st), "max": str(mx)}}
615
      cur_state = (cur_policy, cur_specs)
616
      # We update cur_specs, as we've copied the values to restore already
617
      (cur_policy, cur_specs) = TestClusterSetISpecs(new_vals, fail=not good,
618
                                                     old_values=cur_state)
619

    
620

    
621
def TestClusterInfo():
622
  """gnt-cluster info"""
623
  AssertCommand(["gnt-cluster", "info"])
624

    
625

    
626
def TestClusterRedistConf():
627
  """gnt-cluster redist-conf"""
628
  AssertCommand(["gnt-cluster", "redist-conf"])
629

    
630

    
631
def TestClusterGetmaster():
632
  """gnt-cluster getmaster"""
633
  AssertCommand(["gnt-cluster", "getmaster"])
634

    
635

    
636
def TestClusterVersion():
637
  """gnt-cluster version"""
638
  AssertCommand(["gnt-cluster", "version"])
639

    
640

    
641
def TestClusterRenewCrypto():
642
  """gnt-cluster renew-crypto"""
643
  master = qa_config.GetMasterNode()
644

    
645
  # Conflicting options
646
  cmd = ["gnt-cluster", "renew-crypto", "--force",
647
         "--new-cluster-certificate", "--new-confd-hmac-key"]
648
  conflicting = [
649
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
650
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
651
    ]
652
  for i in conflicting:
653
    AssertCommand(cmd + i, fail=True)
654

    
655
  # Invalid RAPI certificate
656
  cmd = ["gnt-cluster", "renew-crypto", "--force",
657
         "--rapi-certificate=/dev/null"]
658
  AssertCommand(cmd, fail=True)
659

    
660
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
661
                                         pathutils.RAPI_CERT_FILE)
662
  try:
663
    # Custom RAPI certificate
664
    fh = tempfile.NamedTemporaryFile()
665

    
666
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
667
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
668

    
669
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
670

    
671
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
672
    try:
673
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
674
                     "--rapi-certificate=%s" % tmpcert])
675
    finally:
676
      AssertCommand(["rm", "-f", tmpcert])
677

    
678
    # Custom cluster domain secret
679
    cds_fh = tempfile.NamedTemporaryFile()
680
    cds_fh.write(utils.GenerateSecret())
681
    cds_fh.write("\n")
682
    cds_fh.flush()
683

    
684
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
685
    try:
686
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
687
                     "--cluster-domain-secret=%s" % tmpcds])
688
    finally:
689
      AssertCommand(["rm", "-f", tmpcds])
690

    
691
    # Normal case
692
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
693
                   "--new-cluster-certificate", "--new-confd-hmac-key",
694
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
695

    
696
    # Restore RAPI certificate
697
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
698
                   "--rapi-certificate=%s" % rapi_cert_backup])
699
  finally:
700
    AssertCommand(["rm", "-f", rapi_cert_backup])
701

    
702

    
703
def TestClusterBurnin():
704
  """Burnin"""
705
  master = qa_config.GetMasterNode()
706

    
707
  options = qa_config.get("options", {})
708
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
709
  parallel = options.get("burnin-in-parallel", False)
710
  check_inst = options.get("burnin-check-instances", False)
711
  do_rename = options.get("burnin-rename", "")
712
  do_reboot = options.get("burnin-reboot", True)
713
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
714

    
715
  # Get as many instances as we need
716
  instances = []
717
  try:
718
    try:
719
      num = qa_config.get("options", {}).get("burnin-instances", 1)
720
      for _ in range(0, num):
721
        instances.append(qa_config.AcquireInstance())
722
    except qa_error.OutOfInstancesError:
723
      print "Not enough instances, continuing anyway."
724

    
725
    if len(instances) < 1:
726
      raise qa_error.Error("Burnin needs at least one instance")
727

    
728
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
729
    try:
730
      # Run burnin
731
      cmd = [script,
732
             "--os=%s" % qa_config.get("os"),
733
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
734
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
735
             "--disk-size=%s" % ",".join(qa_config.get("disk")),
736
             "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
737
             "--disk-template=%s" % disk_template]
738
      if parallel:
739
        cmd.append("--parallel")
740
        cmd.append("--early-release")
741
      if check_inst:
742
        cmd.append("--http-check")
743
      if do_rename:
744
        cmd.append("--rename=%s" % do_rename)
745
      if not do_reboot:
746
        cmd.append("--no-reboot")
747
      else:
748
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
749
      cmd += [inst.name for inst in instances]
750
      AssertCommand(cmd)
751
    finally:
752
      AssertCommand(["rm", "-f", script])
753

    
754
  finally:
755
    for inst in instances:
756
      inst.Release()
757

    
758

    
759
def TestClusterMasterFailover():
760
  """gnt-cluster master-failover"""
761
  master = qa_config.GetMasterNode()
762
  failovermaster = qa_config.AcquireNode(exclude=master)
763

    
764
  cmd = ["gnt-cluster", "master-failover"]
765
  try:
766
    AssertCommand(cmd, node=failovermaster)
767
    # Back to original master node
768
    AssertCommand(cmd, node=master)
769
  finally:
770
    failovermaster.Release()
771

    
772

    
773
def _NodeQueueDrainFile(node):
774
  """Returns path to queue drain file for a node.
775

776
  """
777
  return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE)
778

    
779

    
780
def _AssertDrainFile(node, **kwargs):
781
  """Checks for the queue drain file.
782

783
  """
784
  AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs)
785

    
786

    
787
def TestClusterMasterFailoverWithDrainedQueue():
788
  """gnt-cluster master-failover with drained queue"""
789
  master = qa_config.GetMasterNode()
790
  failovermaster = qa_config.AcquireNode(exclude=master)
791

    
792
  # Ensure queue is not drained
793
  for node in [master, failovermaster]:
794
    _AssertDrainFile(node, fail=True)
795

    
796
  # Drain queue on failover master
797
  AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)],
798
                node=failovermaster)
799

    
800
  cmd = ["gnt-cluster", "master-failover"]
801
  try:
802
    _AssertDrainFile(failovermaster)
803
    AssertCommand(cmd, node=failovermaster)
804
    _AssertDrainFile(master, fail=True)
805
    _AssertDrainFile(failovermaster, fail=True)
806

    
807
    # Back to original master node
808
    AssertCommand(cmd, node=master)
809
  finally:
810
    failovermaster.Release()
811

    
812
  # Ensure queue is not drained
813
  for node in [master, failovermaster]:
814
    _AssertDrainFile(node, fail=True)
815

    
816

    
817
def TestClusterCopyfile():
818
  """gnt-cluster copyfile"""
819
  master = qa_config.GetMasterNode()
820

    
821
  uniqueid = utils.NewUUID()
822

    
823
  # Create temporary file
824
  f = tempfile.NamedTemporaryFile()
825
  f.write(uniqueid)
826
  f.flush()
827
  f.seek(0)
828

    
829
  # Upload file to master node
830
  testname = qa_utils.UploadFile(master.primary, f.name)
831
  try:
832
    # Copy file to all nodes
833
    AssertCommand(["gnt-cluster", "copyfile", testname])
834
    _CheckFileOnAllNodes(testname, uniqueid)
835
  finally:
836
    _RemoveFileFromAllNodes(testname)
837

    
838

    
839
def TestClusterCommand():
840
  """gnt-cluster command"""
841
  uniqueid = utils.NewUUID()
842
  rfile = "/tmp/gnt%s" % utils.NewUUID()
843
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
844
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
845
                              "%s >%s" % (rcmd, rfile)])
846

    
847
  try:
848
    AssertCommand(cmd)
849
    _CheckFileOnAllNodes(rfile, uniqueid)
850
  finally:
851
    _RemoveFileFromAllNodes(rfile)
852

    
853

    
854
def TestClusterDestroy():
855
  """gnt-cluster destroy"""
856
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
857

    
858

    
859
def TestClusterRepairDiskSizes():
860
  """gnt-cluster repair-disk-sizes"""
861
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
862

    
863

    
864
def TestSetExclStorCluster(newvalue):
865
  """Set the exclusive_storage node parameter at the cluster level.
866

867
  @type newvalue: bool
868
  @param newvalue: New value of exclusive_storage
869
  @rtype: bool
870
  @return: The old value of exclusive_storage
871

872
  """
873
  es_path = ["Default node parameters", "exclusive_storage"]
874
  oldvalue = _GetClusterField(es_path)
875
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
876
                 "exclusive_storage=%s" % newvalue])
877
  effvalue = _GetClusterField(es_path)
878
  if effvalue != newvalue:
879
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
880
                         " of %s" % (effvalue, newvalue))
881
  qa_config.SetExclusiveStorage(newvalue)
882
  return oldvalue
883

    
884

    
885
def TestExclStorSharedPv(node):
886
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
887

888
  """
889
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
890
  lvname1 = _QA_LV_PREFIX + "vol1"
891
  lvname2 = _QA_LV_PREFIX + "vol2"
892
  node_name = node.primary
893
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
894
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
895
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
896
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
897
                                         constants.CV_ENODEORPHANLV])
898
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
899
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
900
  AssertClusterVerify()