Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ ab4832d1

History | View | Annotate | Download (28.2 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import constants
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34

    
35
import qa_config
36
import qa_utils
37
import qa_error
38

    
39
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
40

    
41

    
42
# Prefix for LVM volumes created by QA code during tests
43
_QA_LV_PREFIX = "qa-"
44

    
45
#: cluster verify command
46
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
47

    
48

    
49
def _RemoveFileFromAllNodes(filename):
50
  """Removes a file from all nodes.
51

52
  """
53
  for node in qa_config.get("nodes"):
54
    AssertCommand(["rm", "-f", filename], node=node)
55

    
56

    
57
def _CheckFileOnAllNodes(filename, content):
58
  """Verifies the content of the given file on all nodes.
59

60
  """
61
  cmd = utils.ShellQuoteArgs(["cat", filename])
62
  for node in qa_config.get("nodes"):
63
    AssertEqual(qa_utils.GetCommandOutput(node["primary"], cmd), content)
64

    
65

    
66
# "gnt-cluster info" fields
67
_CIFIELD_RE = re.compile(r"^[-\s]*(?P<field>[^\s:]+):\s*(?P<value>\S.*)$")
68

    
69

    
70
def _GetBoolClusterField(field):
71
  """Get the Boolean value of a cluster field.
72

73
  This function currently assumes that the field name is unique in the cluster
74
  configuration. An assertion checks this assumption.
75

76
  @type field: string
77
  @param field: Name of the field
78
  @rtype: bool
79
  @return: The effective value of the field
80

81
  """
82
  master = qa_config.GetMasterNode()
83
  infocmd = "gnt-cluster info"
84
  info_out = qa_utils.GetCommandOutput(master["primary"], infocmd)
85
  ret = None
86
  for l in info_out.splitlines():
87
    m = _CIFIELD_RE.match(l)
88
    # FIXME: There should be a way to specify a field through a hierarchy
89
    if m and m.group("field") == field:
90
      # Make sure that ignoring the hierarchy doesn't cause a double match
91
      assert ret is None
92
      ret = (m.group("value").lower() == "true")
93
  if ret is not None:
94
    return ret
95
  raise qa_error.Error("Field not found in cluster configuration: %s" % field)
96

    
97

    
98
# Cluster-verify errors (date, "ERROR", then error code)
99
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
100

    
101

    
102
def _GetCVErrorCodes(cvout):
103
  errs = set()
104
  warns = set()
105
  for l in cvout.splitlines():
106
    m = _CVERROR_RE.match(l)
107
    if m:
108
      etype = m.group(1)
109
      ecode = m.group(2)
110
      if etype == "ERROR":
111
        errs.add(ecode)
112
      elif etype == "WARNING":
113
        warns.add(ecode)
114
  return (errs, warns)
115

    
116

    
117
def _CheckVerifyErrors(actual, expected, etype):
118
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
119
  if not actual.issuperset(exp_codes):
120
    missing = exp_codes.difference(actual)
121
    raise qa_error.Error("Cluster-verify didn't return these expected"
122
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))
123

    
124

    
125
def AssertClusterVerify(fail=False, errors=None, warnings=None):
126
  """Run cluster-verify and check the result
127

128
  @type fail: bool
129
  @param fail: if cluster-verify is expected to fail instead of succeeding
130
  @type errors: list of tuples
131
  @param errors: List of CV_XXX errors that are expected; if specified, all the
132
      errors listed must appear in cluster-verify output. A non-empty value
133
      implies C{fail=True}.
134
  @type warnings: list of tuples
135
  @param warnings: Same as C{errors} but for warnings.
136

137
  """
138
  cvcmd = "gnt-cluster verify"
139
  mnode = qa_config.GetMasterNode()
140
  if errors or warnings:
141
    cvout = GetCommandOutput(mnode["primary"], cvcmd + " --error-codes",
142
                             fail=(fail or errors))
143
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
144
    if errors:
145
      _CheckVerifyErrors(act_errs, errors, "error")
146
    if warnings:
147
      _CheckVerifyErrors(act_warns, warnings, "warning")
148
  else:
149
    AssertCommand(cvcmd, fail=fail, node=mnode)
150

    
151

    
152
# data for testing failures due to bad keys/values for disk parameters
153
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
154
                "drbd:nonexistent=1",
155
                "drbd:resync-rate=invalid",
156
                ]
157

    
158

    
159
def TestClusterInitDisk():
160
  """gnt-cluster init -D"""
161
  name = qa_config.get("name")
162
  for param in _FAIL_PARAMS:
163
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
164

    
165

    
166
def TestClusterInit(rapi_user, rapi_secret):
167
  """gnt-cluster init"""
168
  master = qa_config.GetMasterNode()
169

    
170
  rapi_dir = os.path.dirname(pathutils.RAPI_USERS_FILE)
171

    
172
  # First create the RAPI credentials
173
  fh = tempfile.NamedTemporaryFile()
174
  try:
175
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
176
    fh.flush()
177

    
178
    tmpru = qa_utils.UploadFile(master["primary"], fh.name)
179
    try:
180
      AssertCommand(["mkdir", "-p", rapi_dir])
181
      AssertCommand(["mv", tmpru, pathutils.RAPI_USERS_FILE])
182
    finally:
183
      AssertCommand(["rm", "-f", tmpru])
184
  finally:
185
    fh.close()
186

    
187
  # Initialize cluster
188
  cmd = [
189
    "gnt-cluster", "init",
190
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
191
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
192
    ]
193

    
194
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
195
                    "nic-count"):
196
    for spec_val in ("min", "max", "std"):
197
      spec = qa_config.get("ispec_%s_%s" %
198
                           (spec_type.replace("-", "_"), spec_val), None)
199
      if spec:
200
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
201

    
202
  if master.get("secondary", None):
203
    cmd.append("--secondary-ip=%s" % master["secondary"])
204

    
205
  vgname = qa_config.get("vg-name", None)
206
  if vgname:
207
    cmd.append("--vg-name=%s" % vgname)
208

    
209
  master_netdev = qa_config.get("master-netdev", None)
210
  if master_netdev:
211
    cmd.append("--master-netdev=%s" % master_netdev)
212

    
213
  nicparams = qa_config.get("default-nicparams", None)
214
  if nicparams:
215
    cmd.append("--nic-parameters=%s" %
216
               ",".join(utils.FormatKeyValue(nicparams)))
217

    
218
  # Cluster value of the exclusive-storage node parameter
219
  e_s = qa_config.get("exclusive-storage")
220
  if e_s is not None:
221
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
222
  else:
223
    e_s = False
224
  qa_config.SetExclusiveStorage(e_s)
225

    
226
  extra_args = qa_config.get("cluster-init-args")
227
  if extra_args:
228
    cmd.extend(extra_args)
229

    
230
  cmd.append(qa_config.get("name"))
231

    
232
  AssertCommand(cmd)
233

    
234
  cmd = ["gnt-cluster", "modify"]
235

    
236
  # hypervisor parameter modifications
237
  hvp = qa_config.get("hypervisor-parameters", {})
238
  for k, v in hvp.items():
239
    cmd.extend(["-H", "%s:%s" % (k, v)])
240
  # backend parameter modifications
241
  bep = qa_config.get("backend-parameters", "")
242
  if bep:
243
    cmd.extend(["-B", bep])
244

    
245
  if len(cmd) > 2:
246
    AssertCommand(cmd)
247

    
248
  # OS parameters
249
  osp = qa_config.get("os-parameters", {})
250
  for k, v in osp.items():
251
    AssertCommand(["gnt-os", "modify", "-O", v, k])
252

    
253
  # OS hypervisor parameters
254
  os_hvp = qa_config.get("os-hvp", {})
255
  for os_name in os_hvp:
256
    for hv, hvp in os_hvp[os_name].items():
257
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
258

    
259

    
260
def TestClusterRename():
261
  """gnt-cluster rename"""
262
  cmd = ["gnt-cluster", "rename", "-f"]
263

    
264
  original_name = qa_config.get("name")
265
  rename_target = qa_config.get("rename", None)
266
  if rename_target is None:
267
    print qa_utils.FormatError('"rename" entry is missing')
268
    return
269

    
270
  for data in [
271
    cmd + [rename_target],
272
    _CLUSTER_VERIFY,
273
    cmd + [original_name],
274
    _CLUSTER_VERIFY,
275
    ]:
276
    AssertCommand(data)
277

    
278

    
279
def TestClusterOob():
280
  """out-of-band framework"""
281
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
282

    
283
  AssertCommand(_CLUSTER_VERIFY)
284
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
285
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
286
                 utils.NewUUID()])
287

    
288
  AssertCommand(_CLUSTER_VERIFY, fail=True)
289

    
290
  AssertCommand(["touch", oob_path_exists])
291
  AssertCommand(["chmod", "0400", oob_path_exists])
292
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
293

    
294
  try:
295
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
296
                   "oob_program=%s" % oob_path_exists])
297

    
298
    AssertCommand(_CLUSTER_VERIFY, fail=True)
299

    
300
    AssertCommand(["chmod", "0500", oob_path_exists])
301
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
302

    
303
    AssertCommand(_CLUSTER_VERIFY)
304
  finally:
305
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
306

    
307
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
308
                 "oob_program="])
309

    
310

    
311
def TestClusterEpo():
312
  """gnt-cluster epo"""
313
  master = qa_config.GetMasterNode()
314

    
315
  # Assert that OOB is unavailable for all nodes
316
  result_output = GetCommandOutput(master["primary"],
317
                                   "gnt-node list --verbose --no-headers -o"
318
                                   " powered")
319
  AssertEqual(compat.all(powered == "(unavail)"
320
                         for powered in result_output.splitlines()), True)
321

    
322
  # Conflicting
323
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
324
  # --all doesn't expect arguments
325
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
326

    
327
  # Unless --all is given master is not allowed to be in the list
328
  AssertCommand(["gnt-cluster", "epo", "-f", master["primary"]], fail=True)
329

    
330
  # This shouldn't fail
331
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
332

    
333
  # All instances should have been stopped now
334
  result_output = GetCommandOutput(master["primary"],
335
                                   "gnt-instance list --no-headers -o status")
336
  # ERROR_down because the instance is stopped but not recorded as such
337
  AssertEqual(compat.all(status == "ERROR_down"
338
                         for status in result_output.splitlines()), True)
339

    
340
  # Now start everything again
341
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
342

    
343
  # All instances should have been started now
344
  result_output = GetCommandOutput(master["primary"],
345
                                   "gnt-instance list --no-headers -o status")
346
  AssertEqual(compat.all(status == "running"
347
                         for status in result_output.splitlines()), True)
348

    
349

    
350
def TestClusterVerify():
351
  """gnt-cluster verify"""
352
  AssertCommand(_CLUSTER_VERIFY)
353
  AssertCommand(["gnt-cluster", "verify-disks"])
354

    
355

    
356
def TestJobqueue():
357
  """gnt-debug test-jobqueue"""
358
  AssertCommand(["gnt-debug", "test-jobqueue"])
359

    
360

    
361
def TestDelay(node):
362
  """gnt-debug delay"""
363
  AssertCommand(["gnt-debug", "delay", "1"])
364
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
365
  AssertCommand(["gnt-debug", "delay", "--no-master",
366
                 "-n", node["primary"], "1"])
367

    
368

    
369
def TestClusterReservedLvs():
370
  """gnt-cluster reserved lvs"""
371
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
372
  lvname = _QA_LV_PREFIX + "test"
373
  lvfullname = "/".join([vgname, lvname])
374
  for fail, cmd in [
375
    (False, _CLUSTER_VERIFY),
376
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
377
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
378
    (True, _CLUSTER_VERIFY),
379
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
380
             "%s,.*/other-test" % lvfullname]),
381
    (False, _CLUSTER_VERIFY),
382
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
383
             ".*/%s.*" % _QA_LV_PREFIX]),
384
    (False, _CLUSTER_VERIFY),
385
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
386
    (True, _CLUSTER_VERIFY),
387
    (False, ["lvremove", "-f", lvfullname]),
388
    (False, _CLUSTER_VERIFY),
389
    ]:
390
    AssertCommand(cmd, fail=fail)
391

    
392

    
393
def TestClusterModifyEmpty():
394
  """gnt-cluster modify"""
395
  AssertCommand(["gnt-cluster", "modify"], fail=True)
396

    
397

    
398
def TestClusterModifyDisk():
399
  """gnt-cluster modify -D"""
400
  for param in _FAIL_PARAMS:
401
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
402

    
403

    
404
def TestClusterModifyBe():
405
  """gnt-cluster modify -B"""
406
  for fail, cmd in [
407
    # max/min mem
408
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
409
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
410
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
411
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
412
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
413
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
414
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
415
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
416
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
417
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
418
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
419
    # vcpus
420
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
421
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
422
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
423
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
424
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
425
    # auto_balance
426
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
427
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
428
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
429
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
430
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
431
    ]:
432
    AssertCommand(cmd, fail=fail)
433

    
434
  # redo the original-requested BE parameters, if any
435
  bep = qa_config.get("backend-parameters", "")
436
  if bep:
437
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
438

    
439

    
440
_START_IPOLICY_RE = re.compile(r"^(\s*)Instance policy")
441
_START_ISPEC_RE = re.compile(r"^\s+-\s+(std|min|max)")
442
_VALUE_RE = r"([^\s:][^:]*):\s+(\S.*)$"
443
_IPOLICY_PARAM_RE = re.compile(r"^\s+-\s+" + _VALUE_RE)
444
_ISPEC_VALUE_RE = re.compile(r"^\s+" + _VALUE_RE)
445

    
446

    
447
def _GetClusterIPolicy():
448
  """Return the run-time values of the cluster-level instance policy.
449

450
  @rtype: tuple
451
  @return: (policy, specs), where:
452
      - policy is a dictionary of the policy values, instance specs excluded
453
      - specs is dict of dict, specs[par][key] is a spec value, where key is
454
        "min", "max", or "std"
455

456
  """
457
  mnode = qa_config.GetMasterNode()
458
  info = GetCommandOutput(mnode["primary"], "gnt-cluster info")
459
  inside_policy = False
460
  end_ispec_re = None
461
  curr_spec = ""
462
  specs = {}
463
  policy = {}
464
  for line in info.splitlines():
465
    if inside_policy:
466
      # The order of the matching is important, as some REs overlap
467
      m = _START_ISPEC_RE.match(line)
468
      if m:
469
        curr_spec = m.group(1)
470
        continue
471
      m = _IPOLICY_PARAM_RE.match(line)
472
      if m:
473
        policy[m.group(1)] = m.group(2).strip()
474
        continue
475
      m = _ISPEC_VALUE_RE.match(line)
476
      if m:
477
        assert curr_spec
478
        par = m.group(1)
479
        if par == "memory-size":
480
          par = "mem-size"
481
        d = specs.setdefault(par, {})
482
        d[curr_spec] = m.group(2).strip()
483
        continue
484
      assert end_ispec_re is not None
485
      if end_ispec_re.match(line):
486
        inside_policy = False
487
    else:
488
      m = _START_IPOLICY_RE.match(line)
489
      if m:
490
        inside_policy = True
491
        # We stop parsing when we find the same indentation level
492
        re_str = r"^\s{%s}\S" % len(m.group(1))
493
        end_ispec_re = re.compile(re_str)
494
  # Sanity checks
495
  assert len(specs) > 0
496
  good = ("min" in d and "std" in d and "max" in d for d in specs)
497
  assert good, "Missing item in specs: %s" % specs
498
  assert len(policy) > 0
499
  return (policy, specs)
500

    
501

    
502
def TestClusterModifyIPolicy():
503
  """gnt-cluster modify --ipolicy-*"""
504
  basecmd = ["gnt-cluster", "modify"]
505
  (old_policy, old_specs) = _GetClusterIPolicy()
506
  for par in ["vcpu-ratio", "spindle-ratio"]:
507
    curr_val = float(old_policy[par])
508
    test_values = [
509
      (True, 1.0),
510
      (True, 1.5),
511
      (True, 2),
512
      (False, "a"),
513
      # Restore the old value
514
      (True, curr_val),
515
      ]
516
    for (good, val) in test_values:
517
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
518
      AssertCommand(cmd, fail=not good)
519
      if good:
520
        curr_val = val
521
      # Check the affected parameter
522
      (eff_policy, eff_specs) = _GetClusterIPolicy()
523
      AssertEqual(float(eff_policy[par]), curr_val)
524
      # Check everything else
525
      AssertEqual(eff_specs, old_specs)
526
      for p in eff_policy.keys():
527
        if p == par:
528
          continue
529
        AssertEqual(eff_policy[p], old_policy[p])
530

    
531
  # Disk templates are treated slightly differently
532
  par = "disk-templates"
533
  disp_str = "enabled disk templates"
534
  curr_val = old_policy[disp_str]
535
  test_values = [
536
    (True, constants.DT_PLAIN),
537
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
538
    (False, "thisisnotadisktemplate"),
539
    (False, ""),
540
    # Restore the old value
541
    (True, curr_val.replace(" ", "")),
542
    ]
543
  for (good, val) in test_values:
544
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
545
    AssertCommand(cmd, fail=not good)
546
    if good:
547
      curr_val = val
548
    # Check the affected parameter
549
    (eff_policy, eff_specs) = _GetClusterIPolicy()
550
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
551
    # Check everything else
552
    AssertEqual(eff_specs, old_specs)
553
    for p in eff_policy.keys():
554
      if p == disp_str:
555
        continue
556
      AssertEqual(eff_policy[p], old_policy[p])
557

    
558

    
559
def TestClusterSetISpecs(new_specs, fail=False, old_values=None):
560
  """Change instance specs.
561

562
  @type new_specs: dict of dict
563
  @param new_specs: new_specs[par][key], where key is "min", "max", "std". It
564
      can be an empty dictionary.
565
  @type fail: bool
566
  @param fail: if the change is expected to fail
567
  @type old_values: tuple
568
  @param old_values: (old_policy, old_specs), as returned by
569
     L{_GetClusterIPolicy}
570
  @return: same as L{_GetClusterIPolicy}
571

572
  """
573
  if old_values:
574
    (old_policy, old_specs) = old_values
575
  else:
576
    (old_policy, old_specs) = _GetClusterIPolicy()
577
  if new_specs:
578
    cmd = ["gnt-cluster", "modify"]
579
    for (par, keyvals) in new_specs.items():
580
      if par == "spindle-use":
581
        # ignore spindle-use, which is not settable
582
        continue
583
      cmd += [
584
        "--specs-%s" % par,
585
        ",".join(["%s=%s" % (k, v) for (k, v) in keyvals.items()]),
586
        ]
587
    AssertCommand(cmd, fail=fail)
588
  # Check the new state
589
  (eff_policy, eff_specs) = _GetClusterIPolicy()
590
  AssertEqual(eff_policy, old_policy)
591
  if fail:
592
    AssertEqual(eff_specs, old_specs)
593
  else:
594
    for par in eff_specs:
595
      for key in eff_specs[par]:
596
        if par in new_specs and key in new_specs[par]:
597
          AssertEqual(int(eff_specs[par][key]), int(new_specs[par][key]))
598
        else:
599
          AssertEqual(int(eff_specs[par][key]), int(old_specs[par][key]))
600
  return (eff_policy, eff_specs)
601

    
602

    
603
def TestClusterModifyISpecs():
604
  """gnt-cluster modify --specs-*"""
605
  params = ["mem-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
606
  (cur_policy, cur_specs) = _GetClusterIPolicy()
607
  for par in params:
608
    test_values = [
609
      (True, 0, 4, 12),
610
      (True, 4, 4, 12),
611
      (True, 4, 12, 12),
612
      (True, 4, 4, 4),
613
      (False, 4, 0, 12),
614
      (False, 4, 16, 12),
615
      (False, 4, 4, 0),
616
      (False, 12, 4, 4),
617
      (False, 12, 4, 0),
618
      (False, "a", 4, 12),
619
      (False, 0, "a", 12),
620
      (False, 0, 4, "a"),
621
      # This is to restore the old values
622
      (True,
623
       cur_specs[par]["min"], cur_specs[par]["std"], cur_specs[par]["max"])
624
      ]
625
    for (good, mn, st, mx) in test_values:
626
      new_vals = {par: {"min": str(mn), "std": str(st), "max": str(mx)}}
627
      cur_state = (cur_policy, cur_specs)
628
      # We update cur_specs, as we've copied the values to restore already
629
      (cur_policy, cur_specs) = TestClusterSetISpecs(new_vals, fail=not good,
630
                                                     old_values=cur_state)
631

    
632

    
633
def TestClusterInfo():
634
  """gnt-cluster info"""
635
  AssertCommand(["gnt-cluster", "info"])
636

    
637

    
638
def TestClusterRedistConf():
639
  """gnt-cluster redist-conf"""
640
  AssertCommand(["gnt-cluster", "redist-conf"])
641

    
642

    
643
def TestClusterGetmaster():
644
  """gnt-cluster getmaster"""
645
  AssertCommand(["gnt-cluster", "getmaster"])
646

    
647

    
648
def TestClusterVersion():
649
  """gnt-cluster version"""
650
  AssertCommand(["gnt-cluster", "version"])
651

    
652

    
653
def TestClusterRenewCrypto():
654
  """gnt-cluster renew-crypto"""
655
  master = qa_config.GetMasterNode()
656

    
657
  # Conflicting options
658
  cmd = ["gnt-cluster", "renew-crypto", "--force",
659
         "--new-cluster-certificate", "--new-confd-hmac-key"]
660
  conflicting = [
661
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
662
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
663
    ]
664
  for i in conflicting:
665
    AssertCommand(cmd + i, fail=True)
666

    
667
  # Invalid RAPI certificate
668
  cmd = ["gnt-cluster", "renew-crypto", "--force",
669
         "--rapi-certificate=/dev/null"]
670
  AssertCommand(cmd, fail=True)
671

    
672
  rapi_cert_backup = qa_utils.BackupFile(master["primary"],
673
                                         pathutils.RAPI_CERT_FILE)
674
  try:
675
    # Custom RAPI certificate
676
    fh = tempfile.NamedTemporaryFile()
677

    
678
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
679
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
680

    
681
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
682

    
683
    tmpcert = qa_utils.UploadFile(master["primary"], fh.name)
684
    try:
685
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
686
                     "--rapi-certificate=%s" % tmpcert])
687
    finally:
688
      AssertCommand(["rm", "-f", tmpcert])
689

    
690
    # Custom cluster domain secret
691
    cds_fh = tempfile.NamedTemporaryFile()
692
    cds_fh.write(utils.GenerateSecret())
693
    cds_fh.write("\n")
694
    cds_fh.flush()
695

    
696
    tmpcds = qa_utils.UploadFile(master["primary"], cds_fh.name)
697
    try:
698
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
699
                     "--cluster-domain-secret=%s" % tmpcds])
700
    finally:
701
      AssertCommand(["rm", "-f", tmpcds])
702

    
703
    # Normal case
704
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
705
                   "--new-cluster-certificate", "--new-confd-hmac-key",
706
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
707

    
708
    # Restore RAPI certificate
709
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
710
                   "--rapi-certificate=%s" % rapi_cert_backup])
711
  finally:
712
    AssertCommand(["rm", "-f", rapi_cert_backup])
713

    
714

    
715
def TestClusterBurnin():
716
  """Burnin"""
717
  master = qa_config.GetMasterNode()
718

    
719
  options = qa_config.get("options", {})
720
  disk_template = options.get("burnin-disk-template", "drbd")
721
  parallel = options.get("burnin-in-parallel", False)
722
  check_inst = options.get("burnin-check-instances", False)
723
  do_rename = options.get("burnin-rename", "")
724
  do_reboot = options.get("burnin-reboot", True)
725
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
726

    
727
  # Get as many instances as we need
728
  instances = []
729
  try:
730
    try:
731
      num = qa_config.get("options", {}).get("burnin-instances", 1)
732
      for _ in range(0, num):
733
        instances.append(qa_config.AcquireInstance())
734
    except qa_error.OutOfInstancesError:
735
      print "Not enough instances, continuing anyway."
736

    
737
    if len(instances) < 1:
738
      raise qa_error.Error("Burnin needs at least one instance")
739

    
740
    script = qa_utils.UploadFile(master["primary"], "../tools/burnin")
741
    try:
742
      # Run burnin
743
      cmd = [script,
744
             "--os=%s" % qa_config.get("os"),
745
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
746
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
747
             "--disk-size=%s" % ",".join(qa_config.get("disk")),
748
             "--disk-growth=%s" % ",".join(qa_config.get("disk-growth")),
749
             "--disk-template=%s" % disk_template]
750
      if parallel:
751
        cmd.append("--parallel")
752
        cmd.append("--early-release")
753
      if check_inst:
754
        cmd.append("--http-check")
755
      if do_rename:
756
        cmd.append("--rename=%s" % do_rename)
757
      if not do_reboot:
758
        cmd.append("--no-reboot")
759
      else:
760
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
761
      cmd += [inst["name"] for inst in instances]
762
      AssertCommand(cmd)
763
    finally:
764
      AssertCommand(["rm", "-f", script])
765

    
766
  finally:
767
    for inst in instances:
768
      qa_config.ReleaseInstance(inst)
769

    
770

    
771
def TestClusterMasterFailover():
772
  """gnt-cluster master-failover"""
773
  master = qa_config.GetMasterNode()
774
  failovermaster = qa_config.AcquireNode(exclude=master)
775

    
776
  cmd = ["gnt-cluster", "master-failover"]
777
  try:
778
    AssertCommand(cmd, node=failovermaster)
779
    # Back to original master node
780
    AssertCommand(cmd, node=master)
781
  finally:
782
    qa_config.ReleaseNode(failovermaster)
783

    
784

    
785
def TestClusterMasterFailoverWithDrainedQueue():
786
  """gnt-cluster master-failover with drained queue"""
787
  drain_check = ["test", "-f", pathutils.JOB_QUEUE_DRAIN_FILE]
788

    
789
  master = qa_config.GetMasterNode()
790
  failovermaster = qa_config.AcquireNode(exclude=master)
791

    
792
  # Ensure queue is not drained
793
  for node in [master, failovermaster]:
794
    AssertCommand(drain_check, node=node, fail=True)
795

    
796
  # Drain queue on failover master
797
  AssertCommand(["touch", pathutils.JOB_QUEUE_DRAIN_FILE], node=failovermaster)
798

    
799
  cmd = ["gnt-cluster", "master-failover"]
800
  try:
801
    AssertCommand(drain_check, node=failovermaster)
802
    AssertCommand(cmd, node=failovermaster)
803
    AssertCommand(drain_check, fail=True)
804
    AssertCommand(drain_check, node=failovermaster, fail=True)
805

    
806
    # Back to original master node
807
    AssertCommand(cmd, node=master)
808
  finally:
809
    qa_config.ReleaseNode(failovermaster)
810

    
811
  AssertCommand(drain_check, fail=True)
812
  AssertCommand(drain_check, node=failovermaster, fail=True)
813

    
814

    
815
def TestClusterCopyfile():
816
  """gnt-cluster copyfile"""
817
  master = qa_config.GetMasterNode()
818

    
819
  uniqueid = utils.NewUUID()
820

    
821
  # Create temporary file
822
  f = tempfile.NamedTemporaryFile()
823
  f.write(uniqueid)
824
  f.flush()
825
  f.seek(0)
826

    
827
  # Upload file to master node
828
  testname = qa_utils.UploadFile(master["primary"], f.name)
829
  try:
830
    # Copy file to all nodes
831
    AssertCommand(["gnt-cluster", "copyfile", testname])
832
    _CheckFileOnAllNodes(testname, uniqueid)
833
  finally:
834
    _RemoveFileFromAllNodes(testname)
835

    
836

    
837
def TestClusterCommand():
838
  """gnt-cluster command"""
839
  uniqueid = utils.NewUUID()
840
  rfile = "/tmp/gnt%s" % utils.NewUUID()
841
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
842
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
843
                              "%s >%s" % (rcmd, rfile)])
844

    
845
  try:
846
    AssertCommand(cmd)
847
    _CheckFileOnAllNodes(rfile, uniqueid)
848
  finally:
849
    _RemoveFileFromAllNodes(rfile)
850

    
851

    
852
def TestClusterDestroy():
853
  """gnt-cluster destroy"""
854
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
855

    
856

    
857
def TestClusterRepairDiskSizes():
858
  """gnt-cluster repair-disk-sizes"""
859
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
860

    
861

    
862
def TestSetExclStorCluster(newvalue):
863
  """Set the exclusive_storage node parameter at the cluster level.
864

865
  @type newvalue: bool
866
  @param newvalue: New value of exclusive_storage
867
  @rtype: bool
868
  @return: The old value of exclusive_storage
869

870
  """
871
  oldvalue = _GetBoolClusterField("exclusive_storage")
872
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
873
                 "exclusive_storage=%s" % newvalue])
874
  effvalue = _GetBoolClusterField("exclusive_storage")
875
  if effvalue != newvalue:
876
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
877
                         " of %s" % (effvalue, newvalue))
878
  qa_config.SetExclusiveStorage(newvalue)
879
  return oldvalue
880

    
881

    
882
def TestExclStorSharedPv(node):
883
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
884

885
  """
886
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
887
  lvname1 = _QA_LV_PREFIX + "vol1"
888
  lvname2 = _QA_LV_PREFIX + "vol2"
889
  node_name = node["primary"]
890
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
891
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
892
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
893
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
894
                                         constants.CV_ENODEORPHANLV])
895
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
896
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
897
  AssertClusterVerify()