Statistics
| Branch: | Tag: | Revision:

root / qa / qa_cluster.py @ ec996117

History | View | Annotate | Download (30.3 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2010, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Cluster related QA tests.
23

24
"""
25

    
26
import re
27
import tempfile
28
import os.path
29

    
30
from ganeti import constants
31
from ganeti import compat
32
from ganeti import utils
33
from ganeti import pathutils
34

    
35
import qa_config
36
import qa_utils
37
import qa_error
38
import qa_instance
39

    
40
from qa_utils import AssertEqual, AssertCommand, GetCommandOutput
41

    
42

    
43
# Prefix for LVM volumes created by QA code during tests
44
_QA_LV_PREFIX = "qa-"
45

    
46
#: cluster verify command
47
_CLUSTER_VERIFY = ["gnt-cluster", "verify"]
48

    
49

    
50
def _RemoveFileFromAllNodes(filename):
51
  """Removes a file from all nodes.
52

53
  """
54
  for node in qa_config.get("nodes"):
55
    AssertCommand(["rm", "-f", filename], node=node)
56

    
57

    
58
def _CheckFileOnAllNodes(filename, content):
59
  """Verifies the content of the given file on all nodes.
60

61
  """
62
  cmd = utils.ShellQuoteArgs(["cat", filename])
63
  for node in qa_config.get("nodes"):
64
    AssertEqual(qa_utils.GetCommandOutput(node.primary, cmd), content)
65

    
66

    
67
def _GetClusterField(field_path):
68
  """Get the value of a cluster field.
69

70
  @type field_path: list of strings
71
  @param field_path: Names of the groups/fields to navigate to get the desired
72
      value, e.g. C{["Default node parameters", "oob_program"]}
73
  @return: The effective value of the field (the actual type depends on the
74
      chosen field)
75

76
  """
77
  assert isinstance(field_path, list)
78
  assert field_path
79
  ret = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
80
  for key in field_path:
81
    ret = ret[key]
82
  return ret
83

    
84

    
85
# Cluster-verify errors (date, "ERROR", then error code)
86
_CVERROR_RE = re.compile(r"^[\w\s:]+\s+- (ERROR|WARNING):([A-Z0-9_-]+):")
87

    
88

    
89
def _GetCVErrorCodes(cvout):
90
  errs = set()
91
  warns = set()
92
  for l in cvout.splitlines():
93
    m = _CVERROR_RE.match(l)
94
    if m:
95
      etype = m.group(1)
96
      ecode = m.group(2)
97
      if etype == "ERROR":
98
        errs.add(ecode)
99
      elif etype == "WARNING":
100
        warns.add(ecode)
101
  return (errs, warns)
102

    
103

    
104
def _CheckVerifyErrors(actual, expected, etype):
105
  exp_codes = compat.UniqueFrozenset(e for (_, e, _) in expected)
106
  if not actual.issuperset(exp_codes):
107
    missing = exp_codes.difference(actual)
108
    raise qa_error.Error("Cluster-verify didn't return these expected"
109
                         " %ss: %s" % (etype, utils.CommaJoin(missing)))
110

    
111

    
112
def AssertClusterVerify(fail=False, errors=None, warnings=None):
113
  """Run cluster-verify and check the result
114

115
  @type fail: bool
116
  @param fail: if cluster-verify is expected to fail instead of succeeding
117
  @type errors: list of tuples
118
  @param errors: List of CV_XXX errors that are expected; if specified, all the
119
      errors listed must appear in cluster-verify output. A non-empty value
120
      implies C{fail=True}.
121
  @type warnings: list of tuples
122
  @param warnings: Same as C{errors} but for warnings.
123

124
  """
125
  cvcmd = "gnt-cluster verify"
126
  mnode = qa_config.GetMasterNode()
127
  if errors or warnings:
128
    cvout = GetCommandOutput(mnode.primary, cvcmd + " --error-codes",
129
                             fail=(fail or errors))
130
    (act_errs, act_warns) = _GetCVErrorCodes(cvout)
131
    if errors:
132
      _CheckVerifyErrors(act_errs, errors, "error")
133
    if warnings:
134
      _CheckVerifyErrors(act_warns, warnings, "warning")
135
  else:
136
    AssertCommand(cvcmd, fail=fail, node=mnode)
137

    
138

    
139
# data for testing failures due to bad keys/values for disk parameters
140
_FAIL_PARAMS = ["nonexistent:resync-rate=1",
141
                "drbd:nonexistent=1",
142
                "drbd:resync-rate=invalid",
143
                ]
144

    
145

    
146
def TestClusterInitDisk():
147
  """gnt-cluster init -D"""
148
  name = qa_config.get("name")
149
  for param in _FAIL_PARAMS:
150
    AssertCommand(["gnt-cluster", "init", "-D", param, name], fail=True)
151

    
152

    
153
def TestClusterInit(rapi_user, rapi_secret):
154
  """gnt-cluster init"""
155
  master = qa_config.GetMasterNode()
156

    
157
  rapi_users_path = qa_utils.MakeNodePath(master, pathutils.RAPI_USERS_FILE)
158
  rapi_dir = os.path.dirname(rapi_users_path)
159

    
160
  # First create the RAPI credentials
161
  fh = tempfile.NamedTemporaryFile()
162
  try:
163
    fh.write("%s %s write\n" % (rapi_user, rapi_secret))
164
    fh.flush()
165

    
166
    tmpru = qa_utils.UploadFile(master.primary, fh.name)
167
    try:
168
      AssertCommand(["mkdir", "-p", rapi_dir])
169
      AssertCommand(["mv", tmpru, rapi_users_path])
170
    finally:
171
      AssertCommand(["rm", "-f", tmpru])
172
  finally:
173
    fh.close()
174

    
175
  # Initialize cluster
176
  cmd = [
177
    "gnt-cluster", "init",
178
    "--primary-ip-version=%d" % qa_config.get("primary_ip_version", 4),
179
    "--enabled-hypervisors=%s" % ",".join(qa_config.GetEnabledHypervisors()),
180
    "--enabled-disk-templates=%s" %
181
      ",".join(qa_config.GetEnabledDiskTemplates())
182
    ]
183

    
184
  for spec_type in ("mem-size", "disk-size", "disk-count", "cpu-count",
185
                    "nic-count"):
186
    for spec_val in ("min", "max", "std"):
187
      spec = qa_config.get("ispec_%s_%s" %
188
                           (spec_type.replace("-", "_"), spec_val), None)
189
      if spec is not None:
190
        cmd.append("--specs-%s=%s=%d" % (spec_type, spec_val, spec))
191

    
192
  if master.secondary:
193
    cmd.append("--secondary-ip=%s" % master.secondary)
194

    
195
  vgname = qa_config.get("vg-name", None)
196
  if vgname:
197
    cmd.append("--vg-name=%s" % vgname)
198

    
199
  master_netdev = qa_config.get("master-netdev", None)
200
  if master_netdev:
201
    cmd.append("--master-netdev=%s" % master_netdev)
202

    
203
  nicparams = qa_config.get("default-nicparams", None)
204
  if nicparams:
205
    cmd.append("--nic-parameters=%s" %
206
               ",".join(utils.FormatKeyValue(nicparams)))
207

    
208
  # Cluster value of the exclusive-storage node parameter
209
  e_s = qa_config.get("exclusive-storage")
210
  if e_s is not None:
211
    cmd.extend(["--node-parameters", "exclusive_storage=%s" % e_s])
212
  else:
213
    e_s = False
214
  qa_config.SetExclusiveStorage(e_s)
215

    
216
  extra_args = qa_config.get("cluster-init-args")
217
  if extra_args:
218
    cmd.extend(extra_args)
219

    
220
  cmd.append(qa_config.get("name"))
221

    
222
  AssertCommand(cmd)
223

    
224
  cmd = ["gnt-cluster", "modify"]
225

    
226
  # hypervisor parameter modifications
227
  hvp = qa_config.get("hypervisor-parameters", {})
228
  for k, v in hvp.items():
229
    cmd.extend(["-H", "%s:%s" % (k, v)])
230
  # backend parameter modifications
231
  bep = qa_config.get("backend-parameters", "")
232
  if bep:
233
    cmd.extend(["-B", bep])
234

    
235
  if len(cmd) > 2:
236
    AssertCommand(cmd)
237

    
238
  # OS parameters
239
  osp = qa_config.get("os-parameters", {})
240
  for k, v in osp.items():
241
    AssertCommand(["gnt-os", "modify", "-O", v, k])
242

    
243
  # OS hypervisor parameters
244
  os_hvp = qa_config.get("os-hvp", {})
245
  for os_name in os_hvp:
246
    for hv, hvp in os_hvp[os_name].items():
247
      AssertCommand(["gnt-os", "modify", "-H", "%s:%s" % (hv, hvp), os_name])
248

    
249

    
250
def TestClusterRename():
251
  """gnt-cluster rename"""
252
  cmd = ["gnt-cluster", "rename", "-f"]
253

    
254
  original_name = qa_config.get("name")
255
  rename_target = qa_config.get("rename", None)
256
  if rename_target is None:
257
    print qa_utils.FormatError('"rename" entry is missing')
258
    return
259

    
260
  for data in [
261
    cmd + [rename_target],
262
    _CLUSTER_VERIFY,
263
    cmd + [original_name],
264
    _CLUSTER_VERIFY,
265
    ]:
266
    AssertCommand(data)
267

    
268

    
269
def TestClusterOob():
270
  """out-of-band framework"""
271
  oob_path_exists = "/tmp/ganeti-qa-oob-does-exist-%s" % utils.NewUUID()
272

    
273
  AssertCommand(_CLUSTER_VERIFY)
274
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
275
                 "oob_program=/tmp/ganeti-qa-oob-does-not-exist-%s" %
276
                 utils.NewUUID()])
277

    
278
  AssertCommand(_CLUSTER_VERIFY, fail=True)
279

    
280
  AssertCommand(["touch", oob_path_exists])
281
  AssertCommand(["chmod", "0400", oob_path_exists])
282
  AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
283

    
284
  try:
285
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
286
                   "oob_program=%s" % oob_path_exists])
287

    
288
    AssertCommand(_CLUSTER_VERIFY, fail=True)
289

    
290
    AssertCommand(["chmod", "0500", oob_path_exists])
291
    AssertCommand(["gnt-cluster", "copyfile", oob_path_exists])
292

    
293
    AssertCommand(_CLUSTER_VERIFY)
294
  finally:
295
    AssertCommand(["gnt-cluster", "command", "rm", oob_path_exists])
296

    
297
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
298
                 "oob_program="])
299

    
300

    
301
def TestClusterEpo():
302
  """gnt-cluster epo"""
303
  master = qa_config.GetMasterNode()
304

    
305
  # Assert that OOB is unavailable for all nodes
306
  result_output = GetCommandOutput(master.primary,
307
                                   "gnt-node list --verbose --no-headers -o"
308
                                   " powered")
309
  AssertEqual(compat.all(powered == "(unavail)"
310
                         for powered in result_output.splitlines()), True)
311

    
312
  # Conflicting
313
  AssertCommand(["gnt-cluster", "epo", "--groups", "--all"], fail=True)
314
  # --all doesn't expect arguments
315
  AssertCommand(["gnt-cluster", "epo", "--all", "some_arg"], fail=True)
316

    
317
  # Unless --all is given master is not allowed to be in the list
318
  AssertCommand(["gnt-cluster", "epo", "-f", master.primary], fail=True)
319

    
320
  # This shouldn't fail
321
  AssertCommand(["gnt-cluster", "epo", "-f", "--all"])
322

    
323
  # All instances should have been stopped now
324
  result_output = GetCommandOutput(master.primary,
325
                                   "gnt-instance list --no-headers -o status")
326
  # ERROR_down because the instance is stopped but not recorded as such
327
  AssertEqual(compat.all(status == "ERROR_down"
328
                         for status in result_output.splitlines()), True)
329

    
330
  # Now start everything again
331
  AssertCommand(["gnt-cluster", "epo", "--on", "-f", "--all"])
332

    
333
  # All instances should have been started now
334
  result_output = GetCommandOutput(master.primary,
335
                                   "gnt-instance list --no-headers -o status")
336
  AssertEqual(compat.all(status == "running"
337
                         for status in result_output.splitlines()), True)
338

    
339

    
340
def TestClusterVerify():
341
  """gnt-cluster verify"""
342
  AssertCommand(_CLUSTER_VERIFY)
343
  AssertCommand(["gnt-cluster", "verify-disks"])
344

    
345

    
346
def TestJobqueue():
347
  """gnt-debug test-jobqueue"""
348
  AssertCommand(["gnt-debug", "test-jobqueue"])
349

    
350

    
351
def TestDelay(node):
352
  """gnt-debug delay"""
353
  AssertCommand(["gnt-debug", "delay", "1"])
354
  AssertCommand(["gnt-debug", "delay", "--no-master", "1"])
355
  AssertCommand(["gnt-debug", "delay", "--no-master",
356
                 "-n", node.primary, "1"])
357

    
358

    
359
def TestClusterReservedLvs():
360
  """gnt-cluster reserved lvs"""
361
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
362
  lvname = _QA_LV_PREFIX + "test"
363
  lvfullname = "/".join([vgname, lvname])
364
  for fail, cmd in [
365
    (False, _CLUSTER_VERIFY),
366
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
367
    (False, ["lvcreate", "-L1G", "-n", lvname, vgname]),
368
    (True, _CLUSTER_VERIFY),
369
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
370
             "%s,.*/other-test" % lvfullname]),
371
    (False, _CLUSTER_VERIFY),
372
    (False, ["gnt-cluster", "modify", "--reserved-lvs",
373
             ".*/%s.*" % _QA_LV_PREFIX]),
374
    (False, _CLUSTER_VERIFY),
375
    (False, ["gnt-cluster", "modify", "--reserved-lvs", ""]),
376
    (True, _CLUSTER_VERIFY),
377
    (False, ["lvremove", "-f", lvfullname]),
378
    (False, _CLUSTER_VERIFY),
379
    ]:
380
    AssertCommand(cmd, fail=fail)
381

    
382

    
383
def TestClusterModifyEmpty():
384
  """gnt-cluster modify"""
385
  AssertCommand(["gnt-cluster", "modify"], fail=True)
386

    
387

    
388
def TestClusterModifyDisk():
389
  """gnt-cluster modify -D"""
390
  for param in _FAIL_PARAMS:
391
    AssertCommand(["gnt-cluster", "modify", "-D", param], fail=True)
392

    
393

    
394
def TestClusterModifyDiskTemplates():
395
  """gnt-cluster modify --enabled-disk-templates=..."""
396
  enabled_disk_templates = qa_config.GetEnabledDiskTemplates()
397
  default_disk_template = qa_config.GetDefaultDiskTemplate()
398

    
399
  _TestClusterModifyDiskTemplatesArguments(default_disk_template,
400
                                           enabled_disk_templates)
401

    
402
  _RestoreEnabledDiskTemplates()
403
  nodes = qa_config.AcquireManyNodes(2)
404

    
405
  instance_template = enabled_disk_templates[0]
406
  instance = qa_instance.CreateInstanceByDiskTemplate(nodes, instance_template)
407

    
408
  _TestClusterModifyUnusedDiskTemplate(instance_template)
409
  _TestClusterModifyUsedDiskTemplate(instance_template,
410
                                     enabled_disk_templates)
411

    
412
  qa_instance.TestInstanceRemove(instance)
413
  _RestoreEnabledDiskTemplates()
414

    
415

    
416
def _RestoreEnabledDiskTemplates():
417
  """Sets the list of enabled disk templates back to the list of enabled disk
418
     templates from the QA configuration. This can be used to make sure that
419
     the tests that modify the list of disk templates do not interfere with
420
     other tests.
421

422
  """
423
  AssertCommand(
424
    ["gnt-cluster", "modify",
425
     "--enabled-disk-template=%s" %
426
       ",".join(qa_config.GetEnabledDiskTemplates())],
427
    fail=False)
428

    
429

    
430
def _TestClusterModifyDiskTemplatesArguments(default_disk_template,
431
                                             enabled_disk_templates):
432
  """Tests argument handling of 'gnt-cluster modify' with respect to
433
     the parameter '--enabled-disk-templates'. This test is independent
434
     of instances.
435

436
  """
437
  AssertCommand(
438
    ["gnt-cluster", "modify",
439
     "--enabled-disk-template=%s" %
440
       ",".join(enabled_disk_templates)],
441
    fail=False)
442

    
443
  # bogus templates
444
  AssertCommand(["gnt-cluster", "modify",
445
                 "--enabled-disk-templates=pinkbunny"],
446
                fail=True)
447

    
448
  # duplicate entries do no harm
449
  AssertCommand(
450
    ["gnt-cluster", "modify",
451
     "--enabled-disk-templates=%s,%s" %
452
      (default_disk_template, default_disk_template)],
453
    fail=False)
454

    
455

    
456
def _TestClusterModifyUsedDiskTemplate(instance_template,
457
                                       enabled_disk_templates):
458
  """Tests that disk templates that are currently in use by instances cannot
459
     be disabled on the cluster.
460

461
  """
462
  # If the list of enabled disk templates contains only one template
463
  # we need to add some other templates, because the list of enabled disk
464
  # templates can only be set to a non-empty list.
465
  new_disk_templates = list(set(enabled_disk_templates)
466
                              - set([instance_template]))
467
  if not new_disk_templates:
468
    new_disk_templates = list(set(constants.DISK_TEMPLATES)
469
                                - set([instance_template]))
470
  AssertCommand(
471
    ["gnt-cluster", "modify",
472
     "--enabled-disk-templates=%s" %
473
       ",".join(new_disk_templates)],
474
    fail=True)
475

    
476

    
477
def _TestClusterModifyUnusedDiskTemplate(instance_template):
478
  """Tests that unused disk templates can be disabled safely."""
479
  all_disk_templates = constants.DISK_TEMPLATES
480
  AssertCommand(
481
    ["gnt-cluster", "modify",
482
     "--enabled-disk-templates=%s" %
483
       ",".join(all_disk_templates)],
484
    fail=False)
485
  new_disk_templates = [instance_template]
486
  AssertCommand(
487
    ["gnt-cluster", "modify",
488
     "--enabled-disk-templates=%s" %
489
       ",".join(new_disk_templates)],
490
    fail=False)
491

    
492

    
493
def TestClusterModifyBe():
494
  """gnt-cluster modify -B"""
495
  for fail, cmd in [
496
    # max/min mem
497
    (False, ["gnt-cluster", "modify", "-B", "maxmem=256"]),
498
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
499
    (False, ["gnt-cluster", "modify", "-B", "minmem=256"]),
500
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
501
    (True, ["gnt-cluster", "modify", "-B", "maxmem=a"]),
502
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 256$'"]),
503
    (True, ["gnt-cluster", "modify", "-B", "minmem=a"]),
504
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 256$'"]),
505
    (False, ["gnt-cluster", "modify", "-B", "maxmem=128,minmem=128"]),
506
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *maxmem: 128$'"]),
507
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *minmem: 128$'"]),
508
    # vcpus
509
    (False, ["gnt-cluster", "modify", "-B", "vcpus=4"]),
510
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 4$'"]),
511
    (True, ["gnt-cluster", "modify", "-B", "vcpus=a"]),
512
    (False, ["gnt-cluster", "modify", "-B", "vcpus=1"]),
513
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *vcpus: 1$'"]),
514
    # auto_balance
515
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=False"]),
516
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: False$'"]),
517
    (True, ["gnt-cluster", "modify", "-B", "auto_balance=1"]),
518
    (False, ["gnt-cluster", "modify", "-B", "auto_balance=True"]),
519
    (False, ["sh", "-c", "gnt-cluster info|grep '^ *auto_balance: True$'"]),
520
    ]:
521
    AssertCommand(cmd, fail=fail)
522

    
523
  # redo the original-requested BE parameters, if any
524
  bep = qa_config.get("backend-parameters", "")
525
  if bep:
526
    AssertCommand(["gnt-cluster", "modify", "-B", bep])
527

    
528

    
529
def _GetClusterIPolicy():
530
  """Return the run-time values of the cluster-level instance policy.
531

532
  @rtype: tuple
533
  @return: (policy, specs), where:
534
      - policy is a dictionary of the policy values, instance specs excluded
535
      - specs is dict of dict, specs[key][par] is a spec value, where key is
536
        "min", "max", or "std"
537

538
  """
539
  info = qa_utils.GetObjectInfo(["gnt-cluster", "info"])
540
  policy = info["Instance policy - limits for instances"]
541
  (ret_policy, ret_specs) = qa_utils.ParseIPolicy(policy)
542

    
543
  # Sanity checks
544
  assert "min" in ret_specs and "std" in ret_specs and "max" in ret_specs
545
  assert len(ret_policy) > 0
546
  return (ret_policy, ret_specs)
547

    
548

    
549
def TestClusterModifyIPolicy():
550
  """gnt-cluster modify --ipolicy-*"""
551
  basecmd = ["gnt-cluster", "modify"]
552
  (old_policy, old_specs) = _GetClusterIPolicy()
553
  for par in ["vcpu-ratio", "spindle-ratio"]:
554
    curr_val = float(old_policy[par])
555
    test_values = [
556
      (True, 1.0),
557
      (True, 1.5),
558
      (True, 2),
559
      (False, "a"),
560
      # Restore the old value
561
      (True, curr_val),
562
      ]
563
    for (good, val) in test_values:
564
      cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
565
      AssertCommand(cmd, fail=not good)
566
      if good:
567
        curr_val = val
568
      # Check the affected parameter
569
      (eff_policy, eff_specs) = _GetClusterIPolicy()
570
      AssertEqual(float(eff_policy[par]), curr_val)
571
      # Check everything else
572
      AssertEqual(eff_specs, old_specs)
573
      for p in eff_policy.keys():
574
        if p == par:
575
          continue
576
        AssertEqual(eff_policy[p], old_policy[p])
577

    
578
  # Disk templates are treated slightly differently
579
  par = "disk-templates"
580
  disp_str = "enabled disk templates"
581
  curr_val = old_policy[disp_str]
582
  test_values = [
583
    (True, constants.DT_PLAIN),
584
    (True, "%s,%s" % (constants.DT_PLAIN, constants.DT_DRBD8)),
585
    (False, "thisisnotadisktemplate"),
586
    (False, ""),
587
    # Restore the old value
588
    (True, curr_val.replace(" ", "")),
589
    ]
590
  for (good, val) in test_values:
591
    cmd = basecmd + ["--ipolicy-%s=%s" % (par, val)]
592
    AssertCommand(cmd, fail=not good)
593
    if good:
594
      curr_val = val
595
    # Check the affected parameter
596
    (eff_policy, eff_specs) = _GetClusterIPolicy()
597
    AssertEqual(eff_policy[disp_str].replace(" ", ""), curr_val)
598
    # Check everything else
599
    AssertEqual(eff_specs, old_specs)
600
    for p in eff_policy.keys():
601
      if p == disp_str:
602
        continue
603
      AssertEqual(eff_policy[p], old_policy[p])
604

    
605

    
606
def TestClusterSetISpecs(new_specs=None, diff_specs=None, fail=False,
607
                         old_values=None):
608
  """Change instance specs.
609

610
  At most one of new_specs or diff_specs can be specified.
611

612
  @type new_specs: dict
613
  @param new_specs: new complete specs, in the same format returned by
614
      L{_GetClusterIPolicy}
615
  @type diff_specs: dict
616
  @param diff_specs: diff_specs[key][par], where key is "min", "max", "std". It
617
      can be an incomplete specifications or an empty dictionary.
618
  @type fail: bool
619
  @param fail: if the change is expected to fail
620
  @type old_values: tuple
621
  @param old_values: (old_policy, old_specs), as returned by
622
      L{_GetClusterIPolicy}
623
  @return: same as L{_GetClusterIPolicy}
624

625
  """
626
  build_cmd = lambda opts: ["gnt-cluster", "modify"] + opts
627
  return qa_utils.TestSetISpecs(
628
    new_specs=new_specs, diff_specs=diff_specs,
629
    get_policy_fn=_GetClusterIPolicy, build_cmd_fn=build_cmd,
630
    fail=fail, old_values=old_values)
631

    
632

    
633
def TestClusterModifyISpecs():
634
  """gnt-cluster modify --specs-*"""
635
  params = ["memory-size", "disk-size", "disk-count", "cpu-count", "nic-count"]
636
  (cur_policy, cur_specs) = _GetClusterIPolicy()
637
  for par in params:
638
    test_values = [
639
      (True, 0, 4, 12),
640
      (True, 4, 4, 12),
641
      (True, 4, 12, 12),
642
      (True, 4, 4, 4),
643
      (False, 4, 0, 12),
644
      (False, 4, 16, 12),
645
      (False, 4, 4, 0),
646
      (False, 12, 4, 4),
647
      (False, 12, 4, 0),
648
      (False, "a", 4, 12),
649
      (False, 0, "a", 12),
650
      (False, 0, 4, "a"),
651
      # This is to restore the old values
652
      (True,
653
       cur_specs["min"][par], cur_specs["std"][par], cur_specs["max"][par])
654
      ]
655
    for (good, mn, st, mx) in test_values:
656
      new_vals = {
657
        "min": {par: mn},
658
        "std": {par: st},
659
        "max": {par: mx}
660
        }
661
      cur_state = (cur_policy, cur_specs)
662
      # We update cur_specs, as we've copied the values to restore already
663
      (cur_policy, cur_specs) = TestClusterSetISpecs(
664
        diff_specs=new_vals, fail=not good, old_values=cur_state)
665

    
666
    # Get the ipolicy command
667
    mnode = qa_config.GetMasterNode()
668
    initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
669
    modcmd = ["gnt-cluster", "modify"]
670
    opts = initcmd.split()
671
    assert opts[0:2] == ["gnt-cluster", "init"]
672
    for k in range(2, len(opts) - 1):
673
      if opts[k].startswith("--ipolicy-"):
674
        assert k + 2 <= len(opts)
675
        modcmd.extend(opts[k:k + 2])
676
    # Re-apply the ipolicy (this should be a no-op)
677
    AssertCommand(modcmd)
678
    new_initcmd = GetCommandOutput(mnode.primary, "gnt-cluster show-ispecs-cmd")
679
    AssertEqual(initcmd, new_initcmd)
680

    
681

    
682
def TestClusterInfo():
683
  """gnt-cluster info"""
684
  AssertCommand(["gnt-cluster", "info"])
685

    
686

    
687
def TestClusterRedistConf():
688
  """gnt-cluster redist-conf"""
689
  AssertCommand(["gnt-cluster", "redist-conf"])
690

    
691

    
692
def TestClusterGetmaster():
693
  """gnt-cluster getmaster"""
694
  AssertCommand(["gnt-cluster", "getmaster"])
695

    
696

    
697
def TestClusterVersion():
698
  """gnt-cluster version"""
699
  AssertCommand(["gnt-cluster", "version"])
700

    
701

    
702
def TestClusterRenewCrypto():
703
  """gnt-cluster renew-crypto"""
704
  master = qa_config.GetMasterNode()
705

    
706
  # Conflicting options
707
  cmd = ["gnt-cluster", "renew-crypto", "--force",
708
         "--new-cluster-certificate", "--new-confd-hmac-key"]
709
  conflicting = [
710
    ["--new-rapi-certificate", "--rapi-certificate=/dev/null"],
711
    ["--new-cluster-domain-secret", "--cluster-domain-secret=/dev/null"],
712
    ]
713
  for i in conflicting:
714
    AssertCommand(cmd + i, fail=True)
715

    
716
  # Invalid RAPI certificate
717
  cmd = ["gnt-cluster", "renew-crypto", "--force",
718
         "--rapi-certificate=/dev/null"]
719
  AssertCommand(cmd, fail=True)
720

    
721
  rapi_cert_backup = qa_utils.BackupFile(master.primary,
722
                                         pathutils.RAPI_CERT_FILE)
723
  try:
724
    # Custom RAPI certificate
725
    fh = tempfile.NamedTemporaryFile()
726

    
727
    # Ensure certificate doesn't cause "gnt-cluster verify" to complain
728
    validity = constants.SSL_CERT_EXPIRATION_WARN * 3
729

    
730
    utils.GenerateSelfSignedSslCert(fh.name, validity=validity)
731

    
732
    tmpcert = qa_utils.UploadFile(master.primary, fh.name)
733
    try:
734
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
735
                     "--rapi-certificate=%s" % tmpcert])
736
    finally:
737
      AssertCommand(["rm", "-f", tmpcert])
738

    
739
    # Custom cluster domain secret
740
    cds_fh = tempfile.NamedTemporaryFile()
741
    cds_fh.write(utils.GenerateSecret())
742
    cds_fh.write("\n")
743
    cds_fh.flush()
744

    
745
    tmpcds = qa_utils.UploadFile(master.primary, cds_fh.name)
746
    try:
747
      AssertCommand(["gnt-cluster", "renew-crypto", "--force",
748
                     "--cluster-domain-secret=%s" % tmpcds])
749
    finally:
750
      AssertCommand(["rm", "-f", tmpcds])
751

    
752
    # Normal case
753
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
754
                   "--new-cluster-certificate", "--new-confd-hmac-key",
755
                   "--new-rapi-certificate", "--new-cluster-domain-secret"])
756

    
757
    # Restore RAPI certificate
758
    AssertCommand(["gnt-cluster", "renew-crypto", "--force",
759
                   "--rapi-certificate=%s" % rapi_cert_backup])
760
  finally:
761
    AssertCommand(["rm", "-f", rapi_cert_backup])
762

    
763

    
764
def TestClusterBurnin():
765
  """Burnin"""
766
  master = qa_config.GetMasterNode()
767

    
768
  options = qa_config.get("options", {})
769
  disk_template = options.get("burnin-disk-template", constants.DT_DRBD8)
770
  parallel = options.get("burnin-in-parallel", False)
771
  check_inst = options.get("burnin-check-instances", False)
772
  do_rename = options.get("burnin-rename", "")
773
  do_reboot = options.get("burnin-reboot", True)
774
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
775

    
776
  # Get as many instances as we need
777
  instances = []
778
  try:
779
    try:
780
      num = qa_config.get("options", {}).get("burnin-instances", 1)
781
      for _ in range(0, num):
782
        instances.append(qa_config.AcquireInstance())
783
    except qa_error.OutOfInstancesError:
784
      print "Not enough instances, continuing anyway."
785

    
786
    if len(instances) < 1:
787
      raise qa_error.Error("Burnin needs at least one instance")
788

    
789
    script = qa_utils.UploadFile(master.primary, "../tools/burnin")
790
    try:
791
      disks = qa_config.GetDiskOptions()
792
      # Run burnin
793
      cmd = [script,
794
             "--os=%s" % qa_config.get("os"),
795
             "--minmem-size=%s" % qa_config.get(constants.BE_MINMEM),
796
             "--maxmem-size=%s" % qa_config.get(constants.BE_MAXMEM),
797
             "--disk-size=%s" % ",".join([d.get("size") for d in disks]),
798
             "--disk-growth=%s" % ",".join([d.get("growth") for d in disks]),
799
             "--disk-template=%s" % disk_template]
800
      if parallel:
801
        cmd.append("--parallel")
802
        cmd.append("--early-release")
803
      if check_inst:
804
        cmd.append("--http-check")
805
      if do_rename:
806
        cmd.append("--rename=%s" % do_rename)
807
      if not do_reboot:
808
        cmd.append("--no-reboot")
809
      else:
810
        cmd.append("--reboot-types=%s" % ",".join(reboot_types))
811
      cmd += [inst.name for inst in instances]
812
      AssertCommand(cmd)
813
    finally:
814
      AssertCommand(["rm", "-f", script])
815

    
816
  finally:
817
    for inst in instances:
818
      inst.Release()
819

    
820

    
821
def TestClusterMasterFailover():
822
  """gnt-cluster master-failover"""
823
  master = qa_config.GetMasterNode()
824
  failovermaster = qa_config.AcquireNode(exclude=master)
825

    
826
  cmd = ["gnt-cluster", "master-failover"]
827
  try:
828
    AssertCommand(cmd, node=failovermaster)
829
    # Back to original master node
830
    AssertCommand(cmd, node=master)
831
  finally:
832
    failovermaster.Release()
833

    
834

    
835
def _NodeQueueDrainFile(node):
836
  """Returns path to queue drain file for a node.
837

838
  """
839
  return qa_utils.MakeNodePath(node, pathutils.JOB_QUEUE_DRAIN_FILE)
840

    
841

    
842
def _AssertDrainFile(node, **kwargs):
843
  """Checks for the queue drain file.
844

845
  """
846
  AssertCommand(["test", "-f", _NodeQueueDrainFile(node)], node=node, **kwargs)
847

    
848

    
849
def TestClusterMasterFailoverWithDrainedQueue():
850
  """gnt-cluster master-failover with drained queue"""
851
  master = qa_config.GetMasterNode()
852
  failovermaster = qa_config.AcquireNode(exclude=master)
853

    
854
  # Ensure queue is not drained
855
  for node in [master, failovermaster]:
856
    _AssertDrainFile(node, fail=True)
857

    
858
  # Drain queue on failover master
859
  AssertCommand(["touch", _NodeQueueDrainFile(failovermaster)],
860
                node=failovermaster)
861

    
862
  cmd = ["gnt-cluster", "master-failover"]
863
  try:
864
    _AssertDrainFile(failovermaster)
865
    AssertCommand(cmd, node=failovermaster)
866
    _AssertDrainFile(master, fail=True)
867
    _AssertDrainFile(failovermaster, fail=True)
868

    
869
    # Back to original master node
870
    AssertCommand(cmd, node=master)
871
  finally:
872
    failovermaster.Release()
873

    
874
  # Ensure queue is not drained
875
  for node in [master, failovermaster]:
876
    _AssertDrainFile(node, fail=True)
877

    
878

    
879
def TestClusterCopyfile():
880
  """gnt-cluster copyfile"""
881
  master = qa_config.GetMasterNode()
882

    
883
  uniqueid = utils.NewUUID()
884

    
885
  # Create temporary file
886
  f = tempfile.NamedTemporaryFile()
887
  f.write(uniqueid)
888
  f.flush()
889
  f.seek(0)
890

    
891
  # Upload file to master node
892
  testname = qa_utils.UploadFile(master.primary, f.name)
893
  try:
894
    # Copy file to all nodes
895
    AssertCommand(["gnt-cluster", "copyfile", testname])
896
    _CheckFileOnAllNodes(testname, uniqueid)
897
  finally:
898
    _RemoveFileFromAllNodes(testname)
899

    
900

    
901
def TestClusterCommand():
902
  """gnt-cluster command"""
903
  uniqueid = utils.NewUUID()
904
  rfile = "/tmp/gnt%s" % utils.NewUUID()
905
  rcmd = utils.ShellQuoteArgs(["echo", "-n", uniqueid])
906
  cmd = utils.ShellQuoteArgs(["gnt-cluster", "command",
907
                              "%s >%s" % (rcmd, rfile)])
908

    
909
  try:
910
    AssertCommand(cmd)
911
    _CheckFileOnAllNodes(rfile, uniqueid)
912
  finally:
913
    _RemoveFileFromAllNodes(rfile)
914

    
915

    
916
def TestClusterDestroy():
917
  """gnt-cluster destroy"""
918
  AssertCommand(["gnt-cluster", "destroy", "--yes-do-it"])
919

    
920

    
921
def TestClusterRepairDiskSizes():
922
  """gnt-cluster repair-disk-sizes"""
923
  AssertCommand(["gnt-cluster", "repair-disk-sizes"])
924

    
925

    
926
def TestSetExclStorCluster(newvalue):
927
  """Set the exclusive_storage node parameter at the cluster level.
928

929
  @type newvalue: bool
930
  @param newvalue: New value of exclusive_storage
931
  @rtype: bool
932
  @return: The old value of exclusive_storage
933

934
  """
935
  es_path = ["Default node parameters", "exclusive_storage"]
936
  oldvalue = _GetClusterField(es_path)
937
  AssertCommand(["gnt-cluster", "modify", "--node-parameters",
938
                 "exclusive_storage=%s" % newvalue])
939
  effvalue = _GetClusterField(es_path)
940
  if effvalue != newvalue:
941
    raise qa_error.Error("exclusive_storage has the wrong value: %s instead"
942
                         " of %s" % (effvalue, newvalue))
943
  qa_config.SetExclusiveStorage(newvalue)
944
  return oldvalue
945

    
946

    
947
def TestExclStorSharedPv(node):
948
  """cluster-verify reports LVs that share the same PV with exclusive_storage.
949

950
  """
951
  vgname = qa_config.get("vg-name", constants.DEFAULT_VG)
952
  lvname1 = _QA_LV_PREFIX + "vol1"
953
  lvname2 = _QA_LV_PREFIX + "vol2"
954
  node_name = node.primary
955
  AssertCommand(["lvcreate", "-L1G", "-n", lvname1, vgname], node=node_name)
956
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODEORPHANLV])
957
  AssertCommand(["lvcreate", "-L1G", "-n", lvname2, vgname], node=node_name)
958
  AssertClusterVerify(fail=True, errors=[constants.CV_ENODELVM,
959
                                         constants.CV_ENODEORPHANLV])
960
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname1])], node=node_name)
961
  AssertCommand(["lvremove", "-f", "/".join([vgname, lvname2])], node=node_name)
962
  AssertClusterVerify()