Statistics
| Branch: | Tag: | Revision:

root / qa / qa_node.py @ 33c730a2

History | View | Annotate | Download (15.2 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Node-related QA tests.
23

24
"""
25

    
26
from ganeti import utils
27
from ganeti import constants
28
from ganeti import query
29
from ganeti import serializer
30

    
31
import qa_config
32
import qa_error
33
import qa_utils
34

    
35
from qa_utils import AssertCommand, AssertEqual
36

    
37

    
38
def _NodeAdd(node, readd=False):
39
  if not readd and node.added:
40
    raise qa_error.Error("Node %s already in cluster" % node.primary)
41
  elif readd and not node.added:
42
    raise qa_error.Error("Node %s not yet in cluster" % node.primary)
43

    
44
  cmd = ["gnt-node", "add", "--no-ssh-key-check"]
45
  if node.secondary:
46
    cmd.append("--secondary-ip=%s" % node.secondary)
47
  if readd:
48
    cmd.append("--readd")
49
  cmd.append(node.primary)
50

    
51
  AssertCommand(cmd)
52

    
53
  if readd:
54
    assert node.added
55
  else:
56
    node.MarkAdded()
57

    
58

    
59
def _NodeRemove(node):
60
  AssertCommand(["gnt-node", "remove", node.primary])
61
  node.MarkRemoved()
62

    
63

    
64
def MakeNodeOffline(node, value):
65
  """gnt-node modify --offline=value"""
66
  # value in ["yes", "no"]
67
  AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
68

    
69

    
70
def TestNodeAddAll():
71
  """Adding all nodes to cluster."""
72
  master = qa_config.GetMasterNode()
73
  for node in qa_config.get("nodes"):
74
    if node != master:
75
      _NodeAdd(node, readd=False)
76

    
77

    
78
def MarkNodeAddedAll():
79
  """Mark all nodes as added.
80

81
  This is useful if we don't create the cluster ourselves (in qa).
82

83
  """
84
  master = qa_config.GetMasterNode()
85
  for node in qa_config.get("nodes"):
86
    if node != master:
87
      node.MarkAdded()
88

    
89

    
90
def TestNodeRemoveAll():
91
  """Removing all nodes from cluster."""
92
  master = qa_config.GetMasterNode()
93
  for node in qa_config.get("nodes"):
94
    if node != master:
95
      _NodeRemove(node)
96

    
97

    
98
def TestNodeReadd(node):
99
  """gnt-node add --readd"""
100
  _NodeAdd(node, readd=True)
101

    
102

    
103
def TestNodeInfo():
104
  """gnt-node info"""
105
  AssertCommand(["gnt-node", "info"])
106

    
107

    
108
def TestNodeVolumes():
109
  """gnt-node volumes"""
110
  AssertCommand(["gnt-node", "volumes"])
111

    
112

    
113
def TestNodeStorage():
114
  """gnt-node storage"""
115
  master = qa_config.GetMasterNode()
116

    
117
  # FIXME: test all storage_types in constants.VALID_STORAGE_TYPES
118
  # as soon as they are implemented.
119
  for storage_type in [constants.ST_FILE, constants.ST_LVM_VG,
120
                       constants.ST_LVM_PV]:
121

    
122
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
123

    
124
    # Skip file storage if not enabled, otherwise QA will fail; we
125
    # just test for basic failure, but otherwise skip the rest of the
126
    # tests
127
    if storage_type == constants.ST_FILE and not constants.ENABLE_FILE_STORAGE:
128
      AssertCommand(cmd, fail=True)
129
      continue
130

    
131
    # Test simple list
132
    AssertCommand(cmd)
133

    
134
    # Test all storage fields
135
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
136
           "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
137
                                    [constants.SF_NODE, constants.SF_TYPE])]
138
    AssertCommand(cmd)
139

    
140
    # Get list of valid storage devices
141
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
142
           "--output=node,name,allocatable", "--separator=|",
143
           "--no-headers"]
144
    output = qa_utils.GetCommandOutput(master.primary,
145
                                       utils.ShellQuoteArgs(cmd))
146

    
147
    # Test with up to two devices
148
    testdevcount = 2
149

    
150
    for line in output.splitlines()[:testdevcount]:
151
      (node_name, st_name, st_allocatable) = line.split("|")
152

    
153
      # Dummy modification without any changes
154
      cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
155
      AssertCommand(cmd)
156

    
157
      # Make sure we end up with the same value as before
158
      if st_allocatable.lower() == "y":
159
        test_allocatable = ["no", "yes"]
160
      else:
161
        test_allocatable = ["yes", "no"]
162

    
163
      fail = (constants.SF_ALLOCATABLE not in
164
              constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
165

    
166
      for i in test_allocatable:
167
        AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
168
                       node_name, storage_type, st_name], fail=fail)
169

    
170
        # Verify list output
171
        cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
172
               "--output=name,allocatable", "--separator=|",
173
               "--no-headers", node_name]
174
        listout = qa_utils.GetCommandOutput(master.primary,
175
                                            utils.ShellQuoteArgs(cmd))
176
        for line in listout.splitlines():
177
          (vfy_name, vfy_allocatable) = line.split("|")
178
          if vfy_name == st_name and not fail:
179
            AssertEqual(vfy_allocatable, i[0].upper())
180
          else:
181
            AssertEqual(vfy_allocatable, st_allocatable)
182

    
183
      # Test repair functionality
184
      fail = (constants.SO_FIX_CONSISTENCY not in
185
              constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
186
      AssertCommand(["gnt-node", "repair-storage", node_name,
187
                     storage_type, st_name], fail=fail)
188

    
189

    
190
def TestNodeFailover(node, node2):
191
  """gnt-node failover"""
192
  if qa_utils.GetNodeInstances(node2, secondaries=False):
193
    raise qa_error.UnusableNodeError("Secondary node has at least one"
194
                                     " primary instance. This test requires"
195
                                     " it to have no primary instances.")
196

    
197
  # Fail over to secondary node
198
  AssertCommand(["gnt-node", "failover", "-f", node.primary])
199

    
200
  # ... and back again.
201
  AssertCommand(["gnt-node", "failover", "-f", node2.primary])
202

    
203

    
204
def TestNodeEvacuate(node, node2):
205
  """gnt-node evacuate"""
206
  node3 = qa_config.AcquireNode(exclude=[node, node2])
207
  try:
208
    if qa_utils.GetNodeInstances(node3, secondaries=True):
209
      raise qa_error.UnusableNodeError("Evacuation node has at least one"
210
                                       " secondary instance. This test requires"
211
                                       " it to have no secondary instances.")
212

    
213
    # Evacuate all secondary instances
214
    AssertCommand(["gnt-node", "evacuate", "-f",
215
                   "--new-secondary=%s" % node3.primary, node2.primary])
216

    
217
    # ... and back again.
218
    AssertCommand(["gnt-node", "evacuate", "-f",
219
                   "--new-secondary=%s" % node2.primary, node3.primary])
220
  finally:
221
    node3.Release()
222

    
223

    
224
def TestNodeModify(node):
225
  """gnt-node modify"""
226
  for flag in ["master-candidate", "drained", "offline"]:
227
    for value in ["yes", "no"]:
228
      AssertCommand(["gnt-node", "modify", "--force",
229
                     "--%s=%s" % (flag, value), node.primary])
230

    
231
  AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
232
                 "--auto-promote", node.primary])
233

    
234
  # Test setting secondary IP address
235
  AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
236
                 node.primary])
237

    
238

    
239
def _CreateOobScriptStructure():
240
  """Create a simple OOB handling script and its structure."""
241
  master = qa_config.GetMasterNode()
242

    
243
  data_path = qa_utils.UploadData(master.primary, "")
244
  verify_path = qa_utils.UploadData(master.primary, "")
245
  exit_code_path = qa_utils.UploadData(master.primary, "")
246

    
247
  oob_script = (("#!/bin/bash\n"
248
                 "echo \"$@\" > %s\n"
249
                 "cat %s\n"
250
                 "exit $(< %s)\n") %
251
                (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
252
                 utils.ShellQuote(exit_code_path)))
253
  oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
254

    
255
  return [oob_path, verify_path, data_path, exit_code_path]
256

    
257

    
258
def _UpdateOobFile(path, data):
259
  """Updates the data file with data."""
260
  master = qa_config.GetMasterNode()
261
  qa_utils.UploadData(master.primary, data, filename=path)
262

    
263

    
264
def _AssertOobCall(verify_path, expected_args):
265
  """Assert the OOB call was performed with expetected args."""
266
  master = qa_config.GetMasterNode()
267

    
268
  verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
269
  output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
270
                                     tty=False)
271

    
272
  AssertEqual(expected_args, output.strip())
273

    
274

    
275
def TestOutOfBand():
276
  """gnt-node power"""
277
  master = qa_config.GetMasterNode()
278

    
279
  node = qa_config.AcquireNode(exclude=master)
280

    
281
  master_name = master.primary
282
  node_name = node.primary
283
  full_node_name = qa_utils.ResolveNodeName(node)
284

    
285
  (oob_path, verify_path,
286
   data_path, exit_code_path) = _CreateOobScriptStructure()
287

    
288
  try:
289
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
290
                   "oob_program=%s" % oob_path])
291

    
292
    # No data, exit 0
293
    _UpdateOobFile(exit_code_path, "0")
294

    
295
    AssertCommand(["gnt-node", "power", "on", node_name])
296
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
297

    
298
    AssertCommand(["gnt-node", "power", "-f", "off", node_name])
299
    _AssertOobCall(verify_path, "power-off %s" % full_node_name)
300

    
301
    # Power off on master without options should fail
302
    AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
303
    # With force master it should still fail
304
    AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
305
                   master_name],
306
                  fail=True)
307

    
308
    # Verify we can't transform back to online when not yet powered on
309
    AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
310
                  fail=True)
311
    # Now reset state
312
    AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
313
                   node_name])
314

    
315
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
316
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
317

    
318
    # Those commands should fail as they expect output which isn't provided yet
319
    # But they should have called the oob helper nevermind
320
    AssertCommand(["gnt-node", "power", "status", node_name],
321
                  fail=True)
322
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
323

    
324
    AssertCommand(["gnt-node", "health", node_name],
325
                  fail=True)
326
    _AssertOobCall(verify_path, "health %s" % full_node_name)
327

    
328
    AssertCommand(["gnt-node", "health"], fail=True)
329

    
330
    # Correct Data, exit 0
331
    _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
332

    
333
    AssertCommand(["gnt-node", "power", "status", node_name])
334
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
335

    
336
    _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
337
                                                   ["disk0", "CRITICAL"]]))
338

    
339
    AssertCommand(["gnt-node", "health", node_name])
340
    _AssertOobCall(verify_path, "health %s" % full_node_name)
341

    
342
    AssertCommand(["gnt-node", "health"])
343

    
344
    # Those commands should fail as they expect no data regardless of exit 0
345
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
346
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
347

    
348
    try:
349
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
350
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
351
    finally:
352
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
353

    
354
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
355
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
356

    
357
    # Data, exit 1 (all should fail)
358
    _UpdateOobFile(exit_code_path, "1")
359

    
360
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
361
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
362

    
363
    try:
364
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
365
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
366
    finally:
367
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
368

    
369
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
370
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
371

    
372
    AssertCommand(["gnt-node", "power", "status", node_name],
373
                  fail=True)
374
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
375

    
376
    AssertCommand(["gnt-node", "health", node_name],
377
                  fail=True)
378
    _AssertOobCall(verify_path, "health %s" % full_node_name)
379

    
380
    AssertCommand(["gnt-node", "health"], fail=True)
381

    
382
    # No data, exit 1 (all should fail)
383
    _UpdateOobFile(data_path, "")
384
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
385
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
386

    
387
    try:
388
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
389
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
390
    finally:
391
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
392

    
393
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
394
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
395

    
396
    AssertCommand(["gnt-node", "power", "status", node_name],
397
                  fail=True)
398
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
399

    
400
    AssertCommand(["gnt-node", "health", node_name],
401
                  fail=True)
402
    _AssertOobCall(verify_path, "health %s" % full_node_name)
403

    
404
    AssertCommand(["gnt-node", "health"], fail=True)
405

    
406
    # Different OOB script for node
407
    verify_path2 = qa_utils.UploadData(master.primary, "")
408
    oob_script = ("#!/bin/sh\n"
409
                  "echo \"$@\" > %s\n") % verify_path2
410
    oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
411

    
412
    try:
413
      AssertCommand(["gnt-node", "modify", "--node-parameters",
414
                     "oob_program=%s" % oob_path2, node_name])
415
      AssertCommand(["gnt-node", "power", "on", node_name])
416
      _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
417
    finally:
418
      AssertCommand(["gnt-node", "modify", "--node-parameters",
419
                     "oob_program=default", node_name])
420
      AssertCommand(["rm", "-f", oob_path2, verify_path2])
421
  finally:
422
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
423
                   "oob_program="])
424
    AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
425
                   exit_code_path])
426

    
427

    
428
def TestNodeList():
429
  """gnt-node list"""
430
  qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
431

    
432

    
433
def TestNodeListFields():
434
  """gnt-node list-fields"""
435
  qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
436

    
437

    
438
def TestNodeListDrbd(node):
439
  """gnt-node list-drbd"""
440
  AssertCommand(["gnt-node", "list-drbd", node.primary])
441

    
442

    
443
def _BuildSetESCmd(action, value, node_name):
444
  cmd = ["gnt-node"]
445
  if action == "add":
446
    cmd.extend(["add", "--readd"])
447
  else:
448
    cmd.append("modify")
449
  cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
450
  return cmd
451

    
452

    
453
def TestExclStorSingleNode(node):
454
  """gnt-node add/modify cannot change the exclusive_storage flag.
455

456
  """
457
  for action in ["add", "modify"]:
458
    for value in (True, False, "default"):
459
      AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)