Statistics
| Branch: | Tag: | Revision:

root / qa / qa_node.py @ 250a9404

History | View | Annotate | Download (14.9 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Node-related QA tests.
23

24
"""
25

    
26
from ganeti import utils
27
from ganeti import constants
28
from ganeti import query
29
from ganeti import serializer
30

    
31
import qa_config
32
import qa_error
33
import qa_utils
34

    
35
from qa_utils import AssertCommand, AssertEqual
36

    
37

    
38
def _NodeAdd(node, readd=False):
39
  if not readd and node.get("_added", False):
40
    raise qa_error.Error("Node %s already in cluster" % node["primary"])
41
  elif readd and not node.get("_added", False):
42
    raise qa_error.Error("Node %s not yet in cluster" % node["primary"])
43

    
44
  cmd = ["gnt-node", "add", "--no-ssh-key-check"]
45
  if node.get("secondary", None):
46
    cmd.append("--secondary-ip=%s" % node["secondary"])
47
  if readd:
48
    cmd.append("--readd")
49
  cmd.append(node["primary"])
50

    
51
  AssertCommand(cmd)
52

    
53
  node["_added"] = True
54

    
55

    
56
def _NodeRemove(node):
57
  AssertCommand(["gnt-node", "remove", node["primary"]])
58
  node["_added"] = False
59

    
60

    
61
def MakeNodeOffline(node, value):
62
  """gnt-node modify --offline=value"""
63
  # value in ["yes", "no"]
64
  AssertCommand(["gnt-node", "modify", "--offline", value, node["primary"]])
65

    
66

    
67
def TestNodeAddAll():
68
  """Adding all nodes to cluster."""
69
  master = qa_config.GetMasterNode()
70
  for node in qa_config.get("nodes"):
71
    if node != master:
72
      _NodeAdd(node, readd=False)
73

    
74

    
75
def MarkNodeAddedAll():
76
  """Mark all nodes as added.
77

78
  This is useful if we don't create the cluster ourselves (in qa).
79

80
  """
81
  master = qa_config.GetMasterNode()
82
  for node in qa_config.get("nodes"):
83
    if node != master:
84
      node["_added"] = True
85

    
86

    
87
def TestNodeRemoveAll():
88
  """Removing all nodes from cluster."""
89
  master = qa_config.GetMasterNode()
90
  for node in qa_config.get("nodes"):
91
    if node != master:
92
      _NodeRemove(node)
93

    
94

    
95
def TestNodeReadd(node):
96
  """gnt-node add --readd"""
97
  _NodeAdd(node, readd=True)
98

    
99

    
100
def TestNodeInfo():
101
  """gnt-node info"""
102
  AssertCommand(["gnt-node", "info"])
103

    
104

    
105
def TestNodeVolumes():
106
  """gnt-node volumes"""
107
  AssertCommand(["gnt-node", "volumes"])
108

    
109

    
110
def TestNodeStorage():
111
  """gnt-node storage"""
112
  master = qa_config.GetMasterNode()
113

    
114
  for storage_type in constants.VALID_STORAGE_TYPES:
115
    # Test simple list
116
    AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
117

    
118
    # Test all storage fields
119
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
120
           "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
121
                                    [constants.SF_NODE, constants.SF_TYPE])]
122
    AssertCommand(cmd)
123

    
124
    # Get list of valid storage devices
125
    cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
126
           "--output=node,name,allocatable", "--separator=|",
127
           "--no-headers"]
128
    output = qa_utils.GetCommandOutput(master["primary"],
129
                                       utils.ShellQuoteArgs(cmd))
130

    
131
    # Test with up to two devices
132
    testdevcount = 2
133

    
134
    for line in output.splitlines()[:testdevcount]:
135
      (node_name, st_name, st_allocatable) = line.split("|")
136

    
137
      # Dummy modification without any changes
138
      cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
139
      AssertCommand(cmd)
140

    
141
      # Make sure we end up with the same value as before
142
      if st_allocatable.lower() == "y":
143
        test_allocatable = ["no", "yes"]
144
      else:
145
        test_allocatable = ["yes", "no"]
146

    
147
      fail = (constants.SF_ALLOCATABLE not in
148
              constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
149

    
150
      for i in test_allocatable:
151
        AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
152
                       node_name, storage_type, st_name], fail=fail)
153

    
154
        # Verify list output
155
        cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
156
               "--output=name,allocatable", "--separator=|",
157
               "--no-headers", node_name]
158
        listout = qa_utils.GetCommandOutput(master["primary"],
159
                                            utils.ShellQuoteArgs(cmd))
160
        for line in listout.splitlines():
161
          (vfy_name, vfy_allocatable) = line.split("|")
162
          if vfy_name == st_name and not fail:
163
            AssertEqual(vfy_allocatable, i[0].upper())
164
          else:
165
            AssertEqual(vfy_allocatable, st_allocatable)
166

    
167
      # Test repair functionality
168
      fail = (constants.SO_FIX_CONSISTENCY not in
169
              constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
170
      AssertCommand(["gnt-node", "repair-storage", node_name,
171
                     storage_type, st_name], fail=fail)
172

    
173

    
174
def TestNodeFailover(node, node2):
175
  """gnt-node failover"""
176
  if qa_utils.GetNodeInstances(node2, secondaries=False):
177
    raise qa_error.UnusableNodeError("Secondary node has at least one"
178
                                     " primary instance. This test requires"
179
                                     " it to have no primary instances.")
180

    
181
  # Fail over to secondary node
182
  AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
183

    
184
  # ... and back again.
185
  AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
186

    
187

    
188
def TestNodeEvacuate(node, node2):
189
  """gnt-node evacuate"""
190
  node3 = qa_config.AcquireNode(exclude=[node, node2])
191
  try:
192
    if qa_utils.GetNodeInstances(node3, secondaries=True):
193
      raise qa_error.UnusableNodeError("Evacuation node has at least one"
194
                                       " secondary instance. This test requires"
195
                                       " it to have no secondary instances.")
196

    
197
    # Evacuate all secondary instances
198
    AssertCommand(["gnt-node", "evacuate", "-f",
199
                   "--new-secondary=%s" % node3["primary"], node2["primary"]])
200

    
201
    # ... and back again.
202
    AssertCommand(["gnt-node", "evacuate", "-f",
203
                   "--new-secondary=%s" % node2["primary"], node3["primary"]])
204
  finally:
205
    qa_config.ReleaseNode(node3)
206

    
207

    
208
def TestNodeModify(node):
209
  """gnt-node modify"""
210
  for flag in ["master-candidate", "drained", "offline"]:
211
    for value in ["yes", "no"]:
212
      AssertCommand(["gnt-node", "modify", "--force",
213
                     "--%s=%s" % (flag, value), node["primary"]])
214

    
215
  AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
216
                 "--auto-promote", node["primary"]])
217

    
218
  # Test setting secondary IP address
219
  AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node["secondary"],
220
                 node["primary"]])
221

    
222

    
223
def _CreateOobScriptStructure():
224
  """Create a simple OOB handling script and its structure."""
225
  master = qa_config.GetMasterNode()
226

    
227
  data_path = qa_utils.UploadData(master["primary"], "")
228
  verify_path = qa_utils.UploadData(master["primary"], "")
229
  exit_code_path = qa_utils.UploadData(master["primary"], "")
230

    
231
  oob_script = (("#!/bin/bash\n"
232
                 "echo \"$@\" > %s\n"
233
                 "cat %s\n"
234
                 "exit $(< %s)\n") %
235
                (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
236
                 utils.ShellQuote(exit_code_path)))
237
  oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
238

    
239
  return [oob_path, verify_path, data_path, exit_code_path]
240

    
241

    
242
def _UpdateOobFile(path, data):
243
  """Updates the data file with data."""
244
  master = qa_config.GetMasterNode()
245
  qa_utils.UploadData(master["primary"], data, filename=path)
246

    
247

    
248
def _AssertOobCall(verify_path, expected_args):
249
  """Assert the OOB call was performed with expetected args."""
250
  master = qa_config.GetMasterNode()
251

    
252
  verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
253
  output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd,
254
                                     tty=False)
255

    
256
  AssertEqual(expected_args, output.strip())
257

    
258

    
259
def TestOutOfBand():
260
  """gnt-node power"""
261
  master = qa_config.GetMasterNode()
262

    
263
  node = qa_config.AcquireNode(exclude=master)
264

    
265
  master_name = master["primary"]
266
  node_name = node["primary"]
267
  full_node_name = qa_utils.ResolveNodeName(node)
268

    
269
  (oob_path, verify_path,
270
   data_path, exit_code_path) = _CreateOobScriptStructure()
271

    
272
  try:
273
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
274
                   "oob_program=%s" % oob_path])
275

    
276
    # No data, exit 0
277
    _UpdateOobFile(exit_code_path, "0")
278

    
279
    AssertCommand(["gnt-node", "power", "on", node_name])
280
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
281

    
282
    AssertCommand(["gnt-node", "power", "-f", "off", node_name])
283
    _AssertOobCall(verify_path, "power-off %s" % full_node_name)
284

    
285
    # Power off on master without options should fail
286
    AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
287
    # With force master it should still fail
288
    AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
289
                   master_name],
290
                  fail=True)
291

    
292
    # Verify we can't transform back to online when not yet powered on
293
    AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
294
                  fail=True)
295
    # Now reset state
296
    AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
297
                   node_name])
298

    
299
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
300
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
301

    
302
    # Those commands should fail as they expect output which isn't provided yet
303
    # But they should have called the oob helper nevermind
304
    AssertCommand(["gnt-node", "power", "status", node_name],
305
                  fail=True)
306
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
307

    
308
    AssertCommand(["gnt-node", "health", node_name],
309
                  fail=True)
310
    _AssertOobCall(verify_path, "health %s" % full_node_name)
311

    
312
    AssertCommand(["gnt-node", "health"], fail=True)
313

    
314
    # Correct Data, exit 0
315
    _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
316

    
317
    AssertCommand(["gnt-node", "power", "status", node_name])
318
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
319

    
320
    _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
321
                                                   ["disk0", "CRITICAL"]]))
322

    
323
    AssertCommand(["gnt-node", "health", node_name])
324
    _AssertOobCall(verify_path, "health %s" % full_node_name)
325

    
326
    AssertCommand(["gnt-node", "health"])
327

    
328
    # Those commands should fail as they expect no data regardless of exit 0
329
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
330
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
331

    
332
    try:
333
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
334
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
335
    finally:
336
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
337

    
338
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
339
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
340

    
341
    # Data, exit 1 (all should fail)
342
    _UpdateOobFile(exit_code_path, "1")
343

    
344
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
345
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
346

    
347
    try:
348
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
349
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
350
    finally:
351
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
352

    
353
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
354
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
355

    
356
    AssertCommand(["gnt-node", "power", "status", node_name],
357
                  fail=True)
358
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
359

    
360
    AssertCommand(["gnt-node", "health", node_name],
361
                  fail=True)
362
    _AssertOobCall(verify_path, "health %s" % full_node_name)
363

    
364
    AssertCommand(["gnt-node", "health"], fail=True)
365

    
366
    # No data, exit 1 (all should fail)
367
    _UpdateOobFile(data_path, "")
368
    AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
369
    _AssertOobCall(verify_path, "power-on %s" % full_node_name)
370

    
371
    try:
372
      AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
373
      _AssertOobCall(verify_path, "power-off %s" % full_node_name)
374
    finally:
375
      AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
376

    
377
    AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
378
    _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
379

    
380
    AssertCommand(["gnt-node", "power", "status", node_name],
381
                  fail=True)
382
    _AssertOobCall(verify_path, "power-status %s" % full_node_name)
383

    
384
    AssertCommand(["gnt-node", "health", node_name],
385
                  fail=True)
386
    _AssertOobCall(verify_path, "health %s" % full_node_name)
387

    
388
    AssertCommand(["gnt-node", "health"], fail=True)
389

    
390
    # Different OOB script for node
391
    verify_path2 = qa_utils.UploadData(master["primary"], "")
392
    oob_script = ("#!/bin/sh\n"
393
                  "echo \"$@\" > %s\n") % verify_path2
394
    oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
395

    
396
    try:
397
      AssertCommand(["gnt-node", "modify", "--node-parameters",
398
                     "oob_program=%s" % oob_path2, node_name])
399
      AssertCommand(["gnt-node", "power", "on", node_name])
400
      _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
401
    finally:
402
      AssertCommand(["gnt-node", "modify", "--node-parameters",
403
                     "oob_program=default", node_name])
404
      AssertCommand(["rm", "-f", oob_path2, verify_path2])
405
  finally:
406
    AssertCommand(["gnt-cluster", "modify", "--node-parameters",
407
                   "oob_program="])
408
    AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
409
                   exit_code_path])
410

    
411

    
412
def TestNodeList():
413
  """gnt-node list"""
414
  qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
415

    
416

    
417
def TestNodeListFields():
418
  """gnt-node list-fields"""
419
  qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
420

    
421

    
422
def TestNodeListDrbd(node):
423
  """gnt-node list-drbd"""
424
  AssertCommand(["gnt-node", "list-drbd", node["primary"]])
425

    
426

    
427
def _BuildSetESCmd(action, value, node_name):
428
  cmd = ["gnt-node"]
429
  if action == "add":
430
    cmd.extend(["add", "--readd"])
431
  else:
432
    cmd.append("modify")
433
  cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
434
  return cmd
435

    
436

    
437
def TestExclStorSingleNode(node):
438
  """gnt-node add/modify cannot change the exclusive_storage flag.
439

440
  """
441
  for action in ["add", "modify"]:
442
    for value in (True, False, "default"):
443
      AssertCommand(_BuildSetESCmd(action, value, node["primary"]), fail=True)