4 # Copyright (C) 2007, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Node-related QA tests.
26 from ganeti import utils
27 from ganeti import constants
28 from ganeti import query
29 from ganeti import serializer
35 from qa_utils import AssertCommand, AssertEqual
38 def _NodeAdd(node, readd=False):
39 if not readd and node.get("_added", False):
40 raise qa_error.Error("Node %s already in cluster" % node["primary"])
41 elif readd and not node.get("_added", False):
42 raise qa_error.Error("Node %s not yet in cluster" % node["primary"])
44 cmd = ["gnt-node", "add", "--no-ssh-key-check"]
45 if node.get("secondary", None):
46 cmd.append("--secondary-ip=%s" % node["secondary"])
49 cmd.append(node["primary"])
56 def _NodeRemove(node):
57 AssertCommand(["gnt-node", "remove", node["primary"]])
58 node["_added"] = False
61 def MakeNodeOffline(node, value):
62 """gnt-node modify --offline=value"""
63 # value in ["yes", "no"]
64 AssertCommand(["gnt-node", "modify", "--offline", value, node["primary"]])
68 """Adding all nodes to cluster."""
69 master = qa_config.GetMasterNode()
70 for node in qa_config.get("nodes"):
72 _NodeAdd(node, readd=False)
75 def MarkNodeAddedAll():
76 """Mark all nodes as added.
78 This is useful if we don't create the cluster ourselves (in qa).
81 master = qa_config.GetMasterNode()
82 for node in qa_config.get("nodes"):
87 def TestNodeRemoveAll():
88 """Removing all nodes from cluster."""
89 master = qa_config.GetMasterNode()
90 for node in qa_config.get("nodes"):
95 def TestNodeReadd(node):
96 """gnt-node add --readd"""
97 _NodeAdd(node, readd=True)
102 AssertCommand(["gnt-node", "info"])
105 def TestNodeVolumes():
106 """gnt-node volumes"""
107 AssertCommand(["gnt-node", "volumes"])
110 def TestNodeStorage():
111 """gnt-node storage"""
112 master = qa_config.GetMasterNode()
114 for storage_type in constants.VALID_STORAGE_TYPES:
116 AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
118 # Test all storage fields
119 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
120 "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
121 [constants.SF_NODE, constants.SF_TYPE])]
124 # Get list of valid storage devices
125 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
126 "--output=node,name,allocatable", "--separator=|",
128 output = qa_utils.GetCommandOutput(master["primary"],
129 utils.ShellQuoteArgs(cmd))
131 # Test with up to two devices
134 for line in output.splitlines()[:testdevcount]:
135 (node_name, st_name, st_allocatable) = line.split("|")
137 # Dummy modification without any changes
138 cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name]
141 # Make sure we end up with the same value as before
142 if st_allocatable.lower() == "y":
143 test_allocatable = ["no", "yes"]
145 test_allocatable = ["yes", "no"]
147 fail = (constants.SF_ALLOCATABLE not in
148 constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, []))
150 for i in test_allocatable:
151 AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
152 node_name, storage_type, st_name], fail=fail)
155 cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
156 "--output=name,allocatable", "--separator=|",
157 "--no-headers", node_name]
158 listout = qa_utils.GetCommandOutput(master["primary"],
159 utils.ShellQuoteArgs(cmd))
160 for line in listout.splitlines():
161 (vfy_name, vfy_allocatable) = line.split("|")
162 if vfy_name == st_name and not fail:
163 AssertEqual(vfy_allocatable, i[0].upper())
165 AssertEqual(vfy_allocatable, st_allocatable)
167 # Test repair functionality
168 fail = (constants.SO_FIX_CONSISTENCY not in
169 constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
170 AssertCommand(["gnt-node", "repair-storage", node_name,
171 storage_type, st_name], fail=fail)
174 def TestNodeFailover(node, node2):
175 """gnt-node failover"""
176 if qa_utils.GetNodeInstances(node2, secondaries=False):
177 raise qa_error.UnusableNodeError("Secondary node has at least one"
178 " primary instance. This test requires"
179 " it to have no primary instances.")
181 # Fail over to secondary node
182 AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
184 # ... and back again.
185 AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
188 def TestNodeEvacuate(node, node2):
189 """gnt-node evacuate"""
190 node3 = qa_config.AcquireNode(exclude=[node, node2])
192 if qa_utils.GetNodeInstances(node3, secondaries=True):
193 raise qa_error.UnusableNodeError("Evacuation node has at least one"
194 " secondary instance. This test requires"
195 " it to have no secondary instances.")
197 # Evacuate all secondary instances
198 AssertCommand(["gnt-node", "evacuate", "-f",
199 "--new-secondary=%s" % node3["primary"], node2["primary"]])
201 # ... and back again.
202 AssertCommand(["gnt-node", "evacuate", "-f",
203 "--new-secondary=%s" % node2["primary"], node3["primary"]])
205 qa_config.ReleaseNode(node3)
208 def TestNodeModify(node):
209 """gnt-node modify"""
210 for flag in ["master-candidate", "drained", "offline"]:
211 for value in ["yes", "no"]:
212 AssertCommand(["gnt-node", "modify", "--force",
213 "--%s=%s" % (flag, value), node["primary"]])
215 AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
216 "--auto-promote", node["primary"]])
218 # Test setting secondary IP address
219 AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node["secondary"],
223 def _CreateOobScriptStructure():
224 """Create a simple OOB handling script and its structure."""
225 master = qa_config.GetMasterNode()
227 data_path = qa_utils.UploadData(master["primary"], "")
228 verify_path = qa_utils.UploadData(master["primary"], "")
229 exit_code_path = qa_utils.UploadData(master["primary"], "")
231 oob_script = (("#!/bin/bash\n"
235 (utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
236 utils.ShellQuote(exit_code_path)))
237 oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
239 return [oob_path, verify_path, data_path, exit_code_path]
242 def _UpdateOobFile(path, data):
243 """Updates the data file with data."""
244 master = qa_config.GetMasterNode()
245 qa_utils.UploadData(master["primary"], data, filename=path)
248 def _AssertOobCall(verify_path, expected_args):
249 """Assert the OOB call was performed with expetected args."""
250 master = qa_config.GetMasterNode()
252 verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
253 output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd,
256 AssertEqual(expected_args, output.strip())
261 master = qa_config.GetMasterNode()
263 node = qa_config.AcquireNode(exclude=master)
265 master_name = master["primary"]
266 node_name = node["primary"]
267 full_node_name = qa_utils.ResolveNodeName(node)
269 (oob_path, verify_path,
270 data_path, exit_code_path) = _CreateOobScriptStructure()
273 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
274 "oob_program=%s" % oob_path])
277 _UpdateOobFile(exit_code_path, "0")
279 AssertCommand(["gnt-node", "power", "on", node_name])
280 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
282 AssertCommand(["gnt-node", "power", "-f", "off", node_name])
283 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
285 # Power off on master without options should fail
286 AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
287 # With force master it should still fail
288 AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
292 # Verify we can't transform back to online when not yet powered on
293 AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
296 AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
299 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
300 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
302 # Those commands should fail as they expect output which isn't provided yet
303 # But they should have called the oob helper nevermind
304 AssertCommand(["gnt-node", "power", "status", node_name],
306 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
308 AssertCommand(["gnt-node", "health", node_name],
310 _AssertOobCall(verify_path, "health %s" % full_node_name)
312 AssertCommand(["gnt-node", "health"], fail=True)
314 # Correct Data, exit 0
315 _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
317 AssertCommand(["gnt-node", "power", "status", node_name])
318 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
320 _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
321 ["disk0", "CRITICAL"]]))
323 AssertCommand(["gnt-node", "health", node_name])
324 _AssertOobCall(verify_path, "health %s" % full_node_name)
326 AssertCommand(["gnt-node", "health"])
328 # Those commands should fail as they expect no data regardless of exit 0
329 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
330 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
333 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
334 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
336 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
338 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
339 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
341 # Data, exit 1 (all should fail)
342 _UpdateOobFile(exit_code_path, "1")
344 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
345 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
348 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
349 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
351 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
353 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
354 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
356 AssertCommand(["gnt-node", "power", "status", node_name],
358 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
360 AssertCommand(["gnt-node", "health", node_name],
362 _AssertOobCall(verify_path, "health %s" % full_node_name)
364 AssertCommand(["gnt-node", "health"], fail=True)
366 # No data, exit 1 (all should fail)
367 _UpdateOobFile(data_path, "")
368 AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
369 _AssertOobCall(verify_path, "power-on %s" % full_node_name)
372 AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
373 _AssertOobCall(verify_path, "power-off %s" % full_node_name)
375 AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
377 AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
378 _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
380 AssertCommand(["gnt-node", "power", "status", node_name],
382 _AssertOobCall(verify_path, "power-status %s" % full_node_name)
384 AssertCommand(["gnt-node", "health", node_name],
386 _AssertOobCall(verify_path, "health %s" % full_node_name)
388 AssertCommand(["gnt-node", "health"], fail=True)
390 # Different OOB script for node
391 verify_path2 = qa_utils.UploadData(master["primary"], "")
392 oob_script = ("#!/bin/sh\n"
393 "echo \"$@\" > %s\n") % verify_path2
394 oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
397 AssertCommand(["gnt-node", "modify", "--node-parameters",
398 "oob_program=%s" % oob_path2, node_name])
399 AssertCommand(["gnt-node", "power", "on", node_name])
400 _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
402 AssertCommand(["gnt-node", "modify", "--node-parameters",
403 "oob_program=default", node_name])
404 AssertCommand(["rm", "-f", oob_path2, verify_path2])
406 AssertCommand(["gnt-cluster", "modify", "--node-parameters",
408 AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
414 qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys())
417 def TestNodeListFields():
418 """gnt-node list-fields"""
419 qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
422 def TestNodeListDrbd(node):
423 """gnt-node list-drbd"""
424 AssertCommand(["gnt-node", "list-drbd", node["primary"]])