X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/d7e49c13e9c886e2f37af7a9b2c41729f91dc807..0188611b9cf332ff83f95e0bdbdcbd74c946465c:/qa/qa_node.py?ds=sidebyside diff --git a/qa/qa_node.py b/qa/qa_node.py index 968b7f4..647af19 100644 --- a/qa/qa_node.py +++ b/qa/qa_node.py @@ -1,4 +1,7 @@ -# Copyright (C) 2007 Google Inc. +# +# + +# Copyright (C) 2007, 2011, 2012 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,113 +19,406 @@ # 02110-1301, USA. +"""Node-related QA tests. + +""" + from ganeti import utils +from ganeti import constants +from ganeti import query +from ganeti import serializer import qa_config import qa_error import qa_utils -from qa_utils import AssertEqual, StartSSH +from qa_utils import AssertCommand, AssertEqual -def _NodeAdd(node): - master = qa_config.GetMasterNode() +def _NodeAdd(node, readd=False): + if not readd and node.get("_added", False): + raise qa_error.Error("Node %s already in cluster" % node["primary"]) + elif readd and not node.get("_added", False): + raise qa_error.Error("Node %s not yet in cluster" % node["primary"]) - if node.get('_added', False): - raise qa_error.Error("Node %s already in cluster" % node['primary']) + cmd = ["gnt-node", "add", "--no-ssh-key-check"] + if node.get("secondary", None): + cmd.append("--secondary-ip=%s" % node["secondary"]) + if readd: + cmd.append("--readd") + cmd.append(node["primary"]) - cmd = ['gnt-node', 'add'] - if node.get('secondary', None): - cmd.append('--secondary-ip=%s' % node['secondary']) - cmd.append(node['primary']) - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) + AssertCommand(cmd) - node['_added'] = True + node["_added"] = True def _NodeRemove(node): - master = qa_config.GetMasterNode() + AssertCommand(["gnt-node", "remove", node["primary"]]) + node["_added"] = False - cmd = ['gnt-node', 'remove', node['primary']] - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) - node['_added'] = False + +def MakeNodeOffline(node, value): + """gnt-node modify --offline=value""" + # value in ["yes", "no"] + AssertCommand(["gnt-node", "modify", "--offline", value, node["primary"]]) def TestNodeAddAll(): """Adding all nodes to cluster.""" master = qa_config.GetMasterNode() - for node in qa_config.get('nodes'): + for node in qa_config.get("nodes"): if node != master: - _NodeAdd(node) + _NodeAdd(node, readd=False) + + +def MarkNodeAddedAll(): + """Mark all nodes as added. + + This is useful if we don't create the cluster ourselves (in qa). + + """ + master = qa_config.GetMasterNode() + for node in qa_config.get("nodes"): + if node != master: + node["_added"] = True def TestNodeRemoveAll(): """Removing all nodes from cluster.""" master = qa_config.GetMasterNode() - for node in qa_config.get('nodes'): + for node in qa_config.get("nodes"): if node != master: _NodeRemove(node) +def TestNodeReadd(node): + """gnt-node add --readd""" + _NodeAdd(node, readd=True) + + def TestNodeInfo(): """gnt-node info""" - master = qa_config.GetMasterNode() - - cmd = ['gnt-node', 'info'] - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) + AssertCommand(["gnt-node", "info"]) def TestNodeVolumes(): """gnt-node volumes""" + AssertCommand(["gnt-node", "volumes"]) + + +def TestNodeStorage(): + """gnt-node storage""" master = qa_config.GetMasterNode() - cmd = ['gnt-node', 'volumes'] - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) + for storage_type in constants.VALID_STORAGE_TYPES: + # Test simple list + AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type]) + + # Test all storage fields + cmd = ["gnt-node", "list-storage", "--storage-type", storage_type, + "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) + + [constants.SF_NODE, constants.SF_TYPE])] + AssertCommand(cmd) + + # Get list of valid storage devices + cmd = ["gnt-node", "list-storage", "--storage-type", storage_type, + "--output=node,name,allocatable", "--separator=|", + "--no-headers"] + output = qa_utils.GetCommandOutput(master["primary"], + utils.ShellQuoteArgs(cmd)) + + # Test with up to two devices + testdevcount = 2 + + for line in output.splitlines()[:testdevcount]: + (node_name, st_name, st_allocatable) = line.split("|") + + # Dummy modification without any changes + cmd = ["gnt-node", "modify-storage", node_name, storage_type, st_name] + AssertCommand(cmd) + + # Make sure we end up with the same value as before + if st_allocatable.lower() == "y": + test_allocatable = ["no", "yes"] + else: + test_allocatable = ["yes", "no"] + + fail = (constants.SF_ALLOCATABLE not in + constants.MODIFIABLE_STORAGE_FIELDS.get(storage_type, [])) + + for i in test_allocatable: + AssertCommand(["gnt-node", "modify-storage", "--allocatable", i, + node_name, storage_type, st_name], fail=fail) + + # Verify list output + cmd = ["gnt-node", "list-storage", "--storage-type", storage_type, + "--output=name,allocatable", "--separator=|", + "--no-headers", node_name] + listout = qa_utils.GetCommandOutput(master["primary"], + utils.ShellQuoteArgs(cmd)) + for line in listout.splitlines(): + (vfy_name, vfy_allocatable) = line.split("|") + if vfy_name == st_name and not fail: + AssertEqual(vfy_allocatable, i[0].upper()) + else: + AssertEqual(vfy_allocatable, st_allocatable) + + # Test repair functionality + fail = (constants.SO_FIX_CONSISTENCY not in + constants.VALID_STORAGE_OPERATIONS.get(storage_type, [])) + AssertCommand(["gnt-node", "repair-storage", node_name, + storage_type, st_name], fail=fail) def TestNodeFailover(node, node2): """gnt-node failover""" - master = qa_config.GetMasterNode() - if qa_utils.GetNodeInstances(node2, secondaries=False): - raise qa_errors.UnusableNodeError("Secondary node has at least one " - "primary instance. This test requires " - "it to have no primary instances.") + raise qa_error.UnusableNodeError("Secondary node has at least one" + " primary instance. This test requires" + " it to have no primary instances.") # Fail over to secondary node - cmd = ['gnt-node', 'failover', '-f', node['primary']] - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) + AssertCommand(["gnt-node", "failover", "-f", node["primary"]]) # ... and back again. - cmd = ['gnt-node', 'failover', '-f', node2['primary']] - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) + AssertCommand(["gnt-node", "failover", "-f", node2["primary"]]) def TestNodeEvacuate(node, node2): """gnt-node evacuate""" - master = qa_config.GetMasterNode() - node3 = qa_config.AcquireNode(exclude=[node, node2]) try: if qa_utils.GetNodeInstances(node3, secondaries=True): - raise qa_errors.UnusableNodeError("Evacuation node has at least one " - "secondary instance. This test requires " - "it to have no secondary instances.") + raise qa_error.UnusableNodeError("Evacuation node has at least one" + " secondary instance. This test requires" + " it to have no secondary instances.") # Evacuate all secondary instances - cmd = ['gnt-node', 'evacuate', '-f', node2['primary'], node3['primary']] - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) + AssertCommand(["gnt-node", "evacuate", "-f", + "--new-secondary=%s" % node3["primary"], node2["primary"]]) # ... and back again. - cmd = ['gnt-node', 'evacuate', '-f', node3['primary'], node2['primary']] - AssertEqual(StartSSH(master['primary'], - utils.ShellQuoteArgs(cmd)).wait(), 0) + AssertCommand(["gnt-node", "evacuate", "-f", + "--new-secondary=%s" % node2["primary"], node3["primary"]]) finally: qa_config.ReleaseNode(node3) + + +def TestNodeModify(node): + """gnt-node modify""" + for flag in ["master-candidate", "drained", "offline"]: + for value in ["yes", "no"]: + AssertCommand(["gnt-node", "modify", "--force", + "--%s=%s" % (flag, value), node["primary"]]) + + AssertCommand(["gnt-node", "modify", "--master-candidate=yes", + "--auto-promote", node["primary"]]) + + # Test setting secondary IP address + AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node["secondary"], + node["primary"]]) + + +def _CreateOobScriptStructure(): + """Create a simple OOB handling script and its structure.""" + master = qa_config.GetMasterNode() + + data_path = qa_utils.UploadData(master["primary"], "") + verify_path = qa_utils.UploadData(master["primary"], "") + exit_code_path = qa_utils.UploadData(master["primary"], "") + + oob_script = (("#!/bin/bash\n" + "echo \"$@\" > %s\n" + "cat %s\n" + "exit $(< %s)\n") % + (utils.ShellQuote(verify_path), utils.ShellQuote(data_path), + utils.ShellQuote(exit_code_path))) + oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700) + + return [oob_path, verify_path, data_path, exit_code_path] + + +def _UpdateOobFile(path, data): + """Updates the data file with data.""" + master = qa_config.GetMasterNode() + qa_utils.UploadData(master["primary"], data, filename=path) + + +def _AssertOobCall(verify_path, expected_args): + """Assert the OOB call was performed with expetected args.""" + master = qa_config.GetMasterNode() + + verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path]) + output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd, + tty=False) + + AssertEqual(expected_args, output.strip()) + + +def TestOutOfBand(): + """gnt-node power""" + master = qa_config.GetMasterNode() + + node = qa_config.AcquireNode(exclude=master) + + master_name = master["primary"] + node_name = node["primary"] + full_node_name = qa_utils.ResolveNodeName(node) + + (oob_path, verify_path, + data_path, exit_code_path) = _CreateOobScriptStructure() + + try: + AssertCommand(["gnt-cluster", "modify", "--node-parameters", + "oob_program=%s" % oob_path]) + + # No data, exit 0 + _UpdateOobFile(exit_code_path, "0") + + AssertCommand(["gnt-node", "power", "on", node_name]) + _AssertOobCall(verify_path, "power-on %s" % full_node_name) + + AssertCommand(["gnt-node", "power", "-f", "off", node_name]) + _AssertOobCall(verify_path, "power-off %s" % full_node_name) + + # Power off on master without options should fail + AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True) + # With force master it should still fail + AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off", + master_name], + fail=True) + + # Verify we can't transform back to online when not yet powered on + AssertCommand(["gnt-node", "modify", "-O", "no", node_name], + fail=True) + # Now reset state + AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes", + node_name]) + + AssertCommand(["gnt-node", "power", "-f", "cycle", node_name]) + _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) + + # Those commands should fail as they expect output which isn't provided yet + # But they should have called the oob helper nevermind + AssertCommand(["gnt-node", "power", "status", node_name], + fail=True) + _AssertOobCall(verify_path, "power-status %s" % full_node_name) + + AssertCommand(["gnt-node", "health", node_name], + fail=True) + _AssertOobCall(verify_path, "health %s" % full_node_name) + + AssertCommand(["gnt-node", "health"], fail=True) + + # Correct Data, exit 0 + _UpdateOobFile(data_path, serializer.DumpJson({"powered": True})) + + AssertCommand(["gnt-node", "power", "status", node_name]) + _AssertOobCall(verify_path, "power-status %s" % full_node_name) + + _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"], + ["disk0", "CRITICAL"]])) + + AssertCommand(["gnt-node", "health", node_name]) + _AssertOobCall(verify_path, "health %s" % full_node_name) + + AssertCommand(["gnt-node", "health"]) + + # Those commands should fail as they expect no data regardless of exit 0 + AssertCommand(["gnt-node", "power", "on", node_name], fail=True) + _AssertOobCall(verify_path, "power-on %s" % full_node_name) + + try: + AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True) + _AssertOobCall(verify_path, "power-off %s" % full_node_name) + finally: + AssertCommand(["gnt-node", "modify", "-O", "no", node_name]) + + AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True) + _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) + + # Data, exit 1 (all should fail) + _UpdateOobFile(exit_code_path, "1") + + AssertCommand(["gnt-node", "power", "on", node_name], fail=True) + _AssertOobCall(verify_path, "power-on %s" % full_node_name) + + try: + AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True) + _AssertOobCall(verify_path, "power-off %s" % full_node_name) + finally: + AssertCommand(["gnt-node", "modify", "-O", "no", node_name]) + + AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True) + _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) + + AssertCommand(["gnt-node", "power", "status", node_name], + fail=True) + _AssertOobCall(verify_path, "power-status %s" % full_node_name) + + AssertCommand(["gnt-node", "health", node_name], + fail=True) + _AssertOobCall(verify_path, "health %s" % full_node_name) + + AssertCommand(["gnt-node", "health"], fail=True) + + # No data, exit 1 (all should fail) + _UpdateOobFile(data_path, "") + AssertCommand(["gnt-node", "power", "on", node_name], fail=True) + _AssertOobCall(verify_path, "power-on %s" % full_node_name) + + try: + AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True) + _AssertOobCall(verify_path, "power-off %s" % full_node_name) + finally: + AssertCommand(["gnt-node", "modify", "-O", "no", node_name]) + + AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True) + _AssertOobCall(verify_path, "power-cycle %s" % full_node_name) + + AssertCommand(["gnt-node", "power", "status", node_name], + fail=True) + _AssertOobCall(verify_path, "power-status %s" % full_node_name) + + AssertCommand(["gnt-node", "health", node_name], + fail=True) + _AssertOobCall(verify_path, "health %s" % full_node_name) + + AssertCommand(["gnt-node", "health"], fail=True) + + # Different OOB script for node + verify_path2 = qa_utils.UploadData(master["primary"], "") + oob_script = ("#!/bin/sh\n" + "echo \"$@\" > %s\n") % verify_path2 + oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700) + + try: + AssertCommand(["gnt-node", "modify", "--node-parameters", + "oob_program=%s" % oob_path2, node_name]) + AssertCommand(["gnt-node", "power", "on", node_name]) + _AssertOobCall(verify_path2, "power-on %s" % full_node_name) + finally: + AssertCommand(["gnt-node", "modify", "--node-parameters", + "oob_program=default", node_name]) + AssertCommand(["rm", "-f", oob_path2, verify_path2]) + finally: + AssertCommand(["gnt-cluster", "modify", "--node-parameters", + "oob_program="]) + AssertCommand(["rm", "-f", oob_path, verify_path, data_path, + exit_code_path]) + + +def TestNodeList(): + """gnt-node list""" + qa_utils.GenericQueryTest("gnt-node", query.NODE_FIELDS.keys()) + + +def TestNodeListFields(): + """gnt-node list-fields""" + qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys()) + + +def TestNodeListDrbd(node): + """gnt-node list-drbd""" + AssertCommand(["gnt-node", "list-drbd", node["primary"]])