#
#
-# Copyright (C) 2007 Google Inc.
+# Copyright (C) 2007, 2011, 2012, 2013 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# 02110-1301, USA.
+"""Node-related QA tests.
+
+"""
+
from ganeti import utils
from ganeti import constants
from ganeti import query
import qa_error
import qa_utils
-from qa_utils import AssertCommand
+from qa_utils import AssertCommand, AssertEqual
def _NodeAdd(node, readd=False):
- if not readd and node.get('_added', False):
- raise qa_error.Error("Node %s already in cluster" % node['primary'])
- elif readd and not node.get('_added', False):
- raise qa_error.Error("Node %s not yet in cluster" % node['primary'])
-
- cmd = ['gnt-node', 'add', "--no-ssh-key-check"]
- if node.get('secondary', None):
- cmd.append('--secondary-ip=%s' % node['secondary'])
+ if not readd and node.added:
+ raise qa_error.Error("Node %s already in cluster" % node.primary)
+ elif readd and not node.added:
+ raise qa_error.Error("Node %s not yet in cluster" % node.primary)
+
+ cmd = ["gnt-node", "add", "--no-ssh-key-check"]
+ if node.secondary:
+ cmd.append("--secondary-ip=%s" % node.secondary)
if readd:
- cmd.append('--readd')
- cmd.append(node['primary'])
+ cmd.append("--readd")
+ cmd.append(node.primary)
AssertCommand(cmd)
- node['_added'] = True
+ if readd:
+ assert node.added
+ else:
+ node.MarkAdded()
def _NodeRemove(node):
- AssertCommand(["gnt-node", "remove", node["primary"]])
- node['_added'] = False
+ AssertCommand(["gnt-node", "remove", node.primary])
+ node.MarkRemoved()
+
+
+def MakeNodeOffline(node, value):
+ """gnt-node modify --offline=value"""
+ # value in ["yes", "no"]
+ AssertCommand(["gnt-node", "modify", "--offline", value, node.primary])
def TestNodeAddAll():
"""Adding all nodes to cluster."""
master = qa_config.GetMasterNode()
- for node in qa_config.get('nodes'):
+ for node in qa_config.get("nodes"):
if node != master:
_NodeAdd(node, readd=False)
"""
master = qa_config.GetMasterNode()
- for node in qa_config.get('nodes'):
+ for node in qa_config.get("nodes"):
if node != master:
- node['_added'] = True
+ node.MarkAdded()
def TestNodeRemoveAll():
"""Removing all nodes from cluster."""
master = qa_config.GetMasterNode()
- for node in qa_config.get('nodes'):
+ for node in qa_config.get("nodes"):
if node != master:
_NodeRemove(node)
"""gnt-node storage"""
master = qa_config.GetMasterNode()
- for storage_type in constants.VALID_STORAGE_TYPES:
+ # FIXME: test all storage_types in constants.STORAGE_TYPES
+ # as soon as they are implemented.
+ enabled_storage_types = qa_config.GetEnabledStorageTypes()
+ testable_storage_types = list(set(enabled_storage_types).intersection(
+ set([constants.ST_FILE, constants.ST_LVM_VG, constants.ST_LVM_PV])))
+
+ for storage_type in testable_storage_types:
+
+ cmd = ["gnt-node", "list-storage", "--storage-type", storage_type]
+
# Test simple list
- AssertCommand(["gnt-node", "list-storage", "--storage-type", storage_type])
+ AssertCommand(cmd)
# Test all storage fields
cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
- "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS) +
- [constants.SF_NODE, constants.SF_TYPE])]
+ "--output=%s" % ",".join(list(constants.VALID_STORAGE_FIELDS))]
AssertCommand(cmd)
# Get list of valid storage devices
cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
"--output=node,name,allocatable", "--separator=|",
"--no-headers"]
- output = qa_utils.GetCommandOutput(master["primary"],
+ output = qa_utils.GetCommandOutput(master.primary,
utils.ShellQuoteArgs(cmd))
# Test with up to two devices
AssertCommand(["gnt-node", "modify-storage", "--allocatable", i,
node_name, storage_type, st_name], fail=fail)
+ # Verify list output
+ cmd = ["gnt-node", "list-storage", "--storage-type", storage_type,
+ "--output=name,allocatable", "--separator=|",
+ "--no-headers", node_name]
+ listout = qa_utils.GetCommandOutput(master.primary,
+ utils.ShellQuoteArgs(cmd))
+ for line in listout.splitlines():
+ (vfy_name, vfy_allocatable) = line.split("|")
+ if vfy_name == st_name and not fail:
+ AssertEqual(vfy_allocatable, i[0].upper())
+ else:
+ AssertEqual(vfy_allocatable, st_allocatable)
+
# Test repair functionality
fail = (constants.SO_FIX_CONSISTENCY not in
constants.VALID_STORAGE_OPERATIONS.get(storage_type, []))
" it to have no primary instances.")
# Fail over to secondary node
- AssertCommand(["gnt-node", "failover", "-f", node["primary"]])
+ AssertCommand(["gnt-node", "failover", "-f", node.primary])
# ... and back again.
- AssertCommand(["gnt-node", "failover", "-f", node2["primary"]])
+ AssertCommand(["gnt-node", "failover", "-f", node2.primary])
+
+
+def TestNodeMigrate(node, node2):
+ """gnt-node migrate"""
+ if qa_utils.GetNodeInstances(node2, secondaries=False):
+ raise qa_error.UnusableNodeError("Secondary node has at least one"
+ " primary instance. This test requires"
+ " it to have no primary instances.")
+
+ # Migrate to secondary node
+ AssertCommand(["gnt-node", "migrate", "-f", node.primary])
+
+ # ... and back again.
+ AssertCommand(["gnt-node", "migrate", "-f", node2.primary])
def TestNodeEvacuate(node, node2):
# Evacuate all secondary instances
AssertCommand(["gnt-node", "evacuate", "-f",
- "--new-secondary=%s" % node3["primary"], node2["primary"]])
+ "--new-secondary=%s" % node3.primary, node2.primary])
# ... and back again.
AssertCommand(["gnt-node", "evacuate", "-f",
- "--new-secondary=%s" % node2["primary"], node3["primary"]])
+ "--new-secondary=%s" % node2.primary, node3.primary])
finally:
- qa_config.ReleaseNode(node3)
+ node3.Release()
def TestNodeModify(node):
"""gnt-node modify"""
+
+ # make sure enough master candidates will be available by disabling the
+ # master candidate role first with --auto-promote
+ AssertCommand(["gnt-node", "modify", "--master-candidate=no",
+ "--auto-promote", node.primary])
+
+ # now it's save to force-remove the master candidate role
for flag in ["master-candidate", "drained", "offline"]:
for value in ["yes", "no"]:
AssertCommand(["gnt-node", "modify", "--force",
- "--%s=%s" % (flag, value), node["primary"]])
+ "--%s=%s" % (flag, value), node.primary])
- AssertCommand(["gnt-node", "modify", "--master-candidate=yes",
- "--auto-promote", node["primary"]])
+ AssertCommand(["gnt-node", "modify", "--master-candidate=yes", node.primary])
+
+ # Test setting secondary IP address
+ AssertCommand(["gnt-node", "modify", "--secondary-ip=%s" % node.secondary,
+ node.primary])
def _CreateOobScriptStructure():
"""Create a simple OOB handling script and its structure."""
master = qa_config.GetMasterNode()
- data_path = qa_utils.UploadData(master["primary"], "")
- verify_path = qa_utils.UploadData(master["primary"], "")
- exit_code_path = qa_utils.UploadData(master["primary"], "")
+ data_path = qa_utils.UploadData(master.primary, "")
+ verify_path = qa_utils.UploadData(master.primary, "")
+ exit_code_path = qa_utils.UploadData(master.primary, "")
oob_script = (("#!/bin/bash\n"
"echo \"$@\" > %s\n"
"exit $(< %s)\n") %
(utils.ShellQuote(verify_path), utils.ShellQuote(data_path),
utils.ShellQuote(exit_code_path)))
- oob_path = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
+ oob_path = qa_utils.UploadData(master.primary, oob_script, mode=0700)
return [oob_path, verify_path, data_path, exit_code_path]
def _UpdateOobFile(path, data):
"""Updates the data file with data."""
master = qa_config.GetMasterNode()
- qa_utils.UploadData(master["primary"], data, filename=path)
+ qa_utils.UploadData(master.primary, data, filename=path)
def _AssertOobCall(verify_path, expected_args):
master = qa_config.GetMasterNode()
verify_output_cmd = utils.ShellQuoteArgs(["cat", verify_path])
- output = qa_utils.GetCommandOutput(master["primary"], verify_output_cmd)
+ output = qa_utils.GetCommandOutput(master.primary, verify_output_cmd,
+ tty=False)
- qa_utils.AssertEqual(expected_args, output.strip())
+ AssertEqual(expected_args, output.strip())
def TestOutOfBand():
"""gnt-node power"""
master = qa_config.GetMasterNode()
+ node = qa_config.AcquireNode(exclude=master)
+
+ master_name = master.primary
+ node_name = node.primary
+ full_node_name = qa_utils.ResolveNodeName(node)
+
(oob_path, verify_path,
data_path, exit_code_path) = _CreateOobScriptStructure()
# No data, exit 0
_UpdateOobFile(exit_code_path, "0")
- AssertCommand(["gnt-node", "power", "on", master["primary"]])
- _AssertOobCall(verify_path, "power-on %s" % master["primary"])
+ AssertCommand(["gnt-node", "power", "on", node_name])
+ _AssertOobCall(verify_path, "power-on %s" % full_node_name)
+
+ AssertCommand(["gnt-node", "power", "-f", "off", node_name])
+ _AssertOobCall(verify_path, "power-off %s" % full_node_name)
+
+ # Power off on master without options should fail
+ AssertCommand(["gnt-node", "power", "-f", "off", master_name], fail=True)
+ # With force master it should still fail
+ AssertCommand(["gnt-node", "power", "-f", "--ignore-status", "off",
+ master_name],
+ fail=True)
+
+ # Verify we can't transform back to online when not yet powered on
+ AssertCommand(["gnt-node", "modify", "-O", "no", node_name],
+ fail=True)
+ # Now reset state
+ AssertCommand(["gnt-node", "modify", "-O", "no", "--node-powered", "yes",
+ node_name])
- AssertCommand(["gnt-node", "power", "off", master["primary"]])
- _AssertOobCall(verify_path, "power-off %s" % master["primary"])
+ AssertCommand(["gnt-node", "power", "-f", "cycle", node_name])
+ _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
- AssertCommand(["gnt-node", "power", "cycle", master["primary"]])
- _AssertOobCall(verify_path, "power-cycle %s" % master["primary"])
+ # Those commands should fail as they expect output which isn't provided yet
+ # But they should have called the oob helper nevermind
+ AssertCommand(["gnt-node", "power", "status", node_name],
+ fail=True)
+ _AssertOobCall(verify_path, "power-status %s" % full_node_name)
- # This command should fail as it expects output which isn't provided yet
- # But it should have called the oob helper nevermind
- AssertCommand(["gnt-node", "power", "status", master["primary"]],
+ AssertCommand(["gnt-node", "health", node_name],
fail=True)
- _AssertOobCall(verify_path, "power-status %s" % master["primary"])
+ _AssertOobCall(verify_path, "health %s" % full_node_name)
+
+ AssertCommand(["gnt-node", "health"], fail=True)
+
+ # Correct Data, exit 0
+ _UpdateOobFile(data_path, serializer.DumpJson({"powered": True}))
+
+ AssertCommand(["gnt-node", "power", "status", node_name])
+ _AssertOobCall(verify_path, "power-status %s" % full_node_name)
+
+ _UpdateOobFile(data_path, serializer.DumpJson([["temp", "OK"],
+ ["disk0", "CRITICAL"]]))
- # Data, exit 0
- _UpdateOobFile(data_path, serializer.DumpJson({ "powered": True }))
+ AssertCommand(["gnt-node", "health", node_name])
+ _AssertOobCall(verify_path, "health %s" % full_node_name)
- AssertCommand(["gnt-node", "power", "status", master["primary"]])
- _AssertOobCall(verify_path, "power-status %s" % master["primary"])
+ AssertCommand(["gnt-node", "health"])
- AssertCommand(["gnt-node", "power", "on", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-on %s" % master["primary"])
+ # Those commands should fail as they expect no data regardless of exit 0
+ AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-on %s" % full_node_name)
- AssertCommand(["gnt-node", "power", "off", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-off %s" % master["primary"])
+ try:
+ AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-off %s" % full_node_name)
+ finally:
+ AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
- AssertCommand(["gnt-node", "power", "cycle", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-cycle %s" % master["primary"])
+ AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
# Data, exit 1 (all should fail)
_UpdateOobFile(exit_code_path, "1")
- AssertCommand(["gnt-node", "power", "on", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-on %s" % master["primary"])
+ AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-on %s" % full_node_name)
- AssertCommand(["gnt-node", "power", "off", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-off %s" % master["primary"])
+ try:
+ AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-off %s" % full_node_name)
+ finally:
+ AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
- AssertCommand(["gnt-node", "power", "cycle", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-cycle %s" % master["primary"])
+ AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
- AssertCommand(["gnt-node", "power", "status", master["primary"]],
+ AssertCommand(["gnt-node", "power", "status", node_name],
fail=True)
- _AssertOobCall(verify_path, "power-status %s" % master["primary"])
+ _AssertOobCall(verify_path, "power-status %s" % full_node_name)
+
+ AssertCommand(["gnt-node", "health", node_name],
+ fail=True)
+ _AssertOobCall(verify_path, "health %s" % full_node_name)
+
+ AssertCommand(["gnt-node", "health"], fail=True)
# No data, exit 1 (all should fail)
_UpdateOobFile(data_path, "")
- AssertCommand(["gnt-node", "power", "on", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-on %s" % master["primary"])
+ AssertCommand(["gnt-node", "power", "on", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-on %s" % full_node_name)
- AssertCommand(["gnt-node", "power", "off", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-off %s" % master["primary"])
+ try:
+ AssertCommand(["gnt-node", "power", "-f", "off", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-off %s" % full_node_name)
+ finally:
+ AssertCommand(["gnt-node", "modify", "-O", "no", node_name])
- AssertCommand(["gnt-node", "power", "cycle", master["primary"]], fail=True)
- _AssertOobCall(verify_path, "power-cycle %s" % master["primary"])
+ AssertCommand(["gnt-node", "power", "-f", "cycle", node_name], fail=True)
+ _AssertOobCall(verify_path, "power-cycle %s" % full_node_name)
+
+ AssertCommand(["gnt-node", "power", "status", node_name],
+ fail=True)
+ _AssertOobCall(verify_path, "power-status %s" % full_node_name)
- AssertCommand(["gnt-node", "power", "status", master["primary"]],
+ AssertCommand(["gnt-node", "health", node_name],
fail=True)
- _AssertOobCall(verify_path, "power-status %s" % master["primary"])
+ _AssertOobCall(verify_path, "health %s" % full_node_name)
+
+ AssertCommand(["gnt-node", "health"], fail=True)
# Different OOB script for node
- verify_path2 = qa_utils.UploadData(master["primary"], "")
+ verify_path2 = qa_utils.UploadData(master.primary, "")
oob_script = ("#!/bin/sh\n"
"echo \"$@\" > %s\n") % verify_path2
- oob_path2 = qa_utils.UploadData(master["primary"], oob_script, mode=0700)
+ oob_path2 = qa_utils.UploadData(master.primary, oob_script, mode=0700)
try:
AssertCommand(["gnt-node", "modify", "--node-parameters",
- "oob_program=%s" % oob_path2, master["primary"]])
- AssertCommand(["gnt-node", "power", "on", master["primary"]])
- _AssertOobCall(verify_path2, "power-on %s" % master["primary"])
+ "oob_program=%s" % oob_path2, node_name])
+ AssertCommand(["gnt-node", "power", "on", node_name])
+ _AssertOobCall(verify_path2, "power-on %s" % full_node_name)
finally:
AssertCommand(["gnt-node", "modify", "--node-parameters",
- "oob_program=default", master["primary"]])
+ "oob_program=default", node_name])
AssertCommand(["rm", "-f", oob_path2, verify_path2])
finally:
AssertCommand(["gnt-cluster", "modify", "--node-parameters",
- "oob_program=default"])
+ "oob_program="])
AssertCommand(["rm", "-f", oob_path, verify_path, data_path,
exit_code_path])
def TestNodeListFields():
"""gnt-node list-fields"""
qa_utils.GenericQueryFieldsTest("gnt-node", query.NODE_FIELDS.keys())
+
+
+def TestNodeListDrbd(node):
+ """gnt-node list-drbd"""
+ AssertCommand(["gnt-node", "list-drbd", node.primary])
+
+
+def _BuildSetESCmd(action, value, node_name):
+ cmd = ["gnt-node"]
+ if action == "add":
+ cmd.extend(["add", "--readd"])
+ else:
+ cmd.append("modify")
+ cmd.extend(["--node-parameters", "exclusive_storage=%s" % value, node_name])
+ return cmd
+
+
+def TestExclStorSingleNode(node):
+ """gnt-node add/modify cannot change the exclusive_storage flag.
+
+ """
+ for action in ["add", "modify"]:
+ for value in (True, False, "default"):
+ AssertCommand(_BuildSetESCmd(action, value, node.primary), fail=True)