X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/b459a848dc62e314e61e8ae14edd3ff6cc2b2822..1c9e817d5c5090ab2ba4982e4e51d9eacdb206c6:/qa/qa_rapi.py diff --git a/qa/qa_rapi.py b/qa/qa_rapi.py index ece6e64..a9e9bc0 100644 --- a/qa/qa_rapi.py +++ b/qa/qa_rapi.py @@ -1,6 +1,7 @@ # +# -# Copyright (C) 2007, 2008, 2009, 2010, 2011 Google Inc. +# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +25,8 @@ import tempfile import random +import re +import itertools from ganeti import utils from ganeti import constants @@ -34,6 +37,7 @@ from ganeti import objects from ganeti import query from ganeti import compat from ganeti import qlang +from ganeti import pathutils import ganeti.rapi.client # pylint: disable=W0611 import ganeti.rapi.client_utils @@ -42,7 +46,11 @@ import qa_config import qa_utils import qa_error +from qa_instance import IsFailoverSupported +from qa_instance import IsMigrationSupported +from qa_instance import IsDiskReplacingSupported from qa_utils import (AssertEqual, AssertIn, AssertMatch, StartLocalCommand) +from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG _rapi_ca = None @@ -68,11 +76,11 @@ def Setup(username, password): master = qa_config.GetMasterNode() # Load RAPI certificate from master node - cmd = ["cat", constants.RAPI_CERT_FILE] + cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_CERT_FILE)] # Write to temporary file _rapi_ca = tempfile.NamedTemporaryFile() - _rapi_ca.write(qa_utils.GetCommandOutput(master["primary"], + _rapi_ca.write(qa_utils.GetCommandOutput(master.primary, utils.ShellQuoteArgs(cmd))) _rapi_ca.flush() @@ -80,32 +88,39 @@ def Setup(username, password): cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name, proxy="") - _rapi_client = rapi.client.GanetiRapiClient(master["primary"], port=port, - username=username, - password=password, - curl_config_fn=cfg_curl) + if qa_config.UseVirtualCluster(): + # TODO: Implement full support for RAPI on virtual clusters + print qa_utils.FormatWarning("RAPI tests are not yet supported on" + " virtual clusters and will be disabled") + + assert _rapi_client is None + else: + _rapi_client = rapi.client.GanetiRapiClient(master.primary, port=port, + username=username, + password=password, + curl_config_fn=cfg_curl) - print "RAPI protocol version: %s" % _rapi_client.GetVersion() + print "RAPI protocol version: %s" % _rapi_client.GetVersion() INSTANCE_FIELDS = ("name", "os", "pnode", "snodes", "admin_state", - "disk_template", "disk.sizes", + "disk_template", "disk.sizes", "disk.spindles", "nic.ips", "nic.macs", "nic.modes", "nic.links", "beparams", "hvparams", "oper_state", "oper_ram", "oper_vcpus", "status", "tags") -NODE_FIELDS = ("name", "dtotal", "dfree", +NODE_FIELDS = ("name", "dtotal", "dfree", "sptotal", "spfree", "mtotal", "mnode", "mfree", "pinst_cnt", "sinst_cnt", "tags") -GROUP_FIELDS = frozenset([ +GROUP_FIELDS = compat.UniqueFrozenset([ "name", "uuid", "alloc_policy", "node_cnt", "node_list", ]) -JOB_FIELDS = frozenset([ +JOB_FIELDS = compat.UniqueFrozenset([ "id", "ops", "status", "summary", "opstatus", "opresult", "oplog", "received_ts", "start_ts", "end_ts", @@ -118,7 +133,9 @@ def Enabled(): """Return whether remote API tests should be run. """ - return qa_config.TestEnabled("rapi") + # TODO: Implement RAPI tests for virtual clusters + return (qa_config.TestEnabled("rapi") and + not qa_config.UseVirtualCluster()) def _DoTests(uris): @@ -144,7 +161,8 @@ def _DoTests(uris): def _VerifyReturnsJob(data): - AssertMatch(data, r"^\d+$") + if not isinstance(data, int): + AssertMatch(data, r"^\d+$") def TestVersion(): @@ -228,10 +246,22 @@ def TestRapiQuery(): """Testing resource queries via remote API. """ + # FIXME: the tests are failing if no LVM is enabled, investigate + # if it is a bug in the QA or in the code + if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): + return + master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode()) rnd = random.Random(7818) for what in constants.QR_VIA_RAPI: + if what == constants.QR_JOB: + namefield = "id" + elif what == constants.QR_EXPORT: + namefield = "export" + else: + namefield = "name" + all_fields = query.ALL_FIELDS[what].keys() rnd.shuffle(all_fields) @@ -241,7 +271,7 @@ def TestRapiQuery(): AssertEqual(len(qresult.fields), len(all_fields)) # One field - result = _rapi_client.QueryFields(what, fields=["name"]) + result = _rapi_client.QueryFields(what, fields=[namefield]) qresult = objects.QueryFieldsResponse.FromDict(result) AssertEqual(len(qresult.fields), 1) @@ -286,33 +316,36 @@ def TestRapiQuery(): ("/2/query/%s?fields=%s" % (what, ",".join(all_fields)), compat.partial(_Check, all_fields), "GET", None), - ("/2/query/%s?fields=name" % what, - compat.partial(_Check, ["name"]), "GET", None), + ("/2/query/%s?fields=%s" % (what, namefield), + compat.partial(_Check, [namefield]), "GET", None), # Note the spaces - ("/2/query/%s?fields=name,%%20name%%09,name%%20" % what, - compat.partial(_Check, ["name"] * 3), "GET", None), + ("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" % + (what, namefield, namefield, namefield), + compat.partial(_Check, [namefield] * 3), "GET", None), # PUT with fields in query - ("/2/query/%s?fields=name" % what, - compat.partial(_Check, ["name"]), "PUT", {}), + ("/2/query/%s?fields=%s" % (what, namefield), + compat.partial(_Check, [namefield]), "PUT", {}), + + ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", { + "fields": [namefield] * 4, + }), - # Fields in body ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", { "fields": all_fields, }), - ("/2/query/%s" % what, compat.partial(_Check, ["name"] * 4), "PUT", { - "fields": ["name"] * 4, - }), - ]) + ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", { + "fields": [namefield] * 4 + })]) def _CheckFilter(): _DoTests([ # With filter ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", { "fields": all_fields, - "filter": [qlang.OP_TRUE, "name"], + "filter": [qlang.OP_TRUE, namefield], }), ]) @@ -329,17 +362,19 @@ def TestRapiQuery(): if what == constants.QR_NODE: # Test with filter - (nodes, ) = _DoTests([("/2/query/%s" % what, - compat.partial(_Check, ["name", "master"]), "PUT", { - "fields": ["name", "master"], - "filter": [qlang.OP_TRUE, "master"], - })]) + (nodes, ) = _DoTests( + [("/2/query/%s" % what, + compat.partial(_Check, ["name", "master"]), "PUT", + {"fields": ["name", "master"], + "filter": [qlang.OP_TRUE, "master"], + })]) qresult = objects.QueryResponse.FromDict(nodes) AssertEqual(qresult.data, [ [[constants.RS_NORMAL, master_name], [constants.RS_NORMAL, True]], ]) +@InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestInstance(instance): """Testing getting instance(s) info via remote API. @@ -358,19 +393,19 @@ def TestInstance(instance): _VerifyInstance(instance_data) _DoTests([ - ("/2/instances/%s" % instance["name"], _VerifyInstance, "GET", None), + ("/2/instances/%s" % instance.name, _VerifyInstance, "GET", None), ("/2/instances", _VerifyInstancesList, "GET", None), ("/2/instances?bulk=1", _VerifyInstancesBulk, "GET", None), - ("/2/instances/%s/activate-disks" % instance["name"], + ("/2/instances/%s/activate-disks" % instance.name, _VerifyReturnsJob, "PUT", None), - ("/2/instances/%s/deactivate-disks" % instance["name"], + ("/2/instances/%s/deactivate-disks" % instance.name, _VerifyReturnsJob, "PUT", None), ]) # Test OpBackupPrepare (job_id, ) = _DoTests([ ("/2/instances/%s/prepare-export?mode=%s" % - (instance["name"], constants.EXPORT_MODE_REMOTE), + (instance.name, constants.EXPORT_MODE_REMOTE), _VerifyReturnsJob, "PUT", None), ]) @@ -399,12 +434,24 @@ def TestNode(node): _VerifyNode(node_data) _DoTests([ - ("/2/nodes/%s" % node["primary"], _VerifyNode, "GET", None), + ("/2/nodes/%s" % node.primary, _VerifyNode, "GET", None), ("/2/nodes", _VerifyNodesList, "GET", None), ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None), ]) +def _FilterTags(seq): + """Removes unwanted tags from a sequence. + + """ + ignore_re = qa_config.get("ignore-tags-re", None) + + if ignore_re: + return itertools.ifilterfalse(re.compile(ignore_re).match, seq) + else: + return seq + + def TestTags(kind, name, tags): """Tests .../tags resources. @@ -421,7 +468,7 @@ def TestTags(kind, name, tags): raise errors.ProgrammerError("Unknown tag kind") def _VerifyTags(data): - AssertEqual(sorted(tags), sorted(data)) + AssertEqual(sorted(tags), sorted(_FilterTags(data))) queryargs = "&".join("tag=%s" % i for i in tags) @@ -464,9 +511,7 @@ def TestRapiNodeGroups(): """Test several node group operations using RAPI. """ - groups = qa_config.get("groups", {}) - group1, group2, group3 = groups.get("inexistent-groups", - ["group1", "group2", "group3"])[:3] + (group1, group2, group3) = qa_utils.GetNonexistentGroups(3) # Create a group with no attributes body = { @@ -524,33 +569,40 @@ def TestRapiNodeGroups(): def TestRapiInstanceAdd(node, use_client): """Test adding a new instance via RAPI""" + if not qa_config.IsTemplateSupported(constants.DT_PLAIN): + return instance = qa_config.AcquireInstance() + instance.SetDiskTemplate(constants.DT_PLAIN) try: - memory = utils.ParseUnit(qa_config.get("mem")) - disk_sizes = [utils.ParseUnit(size) for size in qa_config.get("disk")] - disks = [{"size": size} for size in disk_sizes] - nics = [{}] + disks = [{"size": utils.ParseUnit(d.get("size")), + "name": str(d.get("name"))} + for d in qa_config.GetDiskOptions()] + nic0_mac = instance.GetNicMacAddr(0, constants.VALUE_GENERATE) + nics = [{ + constants.INIC_MAC: nic0_mac, + }] beparams = { - constants.BE_MEMORY: memory, + constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)), + constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)), } if use_client: job_id = _rapi_client.CreateInstance(constants.INSTANCE_CREATE, - instance["name"], + instance.name, constants.DT_PLAIN, disks, nics, os=qa_config.get("os"), - pnode=node["primary"], + pnode=node.primary, beparams=beparams) else: body = { "__version__": 1, "mode": constants.INSTANCE_CREATE, - "name": instance["name"], + "name": instance.name, "os_type": qa_config.get("os"), "disk_template": constants.DT_PLAIN, - "pnode": node["primary"], + "pnode": node.primary, "beparams": beparams, "disks": disks, "nics": nics, @@ -564,58 +616,120 @@ def TestRapiInstanceAdd(node, use_client): return instance except: - qa_config.ReleaseInstance(instance) + instance.Release() raise +@InstanceCheck(None, INST_DOWN, FIRST_ARG) def TestRapiInstanceRemove(instance, use_client): """Test removing instance via RAPI""" + # FIXME: this does not work if LVM is not enabled. Find out if this is a bug + # in RAPI or in the test + if not qa_config.IsStorageTypeSupported(constants.ST_LVM_VG): + return + if use_client: - job_id = _rapi_client.DeleteInstance(instance["name"]) + job_id = _rapi_client.DeleteInstance(instance.name) else: (job_id, ) = _DoTests([ - ("/2/instances/%s" % instance["name"], _VerifyReturnsJob, "DELETE", None), + ("/2/instances/%s" % instance.name, _VerifyReturnsJob, "DELETE", None), ]) _WaitForRapiJob(job_id) - qa_config.ReleaseInstance(instance) - +@InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceMigrate(instance): """Test migrating instance via RAPI""" + if not IsMigrationSupported(instance): + print qa_utils.FormatInfo("Instance doesn't support migration, skipping" + " test") + return # Move to secondary node - _WaitForRapiJob(_rapi_client.MigrateInstance(instance["name"])) + _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name)) + qa_utils.RunInstanceCheck(instance, True) # And back to previous primary - _WaitForRapiJob(_rapi_client.MigrateInstance(instance["name"])) + _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name)) +@InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceFailover(instance): """Test failing over instance via RAPI""" + if not IsFailoverSupported(instance): + print qa_utils.FormatInfo("Instance doesn't support failover, skipping" + " test") + return # Move to secondary node - _WaitForRapiJob(_rapi_client.FailoverInstance(instance["name"])) + _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name)) + qa_utils.RunInstanceCheck(instance, True) # And back to previous primary - _WaitForRapiJob(_rapi_client.FailoverInstance(instance["name"])) + _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name)) -def TestRapiInstanceRename(rename_source, rename_target): - """Test renaming instance via RAPI""" +@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG) +def TestRapiInstanceShutdown(instance): + """Test stopping an instance via RAPI""" + _WaitForRapiJob(_rapi_client.ShutdownInstance(instance.name)) + + +@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG) +def TestRapiInstanceStartup(instance): + """Test starting an instance via RAPI""" + _WaitForRapiJob(_rapi_client.StartupInstance(instance.name)) + + +@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) +def TestRapiInstanceRenameAndBack(rename_source, rename_target): + """Test renaming instance via RAPI + + This must leave the instance with the original name (in the + non-failure case). + + """ _WaitForRapiJob(_rapi_client.RenameInstance(rename_source, rename_target)) + qa_utils.RunInstanceCheck(rename_source, False) + qa_utils.RunInstanceCheck(rename_target, False) + _WaitForRapiJob(_rapi_client.RenameInstance(rename_target, rename_source)) + qa_utils.RunInstanceCheck(rename_target, False) +@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestRapiInstanceReinstall(instance): """Test reinstalling an instance via RAPI""" - _WaitForRapiJob(_rapi_client.ReinstallInstance(instance["name"])) - - + if instance.disk_template == constants.DT_DISKLESS: + print qa_utils.FormatInfo("Test not supported for diskless instances") + return + + _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name)) + # By default, the instance is started again + qa_utils.RunInstanceCheck(instance, True) + + # Reinstall again without starting + _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name, + no_startup=True)) + + +@InstanceCheck(INST_UP, INST_UP, FIRST_ARG) +def TestRapiInstanceReplaceDisks(instance): + """Test replacing instance disks via RAPI""" + if not IsDiskReplacingSupported(instance): + print qa_utils.FormatInfo("Instance doesn't support disk replacing," + " skipping test") + return + fn = _rapi_client.ReplaceInstanceDisks + _WaitForRapiJob(fn(instance.name, + mode=constants.REPLACE_DISK_AUTO, disks=[])) + _WaitForRapiJob(fn(instance.name, + mode=constants.REPLACE_DISK_SEC, disks="0")) + + +@InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceModify(instance): """Test modifying instance via RAPI""" - def _ModifyInstance(**kwargs): - _WaitForRapiJob(_rapi_client.ModifyInstance(instance["name"], **kwargs)) + default_hv = qa_config.GetDefaultHypervisor() - _ModifyInstance(hvparams={ - constants.HV_KERNEL_ARGS: "single", - }) + def _ModifyInstance(**kwargs): + _WaitForRapiJob(_rapi_client.ModifyInstance(instance.name, **kwargs)) _ModifyInstance(beparams={ constants.BE_VCPUS: 3, @@ -625,23 +739,36 @@ def TestRapiInstanceModify(instance): constants.BE_VCPUS: constants.VALUE_DEFAULT, }) - _ModifyInstance(hvparams={ - constants.HV_KERNEL_ARGS: constants.VALUE_DEFAULT, - }) - - + if default_hv == constants.HT_XEN_PVM: + _ModifyInstance(hvparams={ + constants.HV_KERNEL_ARGS: "single", + }) + _ModifyInstance(hvparams={ + constants.HV_KERNEL_ARGS: constants.VALUE_DEFAULT, + }) + elif default_hv == constants.HT_XEN_HVM: + _ModifyInstance(hvparams={ + constants.HV_BOOT_ORDER: "acn", + }) + _ModifyInstance(hvparams={ + constants.HV_BOOT_ORDER: constants.VALUE_DEFAULT, + }) + + +@InstanceCheck(INST_UP, INST_UP, FIRST_ARG) def TestRapiInstanceConsole(instance): """Test getting instance console information via RAPI""" - result = _rapi_client.GetInstanceConsole(instance["name"]) + result = _rapi_client.GetInstanceConsole(instance.name) console = objects.InstanceConsole.FromDict(result) AssertEqual(console.Validate(), True) - AssertEqual(console.instance, qa_utils.ResolveInstanceName(instance["name"])) + AssertEqual(console.instance, qa_utils.ResolveInstanceName(instance.name)) +@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG) def TestRapiStoppedInstanceConsole(instance): """Test getting stopped instance's console information via RAPI""" try: - _rapi_client.GetInstanceConsole(instance["name"]) + _rapi_client.GetInstanceConsole(instance.name) except rapi.client.GanetiApiError, err: AssertEqual(err.code, 503) else: @@ -657,7 +784,7 @@ def GetOperatingSystems(): def TestInterClusterInstanceMove(src_instance, dest_instance, - pnode, snode, tnode): + inodes, tnode): """Test tools/move-instance""" master = qa_config.GetMasterNode() @@ -665,20 +792,26 @@ def TestInterClusterInstanceMove(src_instance, dest_instance, rapi_pw_file.write(_rapi_password) rapi_pw_file.flush() + dest_instance.SetDiskTemplate(src_instance.disk_template) + # TODO: Run some instance tests before moving back - if snode is None: + if len(inodes) > 1: + # No disk template currently requires more than 1 secondary node. If this + # changes, either this test must be skipped or the script must be updated. + assert len(inodes) == 2 + snode = inodes[1] + else: # instance is not redundant, but we still need to pass a node # (which will be ignored) - fsec = tnode - else: - fsec = snode + snode = tnode + pnode = inodes[0] # note: pnode:snode are the *current* nodes, so we move it first to # tnode:pnode, then back to pnode:snode - for si, di, pn, sn in [(src_instance["name"], dest_instance["name"], - tnode["primary"], pnode["primary"]), - (dest_instance["name"], src_instance["name"], - pnode["primary"], fsec["primary"])]: + for si, di, pn, sn in [(src_instance.name, dest_instance.name, + tnode.primary, pnode.primary), + (dest_instance.name, src_instance.name, + pnode.primary, snode.primary)]: cmd = [ "../tools/move-instance", "--verbose", @@ -689,9 +822,12 @@ def TestInterClusterInstanceMove(src_instance, dest_instance, "--dest-primary-node=%s" % pn, "--dest-secondary-node=%s" % sn, "--net=0:mac=%s" % constants.VALUE_GENERATE, - master["primary"], - master["primary"], + master.primary, + master.primary, si, ] + qa_utils.RunInstanceCheck(di, False) AssertEqual(StartLocalCommand(cmd).wait(), 0) + qa_utils.RunInstanceCheck(si, False) + qa_utils.RunInstanceCheck(di, True)