4 # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Remote API QA tests.
31 from ganeti import utils
32 from ganeti import constants
33 from ganeti import errors
34 from ganeti import cli
35 from ganeti import rapi
36 from ganeti import objects
37 from ganeti import query
38 from ganeti import compat
39 from ganeti import qlang
41 import ganeti.rapi.client # pylint: disable=W0611
42 import ganeti.rapi.client_utils
48 from qa_utils import (AssertEqual, AssertIn, AssertMatch, StartLocalCommand)
49 from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG
58 def Setup(username, password):
59 """Configures the RAPI client.
62 # pylint: disable=W0603
69 _rapi_username = username
70 _rapi_password = password
72 master = qa_config.GetMasterNode()
74 # Load RAPI certificate from master node
75 cmd = ["cat", constants.RAPI_CERT_FILE]
77 # Write to temporary file
78 _rapi_ca = tempfile.NamedTemporaryFile()
79 _rapi_ca.write(qa_utils.GetCommandOutput(master["primary"],
80 utils.ShellQuoteArgs(cmd)))
83 port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT)
84 cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name,
87 _rapi_client = rapi.client.GanetiRapiClient(master["primary"], port=port,
90 curl_config_fn=cfg_curl)
92 print "RAPI protocol version: %s" % _rapi_client.GetVersion()
95 INSTANCE_FIELDS = ("name", "os", "pnode", "snodes",
97 "disk_template", "disk.sizes",
98 "nic.ips", "nic.macs", "nic.modes", "nic.links",
99 "beparams", "hvparams",
100 "oper_state", "oper_ram", "oper_vcpus", "status", "tags")
102 NODE_FIELDS = ("name", "dtotal", "dfree",
103 "mtotal", "mnode", "mfree",
104 "pinst_cnt", "sinst_cnt", "tags")
106 GROUP_FIELDS = frozenset([
109 "node_cnt", "node_list",
112 JOB_FIELDS = frozenset([
113 "id", "ops", "status", "summary",
114 "opstatus", "opresult", "oplog",
115 "received_ts", "start_ts", "end_ts",
118 LIST_FIELDS = ("id", "uri")
122 """Return whether remote API tests should be run.
125 return qa_config.TestEnabled("rapi")
129 # pylint: disable=W0212
130 # due to _SendRequest usage
133 for uri, verify, method, body in uris:
134 assert uri.startswith("/")
136 print "%s %s" % (method, uri)
137 data = _rapi_client._SendRequest(method, uri, None, body)
139 if verify is not None:
143 AssertEqual(data, verify)
150 def _VerifyReturnsJob(data):
151 if not isinstance(data, int):
152 AssertMatch(data, r"^\d+$")
156 """Testing remote API version.
160 ("/version", constants.RAPI_VERSION, "GET", None),
164 def TestEmptyCluster():
165 """Testing remote API on an empty cluster.
168 master = qa_config.GetMasterNode()
169 master_full = qa_utils.ResolveNodeName(master)
171 def _VerifyInfo(data):
172 AssertIn("name", data)
173 AssertIn("master", data)
174 AssertEqual(data["master"], master_full)
176 def _VerifyNodes(data):
179 "uri": "/2/nodes/%s" % master_full,
181 AssertIn(master_entry, data)
183 def _VerifyNodesBulk(data):
185 for entry in NODE_FIELDS:
186 AssertIn(entry, node)
188 def _VerifyGroups(data):
190 "name": constants.INITIAL_NODE_GROUP_NAME,
191 "uri": "/2/groups/" + constants.INITIAL_NODE_GROUP_NAME,
193 AssertIn(default_group, data)
195 def _VerifyGroupsBulk(data):
197 for field in GROUP_FIELDS:
198 AssertIn(field, group)
201 ("/", None, "GET", None),
202 ("/2/info", _VerifyInfo, "GET", None),
203 ("/2/tags", None, "GET", None),
204 ("/2/nodes", _VerifyNodes, "GET", None),
205 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
206 ("/2/groups", _VerifyGroups, "GET", None),
207 ("/2/groups?bulk=1", _VerifyGroupsBulk, "GET", None),
208 ("/2/instances", [], "GET", None),
209 ("/2/instances?bulk=1", [], "GET", None),
210 ("/2/os", None, "GET", None),
213 # Test HTTP Not Found
214 for method in ["GET", "PUT", "POST", "DELETE"]:
216 _DoTests([("/99/resource/not/here/99", None, method, None)])
217 except rapi.client.GanetiApiError, err:
218 AssertEqual(err.code, 404)
220 raise qa_error.Error("Non-existent resource didn't return HTTP 404")
222 # Test HTTP Not Implemented
223 for method in ["PUT", "POST", "DELETE"]:
225 _DoTests([("/version", None, method, None)])
226 except rapi.client.GanetiApiError, err:
227 AssertEqual(err.code, 501)
229 raise qa_error.Error("Non-implemented method didn't fail")
233 """Testing resource queries via remote API.
236 master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode())
237 rnd = random.Random(7818)
239 for what in constants.QR_VIA_RAPI:
240 if what == constants.QR_JOB:
242 elif what == constants.QR_EXPORT:
247 all_fields = query.ALL_FIELDS[what].keys()
248 rnd.shuffle(all_fields)
250 # No fields, should return everything
251 result = _rapi_client.QueryFields(what)
252 qresult = objects.QueryFieldsResponse.FromDict(result)
253 AssertEqual(len(qresult.fields), len(all_fields))
256 result = _rapi_client.QueryFields(what, fields=[namefield])
257 qresult = objects.QueryFieldsResponse.FromDict(result)
258 AssertEqual(len(qresult.fields), 1)
260 # Specify all fields, order must be correct
261 result = _rapi_client.QueryFields(what, fields=all_fields)
262 qresult = objects.QueryFieldsResponse.FromDict(result)
263 AssertEqual(len(qresult.fields), len(all_fields))
264 AssertEqual([fdef.name for fdef in qresult.fields], all_fields)
267 result = _rapi_client.QueryFields(what, fields=["_unknown!"])
268 qresult = objects.QueryFieldsResponse.FromDict(result)
269 AssertEqual(len(qresult.fields), 1)
270 AssertEqual(qresult.fields[0].name, "_unknown!")
271 AssertEqual(qresult.fields[0].kind, constants.QFT_UNKNOWN)
273 # Try once more, this time without the client
275 ("/2/query/%s/fields" % what, None, "GET", None),
276 ("/2/query/%s/fields?fields=name,name,%s" % (what, all_fields[0]),
280 # Try missing query argument
283 ("/2/query/%s" % what, None, "GET", None),
285 except rapi.client.GanetiApiError, err:
286 AssertEqual(err.code, 400)
288 raise qa_error.Error("Request missing 'fields' parameter didn't fail")
290 def _Check(exp_fields, data):
291 qresult = objects.QueryResponse.FromDict(data)
292 AssertEqual([fdef.name for fdef in qresult.fields], exp_fields)
293 if not isinstance(qresult.data, list):
294 raise qa_error.Error("Query did not return a list")
297 # Specify fields in query
298 ("/2/query/%s?fields=%s" % (what, ",".join(all_fields)),
299 compat.partial(_Check, all_fields), "GET", None),
301 ("/2/query/%s?fields=%s" % (what, namefield),
302 compat.partial(_Check, [namefield]), "GET", None),
305 ("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" %
306 (what, namefield, namefield, namefield),
307 compat.partial(_Check, [namefield] * 3), "GET", None),
309 # PUT with fields in query
310 ("/2/query/%s?fields=%s" % (what, namefield),
311 compat.partial(_Check, [namefield]), "PUT", {}),
314 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
315 "fields": all_fields,
318 ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", {
319 "fields": [namefield] * 4,
326 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
327 "fields": all_fields,
328 "filter": [qlang.OP_TRUE, namefield],
332 if what == constants.QR_LOCK:
333 # Locks can't be filtered
336 except rapi.client.GanetiApiError, err:
337 AssertEqual(err.code, 500)
339 raise qa_error.Error("Filtering locks didn't fail")
343 if what == constants.QR_NODE:
345 (nodes, ) = _DoTests([("/2/query/%s" % what,
346 compat.partial(_Check, ["name", "master"]), "PUT", {
347 "fields": ["name", "master"],
348 "filter": [qlang.OP_TRUE, "master"],
350 qresult = objects.QueryResponse.FromDict(nodes)
351 AssertEqual(qresult.data, [
352 [[constants.RS_NORMAL, master_name], [constants.RS_NORMAL, True]],
356 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
357 def TestInstance(instance):
358 """Testing getting instance(s) info via remote API.
361 def _VerifyInstance(data):
362 for entry in INSTANCE_FIELDS:
363 AssertIn(entry, data)
365 def _VerifyInstancesList(data):
366 for instance in data:
367 for entry in LIST_FIELDS:
368 AssertIn(entry, instance)
370 def _VerifyInstancesBulk(data):
371 for instance_data in data:
372 _VerifyInstance(instance_data)
375 ("/2/instances/%s" % instance["name"], _VerifyInstance, "GET", None),
376 ("/2/instances", _VerifyInstancesList, "GET", None),
377 ("/2/instances?bulk=1", _VerifyInstancesBulk, "GET", None),
378 ("/2/instances/%s/activate-disks" % instance["name"],
379 _VerifyReturnsJob, "PUT", None),
380 ("/2/instances/%s/deactivate-disks" % instance["name"],
381 _VerifyReturnsJob, "PUT", None),
384 # Test OpBackupPrepare
385 (job_id, ) = _DoTests([
386 ("/2/instances/%s/prepare-export?mode=%s" %
387 (instance["name"], constants.EXPORT_MODE_REMOTE),
388 _VerifyReturnsJob, "PUT", None),
391 result = _WaitForRapiJob(job_id)[0]
392 AssertEqual(len(result["handshake"]), 3)
393 AssertEqual(result["handshake"][0], constants.RIE_VERSION)
394 AssertEqual(len(result["x509_key_name"]), 3)
395 AssertIn("-----BEGIN CERTIFICATE-----", result["x509_ca"])
399 """Testing getting node(s) info via remote API.
402 def _VerifyNode(data):
403 for entry in NODE_FIELDS:
404 AssertIn(entry, data)
406 def _VerifyNodesList(data):
408 for entry in LIST_FIELDS:
409 AssertIn(entry, node)
411 def _VerifyNodesBulk(data):
412 for node_data in data:
413 _VerifyNode(node_data)
416 ("/2/nodes/%s" % node["primary"], _VerifyNode, "GET", None),
417 ("/2/nodes", _VerifyNodesList, "GET", None),
418 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
422 def _FilterTags(seq):
423 """Removes unwanted tags from a sequence.
426 ignore_re = qa_config.get("ignore-tags-re", None)
429 return itertools.ifilterfalse(re.compile(ignore_re).match, seq)
434 def TestTags(kind, name, tags):
435 """Tests .../tags resources.
438 if kind == constants.TAG_CLUSTER:
440 elif kind == constants.TAG_NODE:
441 uri = "/2/nodes/%s/tags" % name
442 elif kind == constants.TAG_INSTANCE:
443 uri = "/2/instances/%s/tags" % name
444 elif kind == constants.TAG_NODEGROUP:
445 uri = "/2/groups/%s/tags" % name
447 raise errors.ProgrammerError("Unknown tag kind")
449 def _VerifyTags(data):
450 AssertEqual(sorted(tags), sorted(_FilterTags(data)))
452 queryargs = "&".join("tag=%s" % i for i in tags)
455 (job_id, ) = _DoTests([
456 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "PUT", None),
458 _WaitForRapiJob(job_id)
462 (uri, _VerifyTags, "GET", None),
466 (job_id, ) = _DoTests([
467 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "DELETE", None),
469 _WaitForRapiJob(job_id)
472 def _WaitForRapiJob(job_id):
473 """Waits for a job to finish.
476 def _VerifyJob(data):
477 AssertEqual(data["id"], job_id)
478 for field in JOB_FIELDS:
479 AssertIn(field, data)
482 ("/2/jobs/%s" % job_id, _VerifyJob, "GET", None),
485 return rapi.client_utils.PollJob(_rapi_client, job_id,
486 cli.StdioJobPollReportCb())
489 def TestRapiNodeGroups():
490 """Test several node group operations using RAPI.
493 groups = qa_config.get("groups", {})
494 group1, group2, group3 = groups.get("inexistent-groups",
495 ["group1", "group2", "group3"])[:3]
497 # Create a group with no attributes
502 (job_id, ) = _DoTests([
503 ("/2/groups", _VerifyReturnsJob, "POST", body),
506 _WaitForRapiJob(job_id)
508 # Create a group specifying alloc_policy
511 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
514 (job_id, ) = _DoTests([
515 ("/2/groups", _VerifyReturnsJob, "POST", body),
518 _WaitForRapiJob(job_id)
520 # Modify alloc_policy
522 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
525 (job_id, ) = _DoTests([
526 ("/2/groups/%s/modify" % group1, _VerifyReturnsJob, "PUT", body),
529 _WaitForRapiJob(job_id)
536 (job_id, ) = _DoTests([
537 ("/2/groups/%s/rename" % group2, _VerifyReturnsJob, "PUT", body),
540 _WaitForRapiJob(job_id)
543 for group in [group1, group3]:
544 (job_id, ) = _DoTests([
545 ("/2/groups/%s" % group, _VerifyReturnsJob, "DELETE", None),
548 _WaitForRapiJob(job_id)
551 def TestRapiInstanceAdd(node, use_client):
552 """Test adding a new instance via RAPI"""
553 instance = qa_config.AcquireInstance()
555 disk_sizes = [utils.ParseUnit(size) for size in qa_config.get("disk")]
556 disks = [{"size": size} for size in disk_sizes]
557 nic0_mac = qa_config.GetInstanceNicMac(instance,
558 default=constants.VALUE_GENERATE)
560 constants.INIC_MAC: nic0_mac,
564 constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)),
565 constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)),
569 job_id = _rapi_client.CreateInstance(constants.INSTANCE_CREATE,
573 os=qa_config.get("os"),
574 pnode=node["primary"],
579 "mode": constants.INSTANCE_CREATE,
580 "name": instance["name"],
581 "os_type": qa_config.get("os"),
582 "disk_template": constants.DT_PLAIN,
583 "pnode": node["primary"],
584 "beparams": beparams,
589 (job_id, ) = _DoTests([
590 ("/2/instances", _VerifyReturnsJob, "POST", body),
593 _WaitForRapiJob(job_id)
597 qa_config.ReleaseInstance(instance)
601 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
602 def TestRapiInstanceRemove(instance, use_client):
603 """Test removing instance via RAPI"""
605 job_id = _rapi_client.DeleteInstance(instance["name"])
607 (job_id, ) = _DoTests([
608 ("/2/instances/%s" % instance["name"], _VerifyReturnsJob, "DELETE", None),
611 _WaitForRapiJob(job_id)
613 qa_config.ReleaseInstance(instance)
616 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
617 def TestRapiInstanceMigrate(instance):
618 """Test migrating instance via RAPI"""
619 # Move to secondary node
620 _WaitForRapiJob(_rapi_client.MigrateInstance(instance["name"]))
621 qa_utils.RunInstanceCheck(instance, True)
622 # And back to previous primary
623 _WaitForRapiJob(_rapi_client.MigrateInstance(instance["name"]))
626 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
627 def TestRapiInstanceFailover(instance):
628 """Test failing over instance via RAPI"""
629 # Move to secondary node
630 _WaitForRapiJob(_rapi_client.FailoverInstance(instance["name"]))
631 qa_utils.RunInstanceCheck(instance, True)
632 # And back to previous primary
633 _WaitForRapiJob(_rapi_client.FailoverInstance(instance["name"]))
636 @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
637 def TestRapiInstanceShutdown(instance):
638 """Test stopping an instance via RAPI"""
639 _WaitForRapiJob(_rapi_client.ShutdownInstance(instance["name"]))
642 @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
643 def TestRapiInstanceStartup(instance):
644 """Test starting an instance via RAPI"""
645 _WaitForRapiJob(_rapi_client.StartupInstance(instance["name"]))
648 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
649 def TestRapiInstanceRenameAndBack(rename_source, rename_target):
650 """Test renaming instance via RAPI
652 This must leave the instance with the original name (in the
656 _WaitForRapiJob(_rapi_client.RenameInstance(rename_source, rename_target))
657 qa_utils.RunInstanceCheck(rename_source, False)
658 qa_utils.RunInstanceCheck(rename_target, False)
659 _WaitForRapiJob(_rapi_client.RenameInstance(rename_target, rename_source))
660 qa_utils.RunInstanceCheck(rename_target, False)
663 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
664 def TestRapiInstanceReinstall(instance):
665 """Test reinstalling an instance via RAPI"""
666 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance["name"]))
667 # By default, the instance is started again
668 qa_utils.RunInstanceCheck(instance, True)
670 # Reinstall again without starting
671 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance["name"],
675 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
676 def TestRapiInstanceReplaceDisks(instance):
677 """Test replacing instance disks via RAPI"""
678 _WaitForRapiJob(_rapi_client.ReplaceInstanceDisks(instance["name"],
679 mode=constants.REPLACE_DISK_AUTO, disks=[]))
680 _WaitForRapiJob(_rapi_client.ReplaceInstanceDisks(instance["name"],
681 mode=constants.REPLACE_DISK_SEC, disks="0"))
684 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
685 def TestRapiInstanceModify(instance):
686 """Test modifying instance via RAPI"""
687 default_hv = qa_config.GetDefaultHypervisor()
689 def _ModifyInstance(**kwargs):
690 _WaitForRapiJob(_rapi_client.ModifyInstance(instance["name"], **kwargs))
692 _ModifyInstance(beparams={
693 constants.BE_VCPUS: 3,
696 _ModifyInstance(beparams={
697 constants.BE_VCPUS: constants.VALUE_DEFAULT,
700 if default_hv == constants.HT_XEN_PVM:
701 _ModifyInstance(hvparams={
702 constants.HV_KERNEL_ARGS: "single",
704 _ModifyInstance(hvparams={
705 constants.HV_KERNEL_ARGS: constants.VALUE_DEFAULT,
707 elif default_hv == constants.HT_XEN_HVM:
708 _ModifyInstance(hvparams={
709 constants.HV_BOOT_ORDER: "acn",
711 _ModifyInstance(hvparams={
712 constants.HV_BOOT_ORDER: constants.VALUE_DEFAULT,
716 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
717 def TestRapiInstanceConsole(instance):
718 """Test getting instance console information via RAPI"""
719 result = _rapi_client.GetInstanceConsole(instance["name"])
720 console = objects.InstanceConsole.FromDict(result)
721 AssertEqual(console.Validate(), True)
722 AssertEqual(console.instance, qa_utils.ResolveInstanceName(instance["name"]))
725 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
726 def TestRapiStoppedInstanceConsole(instance):
727 """Test getting stopped instance's console information via RAPI"""
729 _rapi_client.GetInstanceConsole(instance["name"])
730 except rapi.client.GanetiApiError, err:
731 AssertEqual(err.code, 503)
733 raise qa_error.Error("Getting console for stopped instance didn't"
737 def GetOperatingSystems():
738 """Retrieves a list of all available operating systems.
741 return _rapi_client.GetOperatingSystems()
744 def TestInterClusterInstanceMove(src_instance, dest_instance,
745 pnode, snode, tnode):
746 """Test tools/move-instance"""
747 master = qa_config.GetMasterNode()
749 rapi_pw_file = tempfile.NamedTemporaryFile()
750 rapi_pw_file.write(_rapi_password)
753 # TODO: Run some instance tests before moving back
756 # instance is not redundant, but we still need to pass a node
757 # (which will be ignored)
761 # note: pnode:snode are the *current* nodes, so we move it first to
762 # tnode:pnode, then back to pnode:snode
763 for si, di, pn, sn in [(src_instance["name"], dest_instance["name"],
764 tnode["primary"], pnode["primary"]),
765 (dest_instance["name"], src_instance["name"],
766 pnode["primary"], fsec["primary"])]:
768 "../tools/move-instance",
770 "--src-ca-file=%s" % _rapi_ca.name,
771 "--src-username=%s" % _rapi_username,
772 "--src-password-file=%s" % rapi_pw_file.name,
773 "--dest-instance-name=%s" % di,
774 "--dest-primary-node=%s" % pn,
775 "--dest-secondary-node=%s" % sn,
776 "--net=0:mac=%s" % constants.VALUE_GENERATE,
782 qa_utils.RunInstanceCheck(di, False)
783 AssertEqual(StartLocalCommand(cmd).wait(), 0)
784 qa_utils.RunInstanceCheck(si, False)
785 qa_utils.RunInstanceCheck(di, True)