4 # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Remote API QA tests.
31 from ganeti import utils
32 from ganeti import constants
33 from ganeti import errors
34 from ganeti import cli
35 from ganeti import rapi
36 from ganeti import objects
37 from ganeti import query
38 from ganeti import compat
39 from ganeti import qlang
40 from ganeti import pathutils
42 import ganeti.rapi.client # pylint: disable=W0611
43 import ganeti.rapi.client_utils
49 from qa_instance import IsFailoverSupported
50 from qa_instance import IsMigrationSupported
51 from qa_instance import IsDiskReplacingSupported
52 from qa_utils import (AssertEqual, AssertIn, AssertMatch, StartLocalCommand)
53 from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG
62 def Setup(username, password):
63 """Configures the RAPI client.
66 # pylint: disable=W0603
73 _rapi_username = username
74 _rapi_password = password
76 master = qa_config.GetMasterNode()
78 # Load RAPI certificate from master node
79 cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_CERT_FILE)]
81 # Write to temporary file
82 _rapi_ca = tempfile.NamedTemporaryFile()
83 _rapi_ca.write(qa_utils.GetCommandOutput(master.primary,
84 utils.ShellQuoteArgs(cmd)))
87 port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT)
88 cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name,
91 if qa_config.UseVirtualCluster():
92 # TODO: Implement full support for RAPI on virtual clusters
93 print qa_utils.FormatWarning("RAPI tests are not yet supported on"
94 " virtual clusters and will be disabled")
96 assert _rapi_client is None
98 _rapi_client = rapi.client.GanetiRapiClient(master.primary, port=port,
101 curl_config_fn=cfg_curl)
103 print "RAPI protocol version: %s" % _rapi_client.GetVersion()
106 INSTANCE_FIELDS = ("name", "os", "pnode", "snodes",
108 "disk_template", "disk.sizes", "disk.spindles",
109 "nic.ips", "nic.macs", "nic.modes", "nic.links",
110 "beparams", "hvparams",
111 "oper_state", "oper_ram", "oper_vcpus", "status", "tags")
113 NODE_FIELDS = ("name", "dtotal", "dfree", "sptotal", "spfree",
114 "mtotal", "mnode", "mfree",
115 "pinst_cnt", "sinst_cnt", "tags")
117 GROUP_FIELDS = compat.UniqueFrozenset([
120 "node_cnt", "node_list",
123 JOB_FIELDS = compat.UniqueFrozenset([
124 "id", "ops", "status", "summary",
125 "opstatus", "opresult", "oplog",
126 "received_ts", "start_ts", "end_ts",
129 LIST_FIELDS = ("id", "uri")
133 """Return whether remote API tests should be run.
136 # TODO: Implement RAPI tests for virtual clusters
137 return (qa_config.TestEnabled("rapi") and
138 not qa_config.UseVirtualCluster())
142 # pylint: disable=W0212
143 # due to _SendRequest usage
146 for uri, verify, method, body in uris:
147 assert uri.startswith("/")
149 print "%s %s" % (method, uri)
150 data = _rapi_client._SendRequest(method, uri, None, body)
152 if verify is not None:
156 AssertEqual(data, verify)
163 def _VerifyReturnsJob(data):
164 if not isinstance(data, int):
165 AssertMatch(data, r"^\d+$")
169 """Testing remote API version.
173 ("/version", constants.RAPI_VERSION, "GET", None),
177 def TestEmptyCluster():
178 """Testing remote API on an empty cluster.
181 master = qa_config.GetMasterNode()
182 master_full = qa_utils.ResolveNodeName(master)
184 def _VerifyInfo(data):
185 AssertIn("name", data)
186 AssertIn("master", data)
187 AssertEqual(data["master"], master_full)
189 def _VerifyNodes(data):
192 "uri": "/2/nodes/%s" % master_full,
194 AssertIn(master_entry, data)
196 def _VerifyNodesBulk(data):
198 for entry in NODE_FIELDS:
199 AssertIn(entry, node)
201 def _VerifyGroups(data):
203 "name": constants.INITIAL_NODE_GROUP_NAME,
204 "uri": "/2/groups/" + constants.INITIAL_NODE_GROUP_NAME,
206 AssertIn(default_group, data)
208 def _VerifyGroupsBulk(data):
210 for field in GROUP_FIELDS:
211 AssertIn(field, group)
214 ("/", None, "GET", None),
215 ("/2/info", _VerifyInfo, "GET", None),
216 ("/2/tags", None, "GET", None),
217 ("/2/nodes", _VerifyNodes, "GET", None),
218 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
219 ("/2/groups", _VerifyGroups, "GET", None),
220 ("/2/groups?bulk=1", _VerifyGroupsBulk, "GET", None),
221 ("/2/instances", [], "GET", None),
222 ("/2/instances?bulk=1", [], "GET", None),
223 ("/2/os", None, "GET", None),
226 # Test HTTP Not Found
227 for method in ["GET", "PUT", "POST", "DELETE"]:
229 _DoTests([("/99/resource/not/here/99", None, method, None)])
230 except rapi.client.GanetiApiError, err:
231 AssertEqual(err.code, 404)
233 raise qa_error.Error("Non-existent resource didn't return HTTP 404")
235 # Test HTTP Not Implemented
236 for method in ["PUT", "POST", "DELETE"]:
238 _DoTests([("/version", None, method, None)])
239 except rapi.client.GanetiApiError, err:
240 AssertEqual(err.code, 501)
242 raise qa_error.Error("Non-implemented method didn't fail")
246 """Testing resource queries via remote API.
249 master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode())
250 rnd = random.Random(7818)
252 for what in constants.QR_VIA_RAPI:
253 if what == constants.QR_JOB:
255 elif what == constants.QR_EXPORT:
260 all_fields = query.ALL_FIELDS[what].keys()
261 rnd.shuffle(all_fields)
263 # No fields, should return everything
264 result = _rapi_client.QueryFields(what)
265 qresult = objects.QueryFieldsResponse.FromDict(result)
266 AssertEqual(len(qresult.fields), len(all_fields))
269 result = _rapi_client.QueryFields(what, fields=[namefield])
270 qresult = objects.QueryFieldsResponse.FromDict(result)
271 AssertEqual(len(qresult.fields), 1)
273 # Specify all fields, order must be correct
274 result = _rapi_client.QueryFields(what, fields=all_fields)
275 qresult = objects.QueryFieldsResponse.FromDict(result)
276 AssertEqual(len(qresult.fields), len(all_fields))
277 AssertEqual([fdef.name for fdef in qresult.fields], all_fields)
280 result = _rapi_client.QueryFields(what, fields=["_unknown!"])
281 qresult = objects.QueryFieldsResponse.FromDict(result)
282 AssertEqual(len(qresult.fields), 1)
283 AssertEqual(qresult.fields[0].name, "_unknown!")
284 AssertEqual(qresult.fields[0].kind, constants.QFT_UNKNOWN)
286 # Try once more, this time without the client
288 ("/2/query/%s/fields" % what, None, "GET", None),
289 ("/2/query/%s/fields?fields=name,name,%s" % (what, all_fields[0]),
293 # Try missing query argument
296 ("/2/query/%s" % what, None, "GET", None),
298 except rapi.client.GanetiApiError, err:
299 AssertEqual(err.code, 400)
301 raise qa_error.Error("Request missing 'fields' parameter didn't fail")
303 def _Check(exp_fields, data):
304 qresult = objects.QueryResponse.FromDict(data)
305 AssertEqual([fdef.name for fdef in qresult.fields], exp_fields)
306 if not isinstance(qresult.data, list):
307 raise qa_error.Error("Query did not return a list")
310 # Specify fields in query
311 ("/2/query/%s?fields=%s" % (what, ",".join(all_fields)),
312 compat.partial(_Check, all_fields), "GET", None),
314 ("/2/query/%s?fields=%s" % (what, namefield),
315 compat.partial(_Check, [namefield]), "GET", None),
318 ("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" %
319 (what, namefield, namefield, namefield),
320 compat.partial(_Check, [namefield] * 3), "GET", None),
322 # PUT with fields in query
323 ("/2/query/%s?fields=%s" % (what, namefield),
324 compat.partial(_Check, [namefield]), "PUT", {}),
327 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
328 "fields": all_fields,
331 ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", {
332 "fields": [namefield] * 4,
339 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
340 "fields": all_fields,
341 "filter": [qlang.OP_TRUE, namefield],
345 if what == constants.QR_LOCK:
346 # Locks can't be filtered
349 except rapi.client.GanetiApiError, err:
350 AssertEqual(err.code, 500)
352 raise qa_error.Error("Filtering locks didn't fail")
356 if what == constants.QR_NODE:
358 (nodes, ) = _DoTests(
359 [("/2/query/%s" % what,
360 compat.partial(_Check, ["name", "master"]), "PUT",
361 {"fields": ["name", "master"],
362 "filter": [qlang.OP_TRUE, "master"],
364 qresult = objects.QueryResponse.FromDict(nodes)
365 AssertEqual(qresult.data, [
366 [[constants.RS_NORMAL, master_name], [constants.RS_NORMAL, True]],
370 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
371 def TestInstance(instance):
372 """Testing getting instance(s) info via remote API.
375 def _VerifyInstance(data):
376 for entry in INSTANCE_FIELDS:
377 AssertIn(entry, data)
379 def _VerifyInstancesList(data):
380 for instance in data:
381 for entry in LIST_FIELDS:
382 AssertIn(entry, instance)
384 def _VerifyInstancesBulk(data):
385 for instance_data in data:
386 _VerifyInstance(instance_data)
389 ("/2/instances/%s" % instance.name, _VerifyInstance, "GET", None),
390 ("/2/instances", _VerifyInstancesList, "GET", None),
391 ("/2/instances?bulk=1", _VerifyInstancesBulk, "GET", None),
392 ("/2/instances/%s/activate-disks" % instance.name,
393 _VerifyReturnsJob, "PUT", None),
394 ("/2/instances/%s/deactivate-disks" % instance.name,
395 _VerifyReturnsJob, "PUT", None),
398 # Test OpBackupPrepare
399 (job_id, ) = _DoTests([
400 ("/2/instances/%s/prepare-export?mode=%s" %
401 (instance.name, constants.EXPORT_MODE_REMOTE),
402 _VerifyReturnsJob, "PUT", None),
405 result = _WaitForRapiJob(job_id)[0]
406 AssertEqual(len(result["handshake"]), 3)
407 AssertEqual(result["handshake"][0], constants.RIE_VERSION)
408 AssertEqual(len(result["x509_key_name"]), 3)
409 AssertIn("-----BEGIN CERTIFICATE-----", result["x509_ca"])
413 """Testing getting node(s) info via remote API.
416 def _VerifyNode(data):
417 for entry in NODE_FIELDS:
418 AssertIn(entry, data)
420 def _VerifyNodesList(data):
422 for entry in LIST_FIELDS:
423 AssertIn(entry, node)
425 def _VerifyNodesBulk(data):
426 for node_data in data:
427 _VerifyNode(node_data)
430 ("/2/nodes/%s" % node.primary, _VerifyNode, "GET", None),
431 ("/2/nodes", _VerifyNodesList, "GET", None),
432 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
436 def _FilterTags(seq):
437 """Removes unwanted tags from a sequence.
440 ignore_re = qa_config.get("ignore-tags-re", None)
443 return itertools.ifilterfalse(re.compile(ignore_re).match, seq)
448 def TestTags(kind, name, tags):
449 """Tests .../tags resources.
452 if kind == constants.TAG_CLUSTER:
454 elif kind == constants.TAG_NODE:
455 uri = "/2/nodes/%s/tags" % name
456 elif kind == constants.TAG_INSTANCE:
457 uri = "/2/instances/%s/tags" % name
458 elif kind == constants.TAG_NODEGROUP:
459 uri = "/2/groups/%s/tags" % name
461 raise errors.ProgrammerError("Unknown tag kind")
463 def _VerifyTags(data):
464 AssertEqual(sorted(tags), sorted(_FilterTags(data)))
466 queryargs = "&".join("tag=%s" % i for i in tags)
469 (job_id, ) = _DoTests([
470 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "PUT", None),
472 _WaitForRapiJob(job_id)
476 (uri, _VerifyTags, "GET", None),
480 (job_id, ) = _DoTests([
481 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "DELETE", None),
483 _WaitForRapiJob(job_id)
486 def _WaitForRapiJob(job_id):
487 """Waits for a job to finish.
490 def _VerifyJob(data):
491 AssertEqual(data["id"], job_id)
492 for field in JOB_FIELDS:
493 AssertIn(field, data)
496 ("/2/jobs/%s" % job_id, _VerifyJob, "GET", None),
499 return rapi.client_utils.PollJob(_rapi_client, job_id,
500 cli.StdioJobPollReportCb())
503 def TestRapiNodeGroups():
504 """Test several node group operations using RAPI.
507 (group1, group2, group3) = qa_utils.GetNonexistentGroups(3)
509 # Create a group with no attributes
514 (job_id, ) = _DoTests([
515 ("/2/groups", _VerifyReturnsJob, "POST", body),
518 _WaitForRapiJob(job_id)
520 # Create a group specifying alloc_policy
523 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
526 (job_id, ) = _DoTests([
527 ("/2/groups", _VerifyReturnsJob, "POST", body),
530 _WaitForRapiJob(job_id)
532 # Modify alloc_policy
534 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
537 (job_id, ) = _DoTests([
538 ("/2/groups/%s/modify" % group1, _VerifyReturnsJob, "PUT", body),
541 _WaitForRapiJob(job_id)
548 (job_id, ) = _DoTests([
549 ("/2/groups/%s/rename" % group2, _VerifyReturnsJob, "PUT", body),
552 _WaitForRapiJob(job_id)
555 for group in [group1, group3]:
556 (job_id, ) = _DoTests([
557 ("/2/groups/%s" % group, _VerifyReturnsJob, "DELETE", None),
560 _WaitForRapiJob(job_id)
563 def TestRapiInstanceAdd(node, use_client):
564 """Test adding a new instance via RAPI"""
565 instance = qa_config.AcquireInstance()
566 instance.SetDiskTemplate(constants.DT_PLAIN)
568 disks = [{"size": utils.ParseUnit(d.get("size")),
569 "name": str(d.get("name"))}
570 for d in qa_config.GetDiskOptions()]
571 nic0_mac = instance.GetNicMacAddr(0, constants.VALUE_GENERATE)
573 constants.INIC_MAC: nic0_mac,
577 constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)),
578 constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)),
582 job_id = _rapi_client.CreateInstance(constants.INSTANCE_CREATE,
586 os=qa_config.get("os"),
592 "mode": constants.INSTANCE_CREATE,
593 "name": instance.name,
594 "os_type": qa_config.get("os"),
595 "disk_template": constants.DT_PLAIN,
596 "pnode": node.primary,
597 "beparams": beparams,
602 (job_id, ) = _DoTests([
603 ("/2/instances", _VerifyReturnsJob, "POST", body),
606 _WaitForRapiJob(job_id)
614 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
615 def TestRapiInstanceRemove(instance, use_client):
616 """Test removing instance via RAPI"""
618 job_id = _rapi_client.DeleteInstance(instance.name)
620 (job_id, ) = _DoTests([
621 ("/2/instances/%s" % instance.name, _VerifyReturnsJob, "DELETE", None),
624 _WaitForRapiJob(job_id)
627 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
628 def TestRapiInstanceMigrate(instance):
629 """Test migrating instance via RAPI"""
630 if not IsMigrationSupported(instance):
631 print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
634 # Move to secondary node
635 _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name))
636 qa_utils.RunInstanceCheck(instance, True)
637 # And back to previous primary
638 _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name))
641 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
642 def TestRapiInstanceFailover(instance):
643 """Test failing over instance via RAPI"""
644 if not IsFailoverSupported(instance):
645 print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
648 # Move to secondary node
649 _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name))
650 qa_utils.RunInstanceCheck(instance, True)
651 # And back to previous primary
652 _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name))
655 @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
656 def TestRapiInstanceShutdown(instance):
657 """Test stopping an instance via RAPI"""
658 _WaitForRapiJob(_rapi_client.ShutdownInstance(instance.name))
661 @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
662 def TestRapiInstanceStartup(instance):
663 """Test starting an instance via RAPI"""
664 _WaitForRapiJob(_rapi_client.StartupInstance(instance.name))
667 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
668 def TestRapiInstanceRenameAndBack(rename_source, rename_target):
669 """Test renaming instance via RAPI
671 This must leave the instance with the original name (in the
675 _WaitForRapiJob(_rapi_client.RenameInstance(rename_source, rename_target))
676 qa_utils.RunInstanceCheck(rename_source, False)
677 qa_utils.RunInstanceCheck(rename_target, False)
678 _WaitForRapiJob(_rapi_client.RenameInstance(rename_target, rename_source))
679 qa_utils.RunInstanceCheck(rename_target, False)
682 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
683 def TestRapiInstanceReinstall(instance):
684 """Test reinstalling an instance via RAPI"""
685 if instance.disk_template == constants.DT_DISKLESS:
686 print qa_utils.FormatInfo("Test not supported for diskless instances")
689 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name))
690 # By default, the instance is started again
691 qa_utils.RunInstanceCheck(instance, True)
693 # Reinstall again without starting
694 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name,
698 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
699 def TestRapiInstanceReplaceDisks(instance):
700 """Test replacing instance disks via RAPI"""
701 if not IsDiskReplacingSupported(instance):
702 print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
705 fn = _rapi_client.ReplaceInstanceDisks
706 _WaitForRapiJob(fn(instance.name,
707 mode=constants.REPLACE_DISK_AUTO, disks=[]))
708 _WaitForRapiJob(fn(instance.name,
709 mode=constants.REPLACE_DISK_SEC, disks="0"))
712 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
713 def TestRapiInstanceModify(instance):
714 """Test modifying instance via RAPI"""
715 default_hv = qa_config.GetDefaultHypervisor()
717 def _ModifyInstance(**kwargs):
718 _WaitForRapiJob(_rapi_client.ModifyInstance(instance.name, **kwargs))
720 _ModifyInstance(beparams={
721 constants.BE_VCPUS: 3,
724 _ModifyInstance(beparams={
725 constants.BE_VCPUS: constants.VALUE_DEFAULT,
728 if default_hv == constants.HT_XEN_PVM:
729 _ModifyInstance(hvparams={
730 constants.HV_KERNEL_ARGS: "single",
732 _ModifyInstance(hvparams={
733 constants.HV_KERNEL_ARGS: constants.VALUE_DEFAULT,
735 elif default_hv == constants.HT_XEN_HVM:
736 _ModifyInstance(hvparams={
737 constants.HV_BOOT_ORDER: "acn",
739 _ModifyInstance(hvparams={
740 constants.HV_BOOT_ORDER: constants.VALUE_DEFAULT,
744 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
745 def TestRapiInstanceConsole(instance):
746 """Test getting instance console information via RAPI"""
747 result = _rapi_client.GetInstanceConsole(instance.name)
748 console = objects.InstanceConsole.FromDict(result)
749 AssertEqual(console.Validate(), True)
750 AssertEqual(console.instance, qa_utils.ResolveInstanceName(instance.name))
753 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
754 def TestRapiStoppedInstanceConsole(instance):
755 """Test getting stopped instance's console information via RAPI"""
757 _rapi_client.GetInstanceConsole(instance.name)
758 except rapi.client.GanetiApiError, err:
759 AssertEqual(err.code, 503)
761 raise qa_error.Error("Getting console for stopped instance didn't"
765 def GetOperatingSystems():
766 """Retrieves a list of all available operating systems.
769 return _rapi_client.GetOperatingSystems()
772 def TestInterClusterInstanceMove(src_instance, dest_instance,
774 """Test tools/move-instance"""
775 master = qa_config.GetMasterNode()
777 rapi_pw_file = tempfile.NamedTemporaryFile()
778 rapi_pw_file.write(_rapi_password)
781 dest_instance.SetDiskTemplate(src_instance.disk_template)
783 # TODO: Run some instance tests before moving back
786 # No disk template currently requires more than 1 secondary node. If this
787 # changes, either this test must be skipped or the script must be updated.
788 assert len(inodes) == 2
791 # instance is not redundant, but we still need to pass a node
792 # (which will be ignored)
795 # note: pnode:snode are the *current* nodes, so we move it first to
796 # tnode:pnode, then back to pnode:snode
797 for si, di, pn, sn in [(src_instance.name, dest_instance.name,
798 tnode.primary, pnode.primary),
799 (dest_instance.name, src_instance.name,
800 pnode.primary, snode.primary)]:
802 "../tools/move-instance",
804 "--src-ca-file=%s" % _rapi_ca.name,
805 "--src-username=%s" % _rapi_username,
806 "--src-password-file=%s" % rapi_pw_file.name,
807 "--dest-instance-name=%s" % di,
808 "--dest-primary-node=%s" % pn,
809 "--dest-secondary-node=%s" % sn,
810 "--net=0:mac=%s" % constants.VALUE_GENERATE,
816 qa_utils.RunInstanceCheck(di, False)
817 AssertEqual(StartLocalCommand(cmd).wait(), 0)
818 qa_utils.RunInstanceCheck(si, False)
819 qa_utils.RunInstanceCheck(di, True)