4 # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Remote API QA tests.
31 from ganeti import utils
32 from ganeti import constants
33 from ganeti import errors
34 from ganeti import cli
35 from ganeti import rapi
36 from ganeti import objects
37 from ganeti import query
38 from ganeti import compat
39 from ganeti import qlang
40 from ganeti import pathutils
42 import ganeti.rapi.client # pylint: disable=W0611
43 import ganeti.rapi.client_utils
49 from qa_instance import IsFailoverSupported
50 from qa_instance import IsMigrationSupported
51 from qa_instance import IsDiskReplacingSupported
52 from qa_utils import (AssertEqual, AssertIn, AssertMatch, StartLocalCommand)
53 from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG
62 def Setup(username, password):
63 """Configures the RAPI client.
66 # pylint: disable=W0603
73 _rapi_username = username
74 _rapi_password = password
76 master = qa_config.GetMasterNode()
78 # Load RAPI certificate from master node
79 cmd = ["cat", qa_utils.MakeNodePath(master, pathutils.RAPI_CERT_FILE)]
81 # Write to temporary file
82 _rapi_ca = tempfile.NamedTemporaryFile()
83 _rapi_ca.write(qa_utils.GetCommandOutput(master.primary,
84 utils.ShellQuoteArgs(cmd)))
87 port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT)
88 cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name,
91 if qa_config.UseVirtualCluster():
92 # TODO: Implement full support for RAPI on virtual clusters
93 print qa_utils.FormatWarning("RAPI tests are not yet supported on"
94 " virtual clusters and will be disabled")
96 assert _rapi_client is None
98 _rapi_client = rapi.client.GanetiRapiClient(master.primary, port=port,
101 curl_config_fn=cfg_curl)
103 print "RAPI protocol version: %s" % _rapi_client.GetVersion()
106 INSTANCE_FIELDS = ("name", "os", "pnode", "snodes",
108 "disk_template", "disk.sizes",
109 "nic.ips", "nic.macs", "nic.modes", "nic.links",
110 "beparams", "hvparams",
111 "oper_state", "oper_ram", "oper_vcpus", "status", "tags")
113 NODE_FIELDS = ("name", "dtotal", "dfree",
114 "mtotal", "mnode", "mfree",
115 "pinst_cnt", "sinst_cnt", "tags")
117 GROUP_FIELDS = compat.UniqueFrozenset([
120 "node_cnt", "node_list",
123 JOB_FIELDS = compat.UniqueFrozenset([
124 "id", "ops", "status", "summary",
125 "opstatus", "opresult", "oplog",
126 "received_ts", "start_ts", "end_ts",
129 LIST_FIELDS = ("id", "uri")
133 """Return whether remote API tests should be run.
136 # TODO: Implement RAPI tests for virtual clusters
137 return (qa_config.TestEnabled("rapi") and
138 not qa_config.UseVirtualCluster())
142 # pylint: disable=W0212
143 # due to _SendRequest usage
146 for uri, verify, method, body in uris:
147 assert uri.startswith("/")
149 print "%s %s" % (method, uri)
150 data = _rapi_client._SendRequest(method, uri, None, body)
152 if verify is not None:
156 AssertEqual(data, verify)
163 def _VerifyReturnsJob(data):
164 if not isinstance(data, int):
165 AssertMatch(data, r"^\d+$")
169 """Testing remote API version.
173 ("/version", constants.RAPI_VERSION, "GET", None),
177 def TestEmptyCluster():
178 """Testing remote API on an empty cluster.
181 master = qa_config.GetMasterNode()
182 master_full = qa_utils.ResolveNodeName(master)
184 def _VerifyInfo(data):
185 AssertIn("name", data)
186 AssertIn("master", data)
187 AssertEqual(data["master"], master_full)
189 def _VerifyNodes(data):
192 "uri": "/2/nodes/%s" % master_full,
194 AssertIn(master_entry, data)
196 def _VerifyNodesBulk(data):
198 for entry in NODE_FIELDS:
199 AssertIn(entry, node)
201 def _VerifyGroups(data):
203 "name": constants.INITIAL_NODE_GROUP_NAME,
204 "uri": "/2/groups/" + constants.INITIAL_NODE_GROUP_NAME,
206 AssertIn(default_group, data)
208 def _VerifyGroupsBulk(data):
210 for field in GROUP_FIELDS:
211 AssertIn(field, group)
214 ("/", None, "GET", None),
215 ("/2/info", _VerifyInfo, "GET", None),
216 ("/2/tags", None, "GET", None),
217 ("/2/nodes", _VerifyNodes, "GET", None),
218 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
219 ("/2/groups", _VerifyGroups, "GET", None),
220 ("/2/groups?bulk=1", _VerifyGroupsBulk, "GET", None),
221 ("/2/instances", [], "GET", None),
222 ("/2/instances?bulk=1", [], "GET", None),
223 ("/2/os", None, "GET", None),
226 # Test HTTP Not Found
227 for method in ["GET", "PUT", "POST", "DELETE"]:
229 _DoTests([("/99/resource/not/here/99", None, method, None)])
230 except rapi.client.GanetiApiError, err:
231 AssertEqual(err.code, 404)
233 raise qa_error.Error("Non-existent resource didn't return HTTP 404")
235 # Test HTTP Not Implemented
236 for method in ["PUT", "POST", "DELETE"]:
238 _DoTests([("/version", None, method, None)])
239 except rapi.client.GanetiApiError, err:
240 AssertEqual(err.code, 501)
242 raise qa_error.Error("Non-implemented method didn't fail")
246 """Testing resource queries via remote API.
249 master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode())
250 rnd = random.Random(7818)
252 for what in constants.QR_VIA_RAPI:
253 if what == constants.QR_JOB:
255 elif what == constants.QR_EXPORT:
260 all_fields = query.ALL_FIELDS[what].keys()
261 rnd.shuffle(all_fields)
263 # No fields, should return everything
264 result = _rapi_client.QueryFields(what)
265 qresult = objects.QueryFieldsResponse.FromDict(result)
266 AssertEqual(len(qresult.fields), len(all_fields))
269 result = _rapi_client.QueryFields(what, fields=[namefield])
270 qresult = objects.QueryFieldsResponse.FromDict(result)
271 AssertEqual(len(qresult.fields), 1)
273 # Specify all fields, order must be correct
274 result = _rapi_client.QueryFields(what, fields=all_fields)
275 qresult = objects.QueryFieldsResponse.FromDict(result)
276 AssertEqual(len(qresult.fields), len(all_fields))
277 AssertEqual([fdef.name for fdef in qresult.fields], all_fields)
280 result = _rapi_client.QueryFields(what, fields=["_unknown!"])
281 qresult = objects.QueryFieldsResponse.FromDict(result)
282 AssertEqual(len(qresult.fields), 1)
283 AssertEqual(qresult.fields[0].name, "_unknown!")
284 AssertEqual(qresult.fields[0].kind, constants.QFT_UNKNOWN)
286 # Try once more, this time without the client
288 ("/2/query/%s/fields" % what, None, "GET", None),
289 ("/2/query/%s/fields?fields=name,name,%s" % (what, all_fields[0]),
293 # Try missing query argument
296 ("/2/query/%s" % what, None, "GET", None),
298 except rapi.client.GanetiApiError, err:
299 AssertEqual(err.code, 400)
301 raise qa_error.Error("Request missing 'fields' parameter didn't fail")
303 def _Check(exp_fields, data):
304 qresult = objects.QueryResponse.FromDict(data)
305 AssertEqual([fdef.name for fdef in qresult.fields], exp_fields)
306 if not isinstance(qresult.data, list):
307 raise qa_error.Error("Query did not return a list")
310 # Specify fields in query
311 ("/2/query/%s?fields=%s" % (what, ",".join(all_fields)),
312 compat.partial(_Check, all_fields), "GET", None),
314 ("/2/query/%s?fields=%s" % (what, namefield),
315 compat.partial(_Check, [namefield]), "GET", None),
318 ("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" %
319 (what, namefield, namefield, namefield),
320 compat.partial(_Check, [namefield] * 3), "GET", None),
322 # PUT with fields in query
323 ("/2/query/%s?fields=%s" % (what, namefield),
324 compat.partial(_Check, [namefield]), "PUT", {}),
327 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
328 "fields": all_fields,
331 ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", {
332 "fields": [namefield] * 4,
339 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
340 "fields": all_fields,
341 "filter": [qlang.OP_TRUE, namefield],
345 if what == constants.QR_LOCK:
346 # Locks can't be filtered
349 except rapi.client.GanetiApiError, err:
350 AssertEqual(err.code, 500)
352 raise qa_error.Error("Filtering locks didn't fail")
356 if what == constants.QR_NODE:
358 (nodes, ) = _DoTests(
359 [("/2/query/%s" % what,
360 compat.partial(_Check, ["name", "master"]), "PUT",
361 {"fields": ["name", "master"],
362 "filter": [qlang.OP_TRUE, "master"],
364 qresult = objects.QueryResponse.FromDict(nodes)
365 AssertEqual(qresult.data, [
366 [[constants.RS_NORMAL, master_name], [constants.RS_NORMAL, True]],
370 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
371 def TestInstance(instance):
372 """Testing getting instance(s) info via remote API.
375 def _VerifyInstance(data):
376 for entry in INSTANCE_FIELDS:
377 AssertIn(entry, data)
379 def _VerifyInstancesList(data):
380 for instance in data:
381 for entry in LIST_FIELDS:
382 AssertIn(entry, instance)
384 def _VerifyInstancesBulk(data):
385 for instance_data in data:
386 _VerifyInstance(instance_data)
389 ("/2/instances/%s" % instance.name, _VerifyInstance, "GET", None),
390 ("/2/instances", _VerifyInstancesList, "GET", None),
391 ("/2/instances?bulk=1", _VerifyInstancesBulk, "GET", None),
392 ("/2/instances/%s/activate-disks" % instance.name,
393 _VerifyReturnsJob, "PUT", None),
394 ("/2/instances/%s/deactivate-disks" % instance.name,
395 _VerifyReturnsJob, "PUT", None),
398 # Test OpBackupPrepare
399 (job_id, ) = _DoTests([
400 ("/2/instances/%s/prepare-export?mode=%s" %
401 (instance.name, constants.EXPORT_MODE_REMOTE),
402 _VerifyReturnsJob, "PUT", None),
405 result = _WaitForRapiJob(job_id)[0]
406 AssertEqual(len(result["handshake"]), 3)
407 AssertEqual(result["handshake"][0], constants.RIE_VERSION)
408 AssertEqual(len(result["x509_key_name"]), 3)
409 AssertIn("-----BEGIN CERTIFICATE-----", result["x509_ca"])
413 """Testing getting node(s) info via remote API.
416 def _VerifyNode(data):
417 for entry in NODE_FIELDS:
418 AssertIn(entry, data)
420 def _VerifyNodesList(data):
422 for entry in LIST_FIELDS:
423 AssertIn(entry, node)
425 def _VerifyNodesBulk(data):
426 for node_data in data:
427 _VerifyNode(node_data)
430 ("/2/nodes/%s" % node.primary, _VerifyNode, "GET", None),
431 ("/2/nodes", _VerifyNodesList, "GET", None),
432 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
436 def _FilterTags(seq):
437 """Removes unwanted tags from a sequence.
440 ignore_re = qa_config.get("ignore-tags-re", None)
443 return itertools.ifilterfalse(re.compile(ignore_re).match, seq)
448 def TestTags(kind, name, tags):
449 """Tests .../tags resources.
452 if kind == constants.TAG_CLUSTER:
454 elif kind == constants.TAG_NODE:
455 uri = "/2/nodes/%s/tags" % name
456 elif kind == constants.TAG_INSTANCE:
457 uri = "/2/instances/%s/tags" % name
458 elif kind == constants.TAG_NODEGROUP:
459 uri = "/2/groups/%s/tags" % name
461 raise errors.ProgrammerError("Unknown tag kind")
463 def _VerifyTags(data):
464 AssertEqual(sorted(tags), sorted(_FilterTags(data)))
466 queryargs = "&".join("tag=%s" % i for i in tags)
469 (job_id, ) = _DoTests([
470 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "PUT", None),
472 _WaitForRapiJob(job_id)
476 (uri, _VerifyTags, "GET", None),
480 (job_id, ) = _DoTests([
481 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "DELETE", None),
483 _WaitForRapiJob(job_id)
486 def _WaitForRapiJob(job_id):
487 """Waits for a job to finish.
490 def _VerifyJob(data):
491 AssertEqual(data["id"], job_id)
492 for field in JOB_FIELDS:
493 AssertIn(field, data)
496 ("/2/jobs/%s" % job_id, _VerifyJob, "GET", None),
499 return rapi.client_utils.PollJob(_rapi_client, job_id,
500 cli.StdioJobPollReportCb())
503 def TestRapiNodeGroups():
504 """Test several node group operations using RAPI.
507 (group1, group2, group3) = qa_utils.GetNonexistentGroups(3)
509 # Create a group with no attributes
514 (job_id, ) = _DoTests([
515 ("/2/groups", _VerifyReturnsJob, "POST", body),
518 _WaitForRapiJob(job_id)
520 # Create a group specifying alloc_policy
523 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
526 (job_id, ) = _DoTests([
527 ("/2/groups", _VerifyReturnsJob, "POST", body),
530 _WaitForRapiJob(job_id)
532 # Modify alloc_policy
534 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
537 (job_id, ) = _DoTests([
538 ("/2/groups/%s/modify" % group1, _VerifyReturnsJob, "PUT", body),
541 _WaitForRapiJob(job_id)
548 (job_id, ) = _DoTests([
549 ("/2/groups/%s/rename" % group2, _VerifyReturnsJob, "PUT", body),
552 _WaitForRapiJob(job_id)
555 for group in [group1, group3]:
556 (job_id, ) = _DoTests([
557 ("/2/groups/%s" % group, _VerifyReturnsJob, "DELETE", None),
560 _WaitForRapiJob(job_id)
563 def TestRapiInstanceAdd(node, use_client):
564 """Test adding a new instance via RAPI"""
565 instance = qa_config.AcquireInstance()
566 instance.SetDiskTemplate(constants.DT_PLAIN)
568 disk_sizes = [utils.ParseUnit(size) for size in qa_config.get("disk")]
569 disks = [{"size": size} for size in disk_sizes]
570 nic0_mac = instance.GetNicMacAddr(0, constants.VALUE_GENERATE)
572 constants.INIC_MAC: nic0_mac,
576 constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)),
577 constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)),
581 job_id = _rapi_client.CreateInstance(constants.INSTANCE_CREATE,
585 os=qa_config.get("os"),
591 "mode": constants.INSTANCE_CREATE,
592 "name": instance.name,
593 "os_type": qa_config.get("os"),
594 "disk_template": constants.DT_PLAIN,
595 "pnode": node.primary,
596 "beparams": beparams,
601 (job_id, ) = _DoTests([
602 ("/2/instances", _VerifyReturnsJob, "POST", body),
605 _WaitForRapiJob(job_id)
613 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
614 def TestRapiInstanceRemove(instance, use_client):
615 """Test removing instance via RAPI"""
617 job_id = _rapi_client.DeleteInstance(instance.name)
619 (job_id, ) = _DoTests([
620 ("/2/instances/%s" % instance.name, _VerifyReturnsJob, "DELETE", None),
623 _WaitForRapiJob(job_id)
626 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
627 def TestRapiInstanceMigrate(instance):
628 """Test migrating instance via RAPI"""
629 if not IsMigrationSupported(instance):
630 print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
633 # Move to secondary node
634 _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name))
635 qa_utils.RunInstanceCheck(instance, True)
636 # And back to previous primary
637 _WaitForRapiJob(_rapi_client.MigrateInstance(instance.name))
640 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
641 def TestRapiInstanceFailover(instance):
642 """Test failing over instance via RAPI"""
643 if not IsFailoverSupported(instance):
644 print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
647 # Move to secondary node
648 _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name))
649 qa_utils.RunInstanceCheck(instance, True)
650 # And back to previous primary
651 _WaitForRapiJob(_rapi_client.FailoverInstance(instance.name))
654 @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
655 def TestRapiInstanceShutdown(instance):
656 """Test stopping an instance via RAPI"""
657 _WaitForRapiJob(_rapi_client.ShutdownInstance(instance.name))
660 @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
661 def TestRapiInstanceStartup(instance):
662 """Test starting an instance via RAPI"""
663 _WaitForRapiJob(_rapi_client.StartupInstance(instance.name))
666 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
667 def TestRapiInstanceRenameAndBack(rename_source, rename_target):
668 """Test renaming instance via RAPI
670 This must leave the instance with the original name (in the
674 _WaitForRapiJob(_rapi_client.RenameInstance(rename_source, rename_target))
675 qa_utils.RunInstanceCheck(rename_source, False)
676 qa_utils.RunInstanceCheck(rename_target, False)
677 _WaitForRapiJob(_rapi_client.RenameInstance(rename_target, rename_source))
678 qa_utils.RunInstanceCheck(rename_target, False)
681 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
682 def TestRapiInstanceReinstall(instance):
683 """Test reinstalling an instance via RAPI"""
684 if instance.disk_template == constants.DT_DISKLESS:
685 print qa_utils.FormatInfo("Test not supported for diskless instances")
688 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name))
689 # By default, the instance is started again
690 qa_utils.RunInstanceCheck(instance, True)
692 # Reinstall again without starting
693 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance.name,
697 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
698 def TestRapiInstanceReplaceDisks(instance):
699 """Test replacing instance disks via RAPI"""
700 if not IsDiskReplacingSupported(instance):
701 print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
704 fn = _rapi_client.ReplaceInstanceDisks
705 _WaitForRapiJob(fn(instance.name,
706 mode=constants.REPLACE_DISK_AUTO, disks=[]))
707 _WaitForRapiJob(fn(instance.name,
708 mode=constants.REPLACE_DISK_SEC, disks="0"))
711 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
712 def TestRapiInstanceModify(instance):
713 """Test modifying instance via RAPI"""
714 default_hv = qa_config.GetDefaultHypervisor()
716 def _ModifyInstance(**kwargs):
717 _WaitForRapiJob(_rapi_client.ModifyInstance(instance.name, **kwargs))
719 _ModifyInstance(beparams={
720 constants.BE_VCPUS: 3,
723 _ModifyInstance(beparams={
724 constants.BE_VCPUS: constants.VALUE_DEFAULT,
727 if default_hv == constants.HT_XEN_PVM:
728 _ModifyInstance(hvparams={
729 constants.HV_KERNEL_ARGS: "single",
731 _ModifyInstance(hvparams={
732 constants.HV_KERNEL_ARGS: constants.VALUE_DEFAULT,
734 elif default_hv == constants.HT_XEN_HVM:
735 _ModifyInstance(hvparams={
736 constants.HV_BOOT_ORDER: "acn",
738 _ModifyInstance(hvparams={
739 constants.HV_BOOT_ORDER: constants.VALUE_DEFAULT,
743 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
744 def TestRapiInstanceConsole(instance):
745 """Test getting instance console information via RAPI"""
746 result = _rapi_client.GetInstanceConsole(instance.name)
747 console = objects.InstanceConsole.FromDict(result)
748 AssertEqual(console.Validate(), True)
749 AssertEqual(console.instance, qa_utils.ResolveInstanceName(instance.name))
752 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
753 def TestRapiStoppedInstanceConsole(instance):
754 """Test getting stopped instance's console information via RAPI"""
756 _rapi_client.GetInstanceConsole(instance.name)
757 except rapi.client.GanetiApiError, err:
758 AssertEqual(err.code, 503)
760 raise qa_error.Error("Getting console for stopped instance didn't"
764 def GetOperatingSystems():
765 """Retrieves a list of all available operating systems.
768 return _rapi_client.GetOperatingSystems()
771 def TestInterClusterInstanceMove(src_instance, dest_instance,
773 """Test tools/move-instance"""
774 master = qa_config.GetMasterNode()
776 rapi_pw_file = tempfile.NamedTemporaryFile()
777 rapi_pw_file.write(_rapi_password)
780 dest_instance.SetDiskTemplate(src_instance.disk_template)
782 # TODO: Run some instance tests before moving back
785 # No disk template currently requires more than 1 secondary node. If this
786 # changes, either this test must be skipped or the script must be updated.
787 assert len(inodes) == 2
790 # instance is not redundant, but we still need to pass a node
791 # (which will be ignored)
794 # note: pnode:snode are the *current* nodes, so we move it first to
795 # tnode:pnode, then back to pnode:snode
796 for si, di, pn, sn in [(src_instance.name, dest_instance.name,
797 tnode.primary, pnode.primary),
798 (dest_instance.name, src_instance.name,
799 pnode.primary, snode.primary)]:
801 "../tools/move-instance",
803 "--src-ca-file=%s" % _rapi_ca.name,
804 "--src-username=%s" % _rapi_username,
805 "--src-password-file=%s" % rapi_pw_file.name,
806 "--dest-instance-name=%s" % di,
807 "--dest-primary-node=%s" % pn,
808 "--dest-secondary-node=%s" % sn,
809 "--net=0:mac=%s" % constants.VALUE_GENERATE,
815 qa_utils.RunInstanceCheck(di, False)
816 AssertEqual(StartLocalCommand(cmd).wait(), 0)
817 qa_utils.RunInstanceCheck(si, False)
818 qa_utils.RunInstanceCheck(di, True)