4 # Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Remote API QA tests.
31 from ganeti import utils
32 from ganeti import constants
33 from ganeti import errors
34 from ganeti import cli
35 from ganeti import rapi
36 from ganeti import objects
37 from ganeti import query
38 from ganeti import compat
39 from ganeti import qlang
40 from ganeti import pathutils
42 import ganeti.rapi.client # pylint: disable=W0611
43 import ganeti.rapi.client_utils
49 from qa_instance import IsFailoverSupported
50 from qa_instance import IsMigrationSupported
51 from qa_instance import IsDiskReplacingSupported
52 from qa_utils import (AssertEqual, AssertIn, AssertMatch, StartLocalCommand)
53 from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG
62 def Setup(username, password):
63 """Configures the RAPI client.
66 # pylint: disable=W0603
73 _rapi_username = username
74 _rapi_password = password
76 master = qa_config.GetMasterNode()
78 # Load RAPI certificate from master node
79 cmd = ["cat", pathutils.RAPI_CERT_FILE]
81 # Write to temporary file
82 _rapi_ca = tempfile.NamedTemporaryFile()
83 _rapi_ca.write(qa_utils.GetCommandOutput(master["primary"],
84 utils.ShellQuoteArgs(cmd)))
87 port = qa_config.get("rapi-port", default=constants.DEFAULT_RAPI_PORT)
88 cfg_curl = rapi.client.GenericCurlConfig(cafile=_rapi_ca.name,
91 _rapi_client = rapi.client.GanetiRapiClient(master["primary"], port=port,
94 curl_config_fn=cfg_curl)
96 print "RAPI protocol version: %s" % _rapi_client.GetVersion()
99 INSTANCE_FIELDS = ("name", "os", "pnode", "snodes",
101 "disk_template", "disk.sizes",
102 "nic.ips", "nic.macs", "nic.modes", "nic.links",
103 "beparams", "hvparams",
104 "oper_state", "oper_ram", "oper_vcpus", "status", "tags")
106 NODE_FIELDS = ("name", "dtotal", "dfree",
107 "mtotal", "mnode", "mfree",
108 "pinst_cnt", "sinst_cnt", "tags")
110 GROUP_FIELDS = compat.UniqueFrozenset([
113 "node_cnt", "node_list",
116 JOB_FIELDS = compat.UniqueFrozenset([
117 "id", "ops", "status", "summary",
118 "opstatus", "opresult", "oplog",
119 "received_ts", "start_ts", "end_ts",
122 LIST_FIELDS = ("id", "uri")
126 """Return whether remote API tests should be run.
129 return qa_config.TestEnabled("rapi")
133 # pylint: disable=W0212
134 # due to _SendRequest usage
137 for uri, verify, method, body in uris:
138 assert uri.startswith("/")
140 print "%s %s" % (method, uri)
141 data = _rapi_client._SendRequest(method, uri, None, body)
143 if verify is not None:
147 AssertEqual(data, verify)
154 def _VerifyReturnsJob(data):
155 if not isinstance(data, int):
156 AssertMatch(data, r"^\d+$")
160 """Testing remote API version.
164 ("/version", constants.RAPI_VERSION, "GET", None),
168 def TestEmptyCluster():
169 """Testing remote API on an empty cluster.
172 master = qa_config.GetMasterNode()
173 master_full = qa_utils.ResolveNodeName(master)
175 def _VerifyInfo(data):
176 AssertIn("name", data)
177 AssertIn("master", data)
178 AssertEqual(data["master"], master_full)
180 def _VerifyNodes(data):
183 "uri": "/2/nodes/%s" % master_full,
185 AssertIn(master_entry, data)
187 def _VerifyNodesBulk(data):
189 for entry in NODE_FIELDS:
190 AssertIn(entry, node)
192 def _VerifyGroups(data):
194 "name": constants.INITIAL_NODE_GROUP_NAME,
195 "uri": "/2/groups/" + constants.INITIAL_NODE_GROUP_NAME,
197 AssertIn(default_group, data)
199 def _VerifyGroupsBulk(data):
201 for field in GROUP_FIELDS:
202 AssertIn(field, group)
205 ("/", None, "GET", None),
206 ("/2/info", _VerifyInfo, "GET", None),
207 ("/2/tags", None, "GET", None),
208 ("/2/nodes", _VerifyNodes, "GET", None),
209 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
210 ("/2/groups", _VerifyGroups, "GET", None),
211 ("/2/groups?bulk=1", _VerifyGroupsBulk, "GET", None),
212 ("/2/instances", [], "GET", None),
213 ("/2/instances?bulk=1", [], "GET", None),
214 ("/2/os", None, "GET", None),
217 # Test HTTP Not Found
218 for method in ["GET", "PUT", "POST", "DELETE"]:
220 _DoTests([("/99/resource/not/here/99", None, method, None)])
221 except rapi.client.GanetiApiError, err:
222 AssertEqual(err.code, 404)
224 raise qa_error.Error("Non-existent resource didn't return HTTP 404")
226 # Test HTTP Not Implemented
227 for method in ["PUT", "POST", "DELETE"]:
229 _DoTests([("/version", None, method, None)])
230 except rapi.client.GanetiApiError, err:
231 AssertEqual(err.code, 501)
233 raise qa_error.Error("Non-implemented method didn't fail")
237 """Testing resource queries via remote API.
240 master_name = qa_utils.ResolveNodeName(qa_config.GetMasterNode())
241 rnd = random.Random(7818)
243 for what in constants.QR_VIA_RAPI:
244 if what == constants.QR_JOB:
246 elif what == constants.QR_EXPORT:
251 all_fields = query.ALL_FIELDS[what].keys()
252 rnd.shuffle(all_fields)
254 # No fields, should return everything
255 result = _rapi_client.QueryFields(what)
256 qresult = objects.QueryFieldsResponse.FromDict(result)
257 AssertEqual(len(qresult.fields), len(all_fields))
260 result = _rapi_client.QueryFields(what, fields=[namefield])
261 qresult = objects.QueryFieldsResponse.FromDict(result)
262 AssertEqual(len(qresult.fields), 1)
264 # Specify all fields, order must be correct
265 result = _rapi_client.QueryFields(what, fields=all_fields)
266 qresult = objects.QueryFieldsResponse.FromDict(result)
267 AssertEqual(len(qresult.fields), len(all_fields))
268 AssertEqual([fdef.name for fdef in qresult.fields], all_fields)
271 result = _rapi_client.QueryFields(what, fields=["_unknown!"])
272 qresult = objects.QueryFieldsResponse.FromDict(result)
273 AssertEqual(len(qresult.fields), 1)
274 AssertEqual(qresult.fields[0].name, "_unknown!")
275 AssertEqual(qresult.fields[0].kind, constants.QFT_UNKNOWN)
277 # Try once more, this time without the client
279 ("/2/query/%s/fields" % what, None, "GET", None),
280 ("/2/query/%s/fields?fields=name,name,%s" % (what, all_fields[0]),
284 # Try missing query argument
287 ("/2/query/%s" % what, None, "GET", None),
289 except rapi.client.GanetiApiError, err:
290 AssertEqual(err.code, 400)
292 raise qa_error.Error("Request missing 'fields' parameter didn't fail")
294 def _Check(exp_fields, data):
295 qresult = objects.QueryResponse.FromDict(data)
296 AssertEqual([fdef.name for fdef in qresult.fields], exp_fields)
297 if not isinstance(qresult.data, list):
298 raise qa_error.Error("Query did not return a list")
301 # Specify fields in query
302 ("/2/query/%s?fields=%s" % (what, ",".join(all_fields)),
303 compat.partial(_Check, all_fields), "GET", None),
305 ("/2/query/%s?fields=%s" % (what, namefield),
306 compat.partial(_Check, [namefield]), "GET", None),
309 ("/2/query/%s?fields=%s,%%20%s%%09,%s%%20" %
310 (what, namefield, namefield, namefield),
311 compat.partial(_Check, [namefield] * 3), "GET", None),
313 # PUT with fields in query
314 ("/2/query/%s?fields=%s" % (what, namefield),
315 compat.partial(_Check, [namefield]), "PUT", {}),
318 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
319 "fields": all_fields,
322 ("/2/query/%s" % what, compat.partial(_Check, [namefield] * 4), "PUT", {
323 "fields": [namefield] * 4,
330 ("/2/query/%s" % what, compat.partial(_Check, all_fields), "PUT", {
331 "fields": all_fields,
332 "filter": [qlang.OP_TRUE, namefield],
336 if what == constants.QR_LOCK:
337 # Locks can't be filtered
340 except rapi.client.GanetiApiError, err:
341 AssertEqual(err.code, 500)
343 raise qa_error.Error("Filtering locks didn't fail")
347 if what == constants.QR_NODE:
349 (nodes, ) = _DoTests(
350 [("/2/query/%s" % what,
351 compat.partial(_Check, ["name", "master"]), "PUT",
352 {"fields": ["name", "master"],
353 "filter": [qlang.OP_TRUE, "master"],
355 qresult = objects.QueryResponse.FromDict(nodes)
356 AssertEqual(qresult.data, [
357 [[constants.RS_NORMAL, master_name], [constants.RS_NORMAL, True]],
361 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
362 def TestInstance(instance):
363 """Testing getting instance(s) info via remote API.
366 def _VerifyInstance(data):
367 for entry in INSTANCE_FIELDS:
368 AssertIn(entry, data)
370 def _VerifyInstancesList(data):
371 for instance in data:
372 for entry in LIST_FIELDS:
373 AssertIn(entry, instance)
375 def _VerifyInstancesBulk(data):
376 for instance_data in data:
377 _VerifyInstance(instance_data)
380 ("/2/instances/%s" % instance["name"], _VerifyInstance, "GET", None),
381 ("/2/instances", _VerifyInstancesList, "GET", None),
382 ("/2/instances?bulk=1", _VerifyInstancesBulk, "GET", None),
383 ("/2/instances/%s/activate-disks" % instance["name"],
384 _VerifyReturnsJob, "PUT", None),
385 ("/2/instances/%s/deactivate-disks" % instance["name"],
386 _VerifyReturnsJob, "PUT", None),
389 # Test OpBackupPrepare
390 (job_id, ) = _DoTests([
391 ("/2/instances/%s/prepare-export?mode=%s" %
392 (instance["name"], constants.EXPORT_MODE_REMOTE),
393 _VerifyReturnsJob, "PUT", None),
396 result = _WaitForRapiJob(job_id)[0]
397 AssertEqual(len(result["handshake"]), 3)
398 AssertEqual(result["handshake"][0], constants.RIE_VERSION)
399 AssertEqual(len(result["x509_key_name"]), 3)
400 AssertIn("-----BEGIN CERTIFICATE-----", result["x509_ca"])
404 """Testing getting node(s) info via remote API.
407 def _VerifyNode(data):
408 for entry in NODE_FIELDS:
409 AssertIn(entry, data)
411 def _VerifyNodesList(data):
413 for entry in LIST_FIELDS:
414 AssertIn(entry, node)
416 def _VerifyNodesBulk(data):
417 for node_data in data:
418 _VerifyNode(node_data)
421 ("/2/nodes/%s" % node["primary"], _VerifyNode, "GET", None),
422 ("/2/nodes", _VerifyNodesList, "GET", None),
423 ("/2/nodes?bulk=1", _VerifyNodesBulk, "GET", None),
427 def _FilterTags(seq):
428 """Removes unwanted tags from a sequence.
431 ignore_re = qa_config.get("ignore-tags-re", None)
434 return itertools.ifilterfalse(re.compile(ignore_re).match, seq)
439 def TestTags(kind, name, tags):
440 """Tests .../tags resources.
443 if kind == constants.TAG_CLUSTER:
445 elif kind == constants.TAG_NODE:
446 uri = "/2/nodes/%s/tags" % name
447 elif kind == constants.TAG_INSTANCE:
448 uri = "/2/instances/%s/tags" % name
449 elif kind == constants.TAG_NODEGROUP:
450 uri = "/2/groups/%s/tags" % name
452 raise errors.ProgrammerError("Unknown tag kind")
454 def _VerifyTags(data):
455 AssertEqual(sorted(tags), sorted(_FilterTags(data)))
457 queryargs = "&".join("tag=%s" % i for i in tags)
460 (job_id, ) = _DoTests([
461 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "PUT", None),
463 _WaitForRapiJob(job_id)
467 (uri, _VerifyTags, "GET", None),
471 (job_id, ) = _DoTests([
472 ("%s?%s" % (uri, queryargs), _VerifyReturnsJob, "DELETE", None),
474 _WaitForRapiJob(job_id)
477 def _WaitForRapiJob(job_id):
478 """Waits for a job to finish.
481 def _VerifyJob(data):
482 AssertEqual(data["id"], job_id)
483 for field in JOB_FIELDS:
484 AssertIn(field, data)
487 ("/2/jobs/%s" % job_id, _VerifyJob, "GET", None),
490 return rapi.client_utils.PollJob(_rapi_client, job_id,
491 cli.StdioJobPollReportCb())
494 def TestRapiNodeGroups():
495 """Test several node group operations using RAPI.
498 (group1, group2, group3) = qa_utils.GetNonexistentGroups(3)
500 # Create a group with no attributes
505 (job_id, ) = _DoTests([
506 ("/2/groups", _VerifyReturnsJob, "POST", body),
509 _WaitForRapiJob(job_id)
511 # Create a group specifying alloc_policy
514 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
517 (job_id, ) = _DoTests([
518 ("/2/groups", _VerifyReturnsJob, "POST", body),
521 _WaitForRapiJob(job_id)
523 # Modify alloc_policy
525 "alloc_policy": constants.ALLOC_POLICY_UNALLOCABLE,
528 (job_id, ) = _DoTests([
529 ("/2/groups/%s/modify" % group1, _VerifyReturnsJob, "PUT", body),
532 _WaitForRapiJob(job_id)
539 (job_id, ) = _DoTests([
540 ("/2/groups/%s/rename" % group2, _VerifyReturnsJob, "PUT", body),
543 _WaitForRapiJob(job_id)
546 for group in [group1, group3]:
547 (job_id, ) = _DoTests([
548 ("/2/groups/%s" % group, _VerifyReturnsJob, "DELETE", None),
551 _WaitForRapiJob(job_id)
554 def TestRapiInstanceAdd(node, use_client):
555 """Test adding a new instance via RAPI"""
556 instance = qa_config.AcquireInstance()
557 qa_config.SetInstanceTemplate(instance, constants.DT_PLAIN)
559 disk_sizes = [utils.ParseUnit(size) for size in qa_config.get("disk")]
560 disks = [{"size": size} for size in disk_sizes]
561 nic0_mac = qa_config.GetInstanceNicMac(instance,
562 default=constants.VALUE_GENERATE)
564 constants.INIC_MAC: nic0_mac,
568 constants.BE_MAXMEM: utils.ParseUnit(qa_config.get(constants.BE_MAXMEM)),
569 constants.BE_MINMEM: utils.ParseUnit(qa_config.get(constants.BE_MINMEM)),
573 job_id = _rapi_client.CreateInstance(constants.INSTANCE_CREATE,
577 os=qa_config.get("os"),
578 pnode=node["primary"],
583 "mode": constants.INSTANCE_CREATE,
584 "name": instance["name"],
585 "os_type": qa_config.get("os"),
586 "disk_template": constants.DT_PLAIN,
587 "pnode": node["primary"],
588 "beparams": beparams,
593 (job_id, ) = _DoTests([
594 ("/2/instances", _VerifyReturnsJob, "POST", body),
597 _WaitForRapiJob(job_id)
601 qa_config.ReleaseInstance(instance)
605 @InstanceCheck(None, INST_DOWN, FIRST_ARG)
606 def TestRapiInstanceRemove(instance, use_client):
607 """Test removing instance via RAPI"""
609 job_id = _rapi_client.DeleteInstance(instance["name"])
611 (job_id, ) = _DoTests([
612 ("/2/instances/%s" % instance["name"], _VerifyReturnsJob, "DELETE", None),
615 _WaitForRapiJob(job_id)
617 qa_config.ReleaseInstance(instance)
620 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
621 def TestRapiInstanceMigrate(instance):
622 """Test migrating instance via RAPI"""
623 if not IsMigrationSupported(instance):
624 print qa_utils.FormatInfo("Instance doesn't support migration, skipping"
627 # Move to secondary node
628 _WaitForRapiJob(_rapi_client.MigrateInstance(instance["name"]))
629 qa_utils.RunInstanceCheck(instance, True)
630 # And back to previous primary
631 _WaitForRapiJob(_rapi_client.MigrateInstance(instance["name"]))
634 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
635 def TestRapiInstanceFailover(instance):
636 """Test failing over instance via RAPI"""
637 if not IsFailoverSupported(instance):
638 print qa_utils.FormatInfo("Instance doesn't support failover, skipping"
641 # Move to secondary node
642 _WaitForRapiJob(_rapi_client.FailoverInstance(instance["name"]))
643 qa_utils.RunInstanceCheck(instance, True)
644 # And back to previous primary
645 _WaitForRapiJob(_rapi_client.FailoverInstance(instance["name"]))
648 @InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
649 def TestRapiInstanceShutdown(instance):
650 """Test stopping an instance via RAPI"""
651 _WaitForRapiJob(_rapi_client.ShutdownInstance(instance["name"]))
654 @InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
655 def TestRapiInstanceStartup(instance):
656 """Test starting an instance via RAPI"""
657 _WaitForRapiJob(_rapi_client.StartupInstance(instance["name"]))
660 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
661 def TestRapiInstanceRenameAndBack(rename_source, rename_target):
662 """Test renaming instance via RAPI
664 This must leave the instance with the original name (in the
668 _WaitForRapiJob(_rapi_client.RenameInstance(rename_source, rename_target))
669 qa_utils.RunInstanceCheck(rename_source, False)
670 qa_utils.RunInstanceCheck(rename_target, False)
671 _WaitForRapiJob(_rapi_client.RenameInstance(rename_target, rename_source))
672 qa_utils.RunInstanceCheck(rename_target, False)
675 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
676 def TestRapiInstanceReinstall(instance):
677 """Test reinstalling an instance via RAPI"""
678 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance["name"]))
679 # By default, the instance is started again
680 qa_utils.RunInstanceCheck(instance, True)
682 # Reinstall again without starting
683 _WaitForRapiJob(_rapi_client.ReinstallInstance(instance["name"],
687 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
688 def TestRapiInstanceReplaceDisks(instance):
689 """Test replacing instance disks via RAPI"""
690 if not IsDiskReplacingSupported(instance):
691 print qa_utils.FormatInfo("Instance doesn't support disk replacing,"
694 fn = _rapi_client.ReplaceInstanceDisks
695 _WaitForRapiJob(fn(instance["name"],
696 mode=constants.REPLACE_DISK_AUTO, disks=[]))
697 _WaitForRapiJob(fn(instance["name"],
698 mode=constants.REPLACE_DISK_SEC, disks="0"))
701 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
702 def TestRapiInstanceModify(instance):
703 """Test modifying instance via RAPI"""
704 default_hv = qa_config.GetDefaultHypervisor()
706 def _ModifyInstance(**kwargs):
707 _WaitForRapiJob(_rapi_client.ModifyInstance(instance["name"], **kwargs))
709 _ModifyInstance(beparams={
710 constants.BE_VCPUS: 3,
713 _ModifyInstance(beparams={
714 constants.BE_VCPUS: constants.VALUE_DEFAULT,
717 if default_hv == constants.HT_XEN_PVM:
718 _ModifyInstance(hvparams={
719 constants.HV_KERNEL_ARGS: "single",
721 _ModifyInstance(hvparams={
722 constants.HV_KERNEL_ARGS: constants.VALUE_DEFAULT,
724 elif default_hv == constants.HT_XEN_HVM:
725 _ModifyInstance(hvparams={
726 constants.HV_BOOT_ORDER: "acn",
728 _ModifyInstance(hvparams={
729 constants.HV_BOOT_ORDER: constants.VALUE_DEFAULT,
733 @InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
734 def TestRapiInstanceConsole(instance):
735 """Test getting instance console information via RAPI"""
736 result = _rapi_client.GetInstanceConsole(instance["name"])
737 console = objects.InstanceConsole.FromDict(result)
738 AssertEqual(console.Validate(), True)
739 AssertEqual(console.instance, qa_utils.ResolveInstanceName(instance["name"]))
742 @InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
743 def TestRapiStoppedInstanceConsole(instance):
744 """Test getting stopped instance's console information via RAPI"""
746 _rapi_client.GetInstanceConsole(instance["name"])
747 except rapi.client.GanetiApiError, err:
748 AssertEqual(err.code, 503)
750 raise qa_error.Error("Getting console for stopped instance didn't"
754 def GetOperatingSystems():
755 """Retrieves a list of all available operating systems.
758 return _rapi_client.GetOperatingSystems()
761 def TestInterClusterInstanceMove(src_instance, dest_instance,
763 """Test tools/move-instance"""
764 master = qa_config.GetMasterNode()
766 rapi_pw_file = tempfile.NamedTemporaryFile()
767 rapi_pw_file.write(_rapi_password)
770 qa_config.SetInstanceTemplate(dest_instance,
771 qa_config.GetInstanceTemplate(src_instance))
773 # TODO: Run some instance tests before moving back
776 # No disk template currently requires more than 1 secondary node. If this
777 # changes, either this test must be skipped or the script must be updated.
778 assert len(inodes) == 2
781 # instance is not redundant, but we still need to pass a node
782 # (which will be ignored)
785 # note: pnode:snode are the *current* nodes, so we move it first to
786 # tnode:pnode, then back to pnode:snode
787 for si, di, pn, sn in [(src_instance["name"], dest_instance["name"],
788 tnode["primary"], pnode["primary"]),
789 (dest_instance["name"], src_instance["name"],
790 pnode["primary"], snode["primary"])]:
792 "../tools/move-instance",
794 "--src-ca-file=%s" % _rapi_ca.name,
795 "--src-username=%s" % _rapi_username,
796 "--src-password-file=%s" % rapi_pw_file.name,
797 "--dest-instance-name=%s" % di,
798 "--dest-primary-node=%s" % pn,
799 "--dest-secondary-node=%s" % sn,
800 "--net=0:mac=%s" % constants.VALUE_GENERATE,
806 qa_utils.RunInstanceCheck(di, False)
807 AssertEqual(StartLocalCommand(cmd).wait(), 0)
808 qa_utils.RunInstanceCheck(si, False)
809 qa_utils.RunInstanceCheck(di, True)