From: Sofia Papagiannaki Date: Tue, 19 Jul 2011 09:06:52 +0000 (+0300) Subject: extend client lib to support extended xml mode X-Git-Tag: pithos/v0.7.8~188 X-Git-Url: https://code.grnet.gr/git/pithos/commitdiff_plain/f0eacc2cdffbec12bd89491b3ce9bd274f361dd2 extend client lib to support extended xml mode -listing methods get specific format parameter instead of the detail paramster -bug fixes --- diff --git a/pithos/api/tests.py b/pithos/api/tests.py index f7c7f89..b966e21 100644 --- a/pithos/api/tests.py +++ b/pithos/api/tests.py @@ -36,6 +36,7 @@ import unittest from django.utils import simplejson as json from xml.dom import minidom from StringIO import StringIO +import time as _time import types import hashlib import os @@ -45,8 +46,6 @@ import datetime import string import re -#from pithos.backends import backend - DATE_FORMATS = ["%a %b %d %H:%M:%S %Y", "%A, %d-%b-%y %H:%M:%S GMT", "%a, %d %b %Y %H:%M:%S GMT"] @@ -54,51 +53,58 @@ DATE_FORMATS = ["%a %b %d %H:%M:%S %Y", DEFAULT_HOST = 'pithos.dev.grnet.gr' #DEFAULT_HOST = '127.0.0.1:8000' DEFAULT_API = 'v1' -DEFAULT_USER = 'papagian' -DEFAULT_AUTH = '0004' +DEFAULT_USER = 'test' +DEFAULT_AUTH = '0000' class BaseTestCase(unittest.TestCase): #TODO unauthorized request def setUp(self): - self.client = Pithos_Client(DEFAULT_HOST, DEFAULT_AUTH, DEFAULT_USER, DEFAULT_API) - self.headers = { - 'account': ('x-account-container-count', - 'x-account-bytes-used', - 'last-modified', - 'content-length', - 'date', - 'content_type', - 'server',), - 'object': ('etag', - 'content-length', - 'content_type', - 'content-encoding', - 'last-modified', - 'date', - 'x-object-manifest', - 'content-range', - 'x-object-modified-by', - 'x-object-version', - 'x-object-version-timestamp', - 'server',), - 'container': ('x-container-object-count', - 'x-container-bytes-used', - 'content_type', - 'last-modified', - 'content-length', - 'date', - 'x-container-block-size', - 'x-container-block-hash', - 'x-container-policy-quota', - 'x-container-policy-versioning', - 'server', - 'x-container-object-meta', - 'x-container-policy-versioning', - 'server',)} - - self.contentTypes = {'xml':'application/xml', - 'json':'application/json', - '':'text/plain'} + self.client = Pithos_Client(DEFAULT_HOST, + DEFAULT_AUTH, + DEFAULT_USER, + DEFAULT_API) + self.invalid_client = Pithos_Client(DEFAULT_HOST, + DEFAULT_AUTH, + 'invalid', + DEFAULT_API) + #self.headers = { + # 'account': ('x-account-container-count', + # 'x-account-bytes-used', + # 'last-modified', + # 'content-length', + # 'date', + # 'content_type', + # 'server',), + # 'object': ('etag', + # 'content-length', + # 'content_type', + # 'content-encoding', + # 'last-modified', + # 'date', + # 'x-object-manifest', + # 'content-range', + # 'x-object-modified-by', + # 'x-object-version', + # 'x-object-version-timestamp', + # 'server',), + # 'container': ('x-container-object-count', + # 'x-container-bytes-used', + # 'content_type', + # 'last-modified', + # 'content-length', + # 'date', + # 'x-container-block-size', + # 'x-container-block-hash', + # 'x-container-policy-quota', + # 'x-container-policy-versioning', + # 'server', + # 'x-container-object-meta', + # 'x-container-policy-versioning', + # 'server',)} + # + #self.contentTypes = {'xml':'application/xml', + # 'json':'application/json', + # '':'text/plain'} self.extended = { 'container':( 'name', @@ -146,48 +152,47 @@ class BaseTestCase(unittest.TestCase): # data = minidom.parseString(data) # return status, headers, data - def assert_headers(self, headers, type, **exp_meta): - prefix = 'x-%s-meta-' %type - system_headers = [h for h in headers if not h.startswith(prefix)] - for k,v in headers.items(): - if k in system_headers: - self.assertTrue(k in headers[type]) - elif exp_meta: - k = k.split(prefix)[-1] - self.assertEqual(v, exp_meta[k]) - - #def assert_extended(self, data, format, type, size): - # if format == 'xml': - # self._assert_xml(data, type, size) - # elif format == 'json': - # self._assert_json(data, type, size) - - #def _assert_json(self, data, type, size): - # print '#', data - # convert = lambda s: s.lower() - # info = [convert(elem) for elem in self.extended[type]] - # data = json.loads(data) - # self.assertTrue(len(data) <= size) - # for item in info: - # for i in data: - # if 'subdir' in i.keys(): - # continue - # self.assertTrue(item in i.keys()) - - #def _assert_xml(self, data, type, size): - # print '#', data - # convert = lambda s: s.lower() - # info = [convert(elem) for elem in self.extended[type]] - # try: - # info.remove('content_encoding') - # except ValueError: - # pass - # xml = minidom.parseString(data) - # entities = xml.getElementsByTagName(type) - # self.assertTrue(len(entities) <= size) - # for e in entities: - # for item in info: - # self.assertTrue(e.hasAttribute(item)) + #def assert_headers(self, headers, type, **exp_meta): + # prefix = 'x-%s-meta-' %type + # system_headers = [h for h in headers if not h.startswith(prefix)] + # for k,v in headers.items(): + # if k in system_headers: + # self.assertTrue(k in headers[type]) + # elif exp_meta: + # k = k.split(prefix)[-1] + # self.assertEqual(v, exp_meta[k]) + + def assert_extended(self, data, format, type, size): + if format == 'xml': + self._assert_xml(data, type, size) + elif format == 'json': + self._assert_json(data, type, size) + + def _assert_json(self, data, type, size): + print '#', data + convert = lambda s: s.lower() + info = [convert(elem) for elem in self.extended[type]] + self.assertTrue(len(data) <= size) + for item in info: + for i in data: + if 'subdir' in i.keys(): + continue + self.assertTrue(item in i.keys()) + + def _assert_xml(self, data, type, size): + convert = lambda s: s.lower() + info = [convert(elem) for elem in self.extended[type]] + try: + info.remove('content_encoding') + except ValueError: + pass + xml = data + entities = xml.getElementsByTagName(type) + self.assertTrue(len(entities) <= size) + for e in entities: + for item in info: + print '#', item + self.assertTrue(e.hasAttribute(item)) def assert_raises_fault(self, status, callableObj, *args, **kwargs): """ @@ -265,6 +270,10 @@ class AccountHead(BaseTestCase): for item in self.containers: self.client.create_container(item) + def tearDown(self): + self.client.delete_account_metadata(['foo']) + BaseTestCase.tearDown(self) + def test_get_account_meta(self): meta = self.client.retrieve_account_metadata() @@ -277,10 +286,30 @@ class AccountHead(BaseTestCase): size = size + int(m['x-container-bytes-used']) self.assertEqual(meta['x-account-bytes-used'], str(size)) - #def test_get_account_401(self): - # response = self.get_account_meta('non-existing-account') - # print response - # self.assertEqual(response.status_code, 401) + def test_get_account_401(self): + self.assert_raises_fault(401, + self.invalid_client.retrieve_account_metadata) + + def test_get_account_meta_until(self): + t = datetime.datetime.utcnow() + past = t - datetime.timedelta(minutes=-15) + past = int(_time.mktime(past.timetuple())) + + meta = {'foo':'bar'} + self.client.update_account_metadata(**meta) + meta = self.client.retrieve_account_metadata(restricted=True, + until=past) + self.assertTrue('foo' not in meta) + + meta = self.client.retrieve_account_metadata(restricted=True) + self.assertTrue('foo' in meta) + + def test_get_account_meta_until_invalid_date(self): + meta = {'foo':'bar'} + self.client.update_account_metadata(**meta) + meta = self.client.retrieve_account_metadata(restricted=True, + until='kshfksfh') + self.assertTrue('foo' in meta) class AccountGet(BaseTestCase): def setUp(self): @@ -296,9 +325,8 @@ class AccountGet(BaseTestCase): containers = self.client.list_containers() self.assertEquals(self.containers, containers) - #def test_list_204(self): - # response = self.list_containers('non-existing-account') - # self.assertEqual(response.status_code, 204) + def test_list_401(self): + self.assert_raises_fault(401, self.invalid_client.list_containers) def test_list_with_limit(self): limit = 2 @@ -318,24 +346,22 @@ class AccountGet(BaseTestCase): i = self.containers.index(m) + 1 self.assertEquals(self.containers[i:(i+l)], containers) - #def test_extended_list(self): - # self.list_containers(self.account, limit=3, format='xml') - # self.list_containers(self.account, limit=3, format='json') - def test_list_json_with_marker(self): l = 2 m = 'bananas' - containers = self.client.list_containers(limit=l, marker=m, detail=True) + containers = self.client.list_containers(limit=l, marker=m, format='json') + self.assert_extended(containers, 'json', 'container', l) self.assertEqual(containers[0]['name'], 'kiwis') self.assertEqual(containers[1]['name'], 'oranges') - #def test_list_xml_with_marker(self): - # l = 2 - # m = 'oranges' - # status, headers, xml = self.list_containers(limit=l, marker=m, format='xml') - # nodes = xml.getElementsByTagName('name') - # self.assertEqual(len(nodes), 1) - # self.assertEqual(nodes[0].childNodes[0].data, 'pears') + def test_list_xml_with_marker(self): + l = 2 + m = 'oranges' + xml = self.client.list_containers(limit=l, marker=m, format='xml') + #self.assert_extended(xml, 'xml', 'container', l) + nodes = xml.getElementsByTagName('name') + self.assertEqual(len(nodes), 1) + self.assertEqual(nodes[0].childNodes[0].data, 'pears') def test_if_modified_since(self): t = datetime.datetime.utcnow() @@ -404,11 +430,12 @@ class AccountPost(BaseTestCase): self.client.update_account_metadata(**meta) self.assertEqual(meta, self.client.retrieve_account_metadata(restricted=True)) - #def test_invalid_account_update_meta(self): - # with AssertMappingInvariant(self.get_account_meta, self.account): - # meta = {'HTTP_X_ACCOUNT_META_TEST':'test', - # 'HTTP_X_ACCOUNT_META_TOST':'tost'} - # response = self.update_account_meta('non-existing-account', **meta) + def test_invalid_account_update_meta(self): + meta = {'HTTP_X_ACCOUNT_META_TEST':'test', + 'HTTP_X_ACCOUNT_META_TOST':'tost'} + self.assert_raises_fault(401, + self.invalid_client.update_account_metadata, + **meta) class ContainerHead(BaseTestCase): def setUp(self): @@ -486,23 +513,23 @@ class ContainerGet(BaseTestCase): self.assertEquals(['photos/me.jpg'], objects) def test_extended_list_json(self): - objects = self.client.list_objects(self.container[1], detail=True, + objects = self.client.list_objects(self.container[1], format='json', limit=2, prefix='photos/animals', delimiter='/') self.assertEqual(objects[0]['subdir'], 'photos/animals/cats/') self.assertEqual(objects[1]['subdir'], 'photos/animals/dogs/') - #def test_extended_list_xml(self): - # xml = self.client.list_objects(self.container[1], format='xml', limit=4, - # prefix='photos', delimiter='/') - # dirs = xml.getElementsByTagName('subdir') - # self.assertEqual(len(dirs), 2) - # self.assertEqual(dirs[0].attributes['name'].value, 'photos/animals/') - # self.assertEqual(dirs[1].attributes['name'].value, 'photos/plants/') - # - # objects = xml.getElementsByTagName('name') - # self.assertEqual(len(objects), 1) - # self.assertEqual(objects[0].childNodes[0].data, 'photos/me.jpg') + def test_extended_list_xml(self): + xml = self.client.list_objects(self.container[1], format='xml', limit=4, + prefix='photos', delimiter='/') + dirs = xml.getElementsByTagName('subdir') + self.assertEqual(len(dirs), 2) + self.assertEqual(dirs[0].attributes['name'].value, 'photos/animals/') + self.assertEqual(dirs[1].attributes['name'].value, 'photos/plants/') + + objects = xml.getElementsByTagName('name') + self.assertEqual(len(objects), 1) + self.assertEqual(objects[0].childNodes[0].data, 'photos/me.jpg') def test_list_meta_double_matching(self): meta = {'quality':'aaa', 'stock':'true'} @@ -978,7 +1005,8 @@ class ObjectGet(BaseTestCase): fname = 'largefile' o = self.upload_random_data(self.containers[1], fname, l) if o: - data = self.client.retrieve_object(self.containers[1], fname, detail=True) + data = self.client.retrieve_object(self.containers[1], fname, + format='json') body = json.loads(data) hashes = body['hashes'] block_size = body['block_size'] @@ -1026,7 +1054,7 @@ class ObjectPut(BaseTestCase): self.assert_raises_fault(422, self.upload_random_data, self.container, o_names[0], **meta) - def test_chucked_transfer(self): + def test_chunked_transfer(self): data = get_random_data() objname = 'object' self.client.create_object_using_chunks(self.container, objname, @@ -1197,13 +1225,11 @@ class ObjectPost(BaseTestCase): self.containers[0], self.obj['name']): self.test_update_object(499, 0, True) - #no use if the server resets the content-legth def test_update_object_invalid_range_and_length(self): with AssertContentInvariant(self.client.retrieve_object, self.containers[0], self.obj['name']): self.test_update_object(499, 0, True, -1) - #no use if the server resets the content-legth def test_update_object_invalid_range_with_no_content_length(self): with AssertContentInvariant(self.client.retrieve_object, self.containers[0], self.obj['name']): diff --git a/pithos/lib/client.py b/pithos/lib/client.py index 4376d0b..fa2e10c 100644 --- a/pithos/lib/client.py +++ b/pithos/lib/client.py @@ -33,6 +33,7 @@ from httplib import HTTPConnection, HTTP from sys import stdin +from xml.dom import minidom import json import types @@ -93,7 +94,7 @@ class Client(object): kwargs['headers']['X-Auth-Token'] = self.token if body: kwargs['body'] = body - else: + elif 'content-type' not in kwargs['headers']: kwargs['headers']['content-type'] = '' kwargs['headers'].setdefault('content-length', len(body) if body else 0) kwargs['headers'].setdefault('content-type', 'application/octet-stream') @@ -141,12 +142,14 @@ class Client(object): def put(self, path, body=None, format='text', headers=None): return self._req('PUT', path, body, headers=headers, format=format) - def _list(self, path, detail=False, params={}, **headers): - format = 'json' if detail else 'text' + def _list(self, path, format='text', params={}, **headers): status, headers, data = self.get(path, format=format, headers=headers, params=params) - if detail: + if format == 'json': data = json.loads(data) if data else '' + elif format == 'xml': + print '#', data + data = minidom.parseString(data) else: data = data.strip().split('\n') if data else '' return data @@ -178,7 +181,7 @@ class Client(object): return ll class OOS_Client(Client): - """Openstack Objest Storage Client""" + """Openstack Object Storage Client""" def _update_metadata(self, path, entity, **meta): """adds new and updates the values of previously set metadata""" @@ -203,13 +206,13 @@ class OOS_Client(Client): # Storage Account Services - def list_containers(self, detail=False, limit=10000, marker=None, params={}, + def list_containers(self, format='text', limit=10000, marker=None, params={}, **headers): """lists containers""" if not params: params = {} params.update({'limit':limit, 'marker':marker}) - return self._list('', detail, params, **headers) + return self._list('', format, params, **headers) def retrieve_account_metadata(self, restricted=False, **params): """returns the account metadata""" @@ -229,14 +232,15 @@ class OOS_Client(Client): def _filter_trashed(self, l): return self._filter(l, {'trash':'true'}) - def list_objects(self, container, detail=False, limit=10000, marker=None, + def list_objects(self, container, format='text', limit=10000, marker=None, prefix=None, delimiter=None, path=None, include_trashed=False, params={}, **headers): """returns a list with the container objects""" params.update({'limit':limit, 'marker':marker, 'prefix':prefix, 'delimiter':delimiter, 'path':path}) - l = self._list('/' + container, detail, params, **headers) - if not include_trashed: + l = self._list('/' + container, format, params, **headers) + #TODO support filter trashed with xml also + if format != 'xml' and not include_trashed: l = self._filter_trashed(l) return l @@ -272,18 +276,17 @@ class OOS_Client(Client): # Storage Object Services - def request_object(self, container, object, detail=False, params={}, + def request_object(self, container, object, format='text', params={}, **headers): """returns tuple containing the status, headers and data response for an object request""" path = '/%s/%s' % (container, object) - format = 'json' if detail else 'text' status, headers, data = self.get(path, format, headers, params) return status, headers, data - def retrieve_object(self, container, object, detail=False, params={}, + def retrieve_object(self, container, object, format='text', params={}, **headers): """returns an object's data""" - t = self.request_object(container, object, detail, params, **headers) + t = self.request_object(container, object, format, params, **headers) return t[2] def create_directory_marker(self, container, object): @@ -469,19 +472,19 @@ class Pithos_Client(OOS_Client): headers = {} prefix = 'x-%s-meta-' % entity for m in meta: - headers['%s%s' % (prefix, m)] = None + headers['%s%s' % (prefix, m)] = '' return self.post(path, headers=headers, params=params) # Storage Account Services - def list_containers(self, detail=False, if_modified_since=None, + def list_containers(self, format='text', if_modified_since=None, if_unmodified_since=None, limit=1000, marker=None, until=None): """returns a list with the account containers""" params = {'until':until} if until else None headers = {'if-modified-since':if_modified_since, 'if-unmodified-since':if_unmodified_since} - return OOS_Client.list_containers(self, detail=detail, limit=limit, + return OOS_Client.list_containers(self, format=format, limit=limit, marker=marker, params=params, **headers) @@ -509,7 +512,7 @@ class Pithos_Client(OOS_Client): # Storage Container Services - def list_objects(self, container, detail=False, limit=10000, marker=None, + def list_objects(self, container, format='text', limit=10000, marker=None, prefix=None, delimiter=None, path=None, include_trashed=False, params={}, if_modified_since=None, if_unmodified_since=None, meta={}, until=None): @@ -545,7 +548,7 @@ class Pithos_Client(OOS_Client): # Storage Object Services - def retrieve_object(self, container, object, params={}, detail=False, range=None, + def retrieve_object(self, container, object, params={}, format='text', range=None, if_range=None, if_match=None, if_none_match=None, if_modified_since=None, if_unmodified_since=None, **headers): @@ -556,7 +559,7 @@ class Pithos_Client(OOS_Client): l = [elem for elem in l if eval(elem)] for elem in l: headers.update({elem:eval(elem)}) - return OOS_Client.retrieve_object(self, container, object, detail=detail, + return OOS_Client.retrieve_object(self, container, object, format=format, params=params, **headers) def retrieve_object_version(self, container, object, version, detail=False, diff --git a/pithos/test-settings.py b/pithos/test-settings.py index af13f93..db0dbb7 100644 --- a/pithos/test-settings.py +++ b/pithos/test-settings.py @@ -60,9 +60,9 @@ DATABASES = { # The backend to use and its initilization options. if TEST: - BACKEND = ('SimpleBackend', (os.path.join(PROJECT_PATH, 'data/testpithos.db'),)) + BACKEND = ('SimpleBackend', (os.path.join(PROJECT_PATH, 'data/test/'),)) else: - BACKEND = ('SimpleBackend', (os.path.join(PROJECT_PATH, 'data/pithos.db'),)) + BACKEND = ('SimpleBackend', (os.path.join(PROJECT_PATH, 'data/pithos/'),)) # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name @@ -149,4 +149,5 @@ AUTH_TOKENS = { '0004': 'papagian', '0005': 'louridas', '0006': 'chstath', - '0007': 'pkanavos'} + '0007': 'pkanavos', + '0008': 'mvasilak'} diff --git a/tools/store b/tools/store index 0ef56ce..1d555a3 100755 --- a/tools/store +++ b/tools/store @@ -143,7 +143,7 @@ class List(Command): dest='if_unmodified_since', default=None, help='show output if not modified since then') parser.add_option('--until', action='store', dest='until', - default=False, help='show metadata until that date') + default=None, help='show metadata until that date') parser.add_option('--format', action='store', dest='format', default='%d/%m/%Y', help='format to parse until date') @@ -154,11 +154,12 @@ class List(Command): self.list_containers() def list_containers(self): - attrs = ['detail', 'limit', 'marker', 'if_modified_since', + attrs = ['limit', 'marker', 'if_modified_since', 'if_unmodified_since'] args = self._build_args(attrs) + args['format'] = 'json' if self.detail else 'text' - if self.until: + if getattr(self, 'until'): t = _time.strptime(self.until, self.format) args['until'] = int(_time.mktime(t)) @@ -168,9 +169,10 @@ class List(Command): def list_objects(self, container): #prepate params params = {} - attrs = ['detail', 'limit', 'marker', 'prefix', 'delimiter', 'path', + attrs = ['limit', 'marker', 'prefix', 'delimiter', 'path', 'meta', 'if_modified_since', 'if_unmodified_since'] args = self._build_args(attrs) + args['format'] = 'json' if self.detail else 'text' if self.until: t = _time.strptime(self.until, self.format) @@ -195,7 +197,7 @@ class Meta(Command): parser.add_option('-r', action='store_true', dest='restricted', default=False, help='show only user defined metadata') parser.add_option('--until', action='store', dest='until', - default=False, help='show metadata until that date') + default=None, help='show metadata until that date') parser.add_option('--format', action='store', dest='format', default='%d/%m/%Y', help='format to parse until date') parser.add_option('--version', action='store', dest='version', @@ -205,7 +207,7 @@ class Meta(Command): def execute(self, path=''): container, sep, object = path.partition('/') args = {'restricted':self.restricted} - if self.until: + if getattr(self, 'until'): t = _time.strptime(self.until, self.format) args['until'] = int(_time.mktime(t)) @@ -243,13 +245,14 @@ class Delete(Command): def add_options(self, parser): parser.add_option('--until', action='store', dest='until', - default=False, help='remove history until that date') + default=None, help='remove history until that date') parser.add_option('--format', action='store', dest='format', default='%d/%m/%Y', help='format to parse until date') def execute(self, path): container, sep, object = path.partition('/') - if self.until: + until = None + if getattr(self, 'until'): t = _time.strptime(self.until, self.format) until = int(_time.mktime(t)) @@ -293,10 +296,10 @@ class GetObject(Command): help='get the full object version list') def execute(self, path): - attrs = ['detail', 'if_match', 'if_none_match', 'if_modified_since', + attrs = ['if_match', 'if_none_match', 'if_modified_since', 'if_unmodified_since'] args = self._build_args(attrs) - + args['format'] = 'json' if self.detail else 'text' if self.range: args['range'] = 'bytes=%s' %self.range if getattr(self, 'if_range'):