root / snf-pithos-app / pithos / api / test / __init__.py @ 0534576c
History | View | Annotate | Download (17.8 kB)
1 |
#!/usr/bin/env python
|
---|---|
2 |
#coding=utf8
|
3 |
|
4 |
# Copyright 2011-2013 GRNET S.A. All rights reserved.
|
5 |
#
|
6 |
# Redistribution and use in source and binary forms, with or
|
7 |
# without modification, are permitted provided that the following
|
8 |
# conditions are met:
|
9 |
#
|
10 |
# 1. Redistributions of source code must retain the above
|
11 |
# copyright notice, this list of conditions and the following
|
12 |
# disclaimer.
|
13 |
#
|
14 |
# 2. Redistributions in binary form must reproduce the above
|
15 |
# copyright notice, this list of conditions and the following
|
16 |
# disclaimer in the documentation and/or other materials
|
17 |
# provided with the distribution.
|
18 |
#
|
19 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
20 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
21 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
22 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
23 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
26 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
27 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
28 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
29 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
30 |
# POSSIBILITY OF SUCH DAMAGE.
|
31 |
#
|
32 |
# The views and conclusions contained in the software and
|
33 |
# documentation are those of the authors and should not be
|
34 |
# interpreted as representing official policies, either expressed
|
35 |
# or implied, of GRNET S.A.
|
36 |
|
37 |
from urlparse import urlunsplit, urlsplit |
38 |
from xml.dom import minidom |
39 |
|
40 |
from snf_django.utils.testing import with_settings, astakos_user |
41 |
|
42 |
from pithos.api import settings as pithos_settings |
43 |
from pithos.api.test.util import is_date, get_random_data |
44 |
|
45 |
from synnefo.lib.services import get_service_path |
46 |
from synnefo.lib import join_urls |
47 |
|
48 |
from django.test import TestCase |
49 |
from django.utils.http import urlencode |
50 |
|
51 |
import django.utils.simplejson as json |
52 |
|
53 |
import random |
54 |
import threading |
55 |
import functools |
56 |
|
57 |
|
58 |
pithos_test_settings = functools.partial(with_settings, pithos_settings) |
59 |
|
60 |
DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
|
61 |
"%A, %d-%b-%y %H:%M:%S GMT",
|
62 |
"%a, %d %b %Y %H:%M:%S GMT"]
|
63 |
|
64 |
o_names = ['kate.jpg',
|
65 |
'kate_beckinsale.jpg',
|
66 |
'How To Win Friends And Influence People.pdf',
|
67 |
'moms_birthday.jpg',
|
68 |
'poodle_strut.mov',
|
69 |
'Disturbed - Down With The Sickness.mp3',
|
70 |
'army_of_darkness.avi',
|
71 |
'the_mad.avi',
|
72 |
'photos/animals/dogs/poodle.jpg',
|
73 |
'photos/animals/dogs/terrier.jpg',
|
74 |
'photos/animals/cats/persian.jpg',
|
75 |
'photos/animals/cats/siamese.jpg',
|
76 |
'photos/plants/fern.jpg',
|
77 |
'photos/plants/rose.jpg',
|
78 |
'photos/me.jpg']
|
79 |
|
80 |
details = {'container': ('name', 'count', 'bytes', 'last_modified', |
81 |
'x_container_policy'),
|
82 |
'object': ('name', 'hash', 'bytes', 'content_type', |
83 |
'content_encoding', 'last_modified',)} |
84 |
|
85 |
return_codes = (400, 401, 403, 404, 503) |
86 |
|
87 |
TEST_BLOCK_SIZE = 1024
|
88 |
TEST_HASH_ALGORITHM = 'sha256'
|
89 |
|
90 |
|
91 |
class PithosAPITest(TestCase): |
92 |
def setUp(self): |
93 |
# Override default block size to spead up tests
|
94 |
pithos_settings.BACKEND_BLOCK_SIZE = TEST_BLOCK_SIZE |
95 |
pithos_settings.BACKEND_HASH_ALGORITHM = TEST_HASH_ALGORITHM |
96 |
|
97 |
self.user = 'user' |
98 |
self.pithos_path = join_urls(get_service_path(
|
99 |
pithos_settings.pithos_services, 'object-store'))
|
100 |
|
101 |
def tearDown(self): |
102 |
#delete additionally created metadata
|
103 |
meta = self.get_account_meta()
|
104 |
self.delete_account_meta(meta)
|
105 |
|
106 |
#delete additionally created groups
|
107 |
groups = self.get_account_groups()
|
108 |
self.delete_account_groups(groups)
|
109 |
|
110 |
self._clean_account()
|
111 |
|
112 |
def head(self, url, user='user', *args, **kwargs): |
113 |
with astakos_user(user):
|
114 |
response = self.client.head(url, *args, **kwargs)
|
115 |
return response
|
116 |
|
117 |
def get(self, url, user='user', *args, **kwargs): |
118 |
with astakos_user(user):
|
119 |
response = self.client.get(url, *args, **kwargs)
|
120 |
return response
|
121 |
|
122 |
def delete(self, url, user='user', *args, **kwargs): |
123 |
with astakos_user(user):
|
124 |
response = self.client.delete(url, *args, **kwargs)
|
125 |
return response
|
126 |
|
127 |
def post(self, url, user='user', *args, **kwargs): |
128 |
with astakos_user(user):
|
129 |
kwargs.setdefault('content_type', 'application/octet-stream') |
130 |
response = self.client.post(url, *args, **kwargs)
|
131 |
return response
|
132 |
|
133 |
def put(self, url, user='user', *args, **kwargs): |
134 |
with astakos_user(user):
|
135 |
kwargs.setdefault('content_type', 'application/octet-stream') |
136 |
response = self.client.put(url, *args, **kwargs)
|
137 |
return response
|
138 |
|
139 |
def _clean_account(self): |
140 |
for c in self.list_containers(): |
141 |
self.delete_container_content(c['name']) |
142 |
self.delete_container(c['name']) |
143 |
|
144 |
def update_account_meta(self, meta): |
145 |
kwargs = dict(
|
146 |
('HTTP_X_ACCOUNT_META_%s' % k, str(v)) for k, v in meta.items()) |
147 |
url = join_urls(self.pithos_path, self.user) |
148 |
r = self.post('%s?update=' % url, **kwargs) |
149 |
self.assertEqual(r.status_code, 202) |
150 |
account_meta = self.get_account_meta()
|
151 |
(self.assertTrue('X-Account-Meta-%s' % k in account_meta) for |
152 |
k in meta.keys())
|
153 |
(self.assertEqual(account_meta['X-Account-Meta-%s' % k], v) for |
154 |
k, v in meta.items())
|
155 |
|
156 |
def reset_account_meta(self, meta): |
157 |
kwargs = dict(
|
158 |
('HTTP_X_ACCOUNT_META_%s' % k, str(v)) for k, v in meta.items()) |
159 |
url = join_urls(self.pithos_path, self.user) |
160 |
r = self.post(url, **kwargs)
|
161 |
self.assertEqual(r.status_code, 202) |
162 |
account_meta = self.get_account_meta()
|
163 |
(self.assertTrue('X-Account-Meta-%s' % k in account_meta) for |
164 |
k in meta.keys())
|
165 |
(self.assertEqual(account_meta['X-Account-Meta-%s' % k], v) for |
166 |
k, v in meta.items())
|
167 |
|
168 |
def delete_account_meta(self, meta): |
169 |
transform = lambda k: 'HTTP_%s' % k.replace('-', '_').upper() |
170 |
kwargs = dict((transform(k), '') for k, v in meta.items()) |
171 |
url = join_urls(self.pithos_path, self.user) |
172 |
r = self.post('%s?update=' % url, **kwargs) |
173 |
self.assertEqual(r.status_code, 202) |
174 |
account_meta = self.get_account_meta()
|
175 |
(self.assertTrue('X-Account-Meta-%s' % k not in account_meta) for |
176 |
k in meta.keys())
|
177 |
return r
|
178 |
|
179 |
def delete_account_groups(self, groups): |
180 |
url = join_urls(self.pithos_path, self.user) |
181 |
r = self.post('%s?update=' % url, **groups) |
182 |
self.assertEqual(r.status_code, 202) |
183 |
return r
|
184 |
|
185 |
def get_account_info(self, until=None): |
186 |
url = join_urls(self.pithos_path, self.user) |
187 |
if until is not None: |
188 |
parts = list(urlsplit(url))
|
189 |
parts[3] = urlencode({
|
190 |
'until': until
|
191 |
}) |
192 |
url = urlunsplit(parts) |
193 |
r = self.head(url)
|
194 |
self.assertEqual(r.status_code, 204) |
195 |
return r
|
196 |
|
197 |
def get_account_meta(self, until=None): |
198 |
r = self.get_account_info(until=until)
|
199 |
headers = dict(r._headers.values())
|
200 |
map(headers.pop,
|
201 |
[k for k in headers.keys() |
202 |
if not k.startswith('X-Account-Meta-')]) |
203 |
return headers
|
204 |
|
205 |
def get_account_groups(self, until=None): |
206 |
r = self.get_account_info(until=until)
|
207 |
headers = dict(r._headers.values())
|
208 |
map(headers.pop,
|
209 |
[k for k in headers.keys() |
210 |
if not k.startswith('X-Account-Group-')]) |
211 |
return headers
|
212 |
|
213 |
def get_container_info(self, container, until=None): |
214 |
url = join_urls(self.pithos_path, self.user, container) |
215 |
if until is not None: |
216 |
parts = list(urlsplit(url))
|
217 |
parts[3] = urlencode({
|
218 |
'until': until
|
219 |
}) |
220 |
url = urlunsplit(parts) |
221 |
r = self.head(url)
|
222 |
self.assertEqual(r.status_code, 204) |
223 |
return r
|
224 |
|
225 |
def get_container_meta(self, container, until=None): |
226 |
r = self.get_container_info(container, until=until)
|
227 |
headers = dict(r._headers.values())
|
228 |
map(headers.pop,
|
229 |
[k for k in headers.keys() |
230 |
if not k.startswith('X-Container-Meta-')]) |
231 |
return headers
|
232 |
|
233 |
def update_container_meta(self, container, meta): |
234 |
kwargs = dict(
|
235 |
('HTTP_X_CONTAINER_META_%s' % k, str(v)) for k, v in meta.items()) |
236 |
url = join_urls(self.pithos_path, self.user, container) |
237 |
r = self.post('%s?update=' % url, **kwargs) |
238 |
self.assertEqual(r.status_code, 202) |
239 |
container_meta = self.get_container_meta(container)
|
240 |
(self.assertTrue('X-Container-Meta-%s' % k in container_meta) for |
241 |
k in meta.keys())
|
242 |
(self.assertEqual(container_meta['X-Container-Meta-%s' % k], v) for |
243 |
k, v in meta.items())
|
244 |
|
245 |
def list_containers(self, format='json', headers={}, **params): |
246 |
_url = join_urls(self.pithos_path, self.user) |
247 |
parts = list(urlsplit(_url))
|
248 |
params['format'] = format
|
249 |
parts[3] = urlencode(params)
|
250 |
url = urlunsplit(parts) |
251 |
_headers = dict(('HTTP_%s' % k.upper(), str(v)) |
252 |
for k, v in headers.items()) |
253 |
r = self.get(url, **_headers)
|
254 |
|
255 |
if format is None: |
256 |
containers = r.content.split('\n')
|
257 |
if '' in containers: |
258 |
containers.remove('')
|
259 |
return containers
|
260 |
elif format == 'json': |
261 |
try:
|
262 |
containers = json.loads(r.content) |
263 |
except:
|
264 |
self.fail('json format expected') |
265 |
return containers
|
266 |
elif format == 'xml': |
267 |
return minidom.parseString(r.content)
|
268 |
|
269 |
def delete_container_content(self, cname): |
270 |
url = join_urls(self.pithos_path, self.user, cname) |
271 |
r = self.delete('%s?delimiter=/' % url) |
272 |
self.assertEqual(r.status_code, 204) |
273 |
return r
|
274 |
|
275 |
def delete_container(self, cname): |
276 |
url = join_urls(self.pithos_path, self.user, cname) |
277 |
r = self.delete(url)
|
278 |
self.assertEqual(r.status_code, 204) |
279 |
return r
|
280 |
|
281 |
def create_container(self, cname): |
282 |
url = join_urls(self.pithos_path, self.user, cname) |
283 |
r = self.put(url, data='') |
284 |
self.assertTrue(r.status_code in (202, 201)) |
285 |
return r
|
286 |
|
287 |
def upload_object(self, cname, oname=None, length=None, verify=True, |
288 |
**meta): |
289 |
oname = oname or get_random_data(8) |
290 |
length = length or random.randint(TEST_BLOCK_SIZE, 2 * TEST_BLOCK_SIZE) |
291 |
data = get_random_data(length=length) |
292 |
headers = dict(('HTTP_X_OBJECT_META_%s' % k.upper(), v) |
293 |
for k, v in meta.iteritems()) |
294 |
url = join_urls(self.pithos_path, self.user, cname, oname) |
295 |
r = self.put(url, data=data, **headers)
|
296 |
if verify:
|
297 |
self.assertEqual(r.status_code, 201) |
298 |
return oname, data, r
|
299 |
|
300 |
def update_object_data(self, cname, oname=None, length=None, |
301 |
content_type=None, content_range=None, |
302 |
verify=True, **meta):
|
303 |
oname = oname or get_random_data(8) |
304 |
length = length or random.randint(TEST_BLOCK_SIZE, 2 * TEST_BLOCK_SIZE) |
305 |
content_type = content_type or 'application/octet-stream' |
306 |
data = get_random_data(length=length) |
307 |
headers = dict(('HTTP_X_OBJECT_META_%s' % k.upper(), v) |
308 |
for k, v in meta.iteritems()) |
309 |
if content_range:
|
310 |
headers['HTTP_CONTENT_RANGE'] = content_range
|
311 |
url = join_urls(self.pithos_path, self.user, cname, oname) |
312 |
r = self.post(url, data=data, content_type=content_type, **headers)
|
313 |
if verify:
|
314 |
self.assertEqual(r.status_code, 204) |
315 |
return oname, data, r
|
316 |
|
317 |
def append_object_data(self, cname, oname=None, length=None, |
318 |
content_type=None):
|
319 |
return self.update_object_data(cname, oname=oname, |
320 |
length=length, |
321 |
content_type=content_type, |
322 |
content_range='bytes */*')
|
323 |
|
324 |
def create_folder(self, cname, oname=None, **headers): |
325 |
oname = oname or get_random_data(8) |
326 |
url = join_urls(self.pithos_path, self.user, cname, oname) |
327 |
r = self.put(url, data='', content_type='application/directory', |
328 |
**headers) |
329 |
self.assertEqual(r.status_code, 201) |
330 |
return oname, r
|
331 |
|
332 |
def list_objects(self, cname, prefix=None): |
333 |
url = join_urls(self.pithos_path, self.user, cname) |
334 |
path = '%s?format=json' % url
|
335 |
if prefix is not None: |
336 |
path = '%s&prefix=%s' % (path, prefix)
|
337 |
r = self.get(path)
|
338 |
self.assertTrue(r.status_code in (200, 204)) |
339 |
try:
|
340 |
objects = json.loads(r.content) |
341 |
except:
|
342 |
self.fail('json format expected') |
343 |
return objects
|
344 |
|
345 |
def get_object_info(self, container, object, version=None, until=None): |
346 |
url = join_urls(self.pithos_path, self.user, container, object) |
347 |
if until is not None: |
348 |
parts = list(urlsplit(url))
|
349 |
parts[3] = urlencode({
|
350 |
'until': until
|
351 |
}) |
352 |
url = urlunsplit(parts) |
353 |
if version:
|
354 |
url = '%s?version=%s' % (url, version)
|
355 |
r = self.head(url)
|
356 |
self.assertEqual(r.status_code, 200) |
357 |
return r
|
358 |
|
359 |
def get_object_meta(self, container, object, version=None, until=None): |
360 |
r = self.get_object_info(container, object, version, until=until) |
361 |
headers = dict(r._headers.values())
|
362 |
map(headers.pop,
|
363 |
[k for k in headers.keys() |
364 |
if not k.startswith('X-Object-Meta-')]) |
365 |
return headers
|
366 |
|
367 |
def update_object_meta(self, container, object, meta): |
368 |
kwargs = dict(
|
369 |
('HTTP_X_OBJECT_META_%s' % k, str(v)) for k, v in meta.items()) |
370 |
url = join_urls(self.pithos_path, self.user, container, object) |
371 |
r = self.post('%s?update=' % url, content_type='', **kwargs) |
372 |
self.assertEqual(r.status_code, 202) |
373 |
object_meta = self.get_object_meta(container, object) |
374 |
(self.assertTrue('X-Objecr-Meta-%s' % k in object_meta) for |
375 |
k in meta.keys())
|
376 |
(self.assertEqual(object_meta['X-Object-Meta-%s' % k], v) for |
377 |
k, v in meta.items())
|
378 |
|
379 |
def assert_status(self, status, codes): |
380 |
l = [elem for elem in return_codes] |
381 |
if isinstance(codes, list): |
382 |
l.extend(codes) |
383 |
else:
|
384 |
l.append(codes) |
385 |
self.assertTrue(status in l) |
386 |
|
387 |
def assert_extended(self, data, format, type, size=10000): |
388 |
if format == 'xml': |
389 |
self._assert_xml(data, type, size) |
390 |
elif format == 'json': |
391 |
self._assert_json(data, type, size) |
392 |
|
393 |
def _assert_json(self, data, type, size): |
394 |
convert = lambda s: s.lower()
|
395 |
info = [convert(elem) for elem in details[type]] |
396 |
self.assertTrue(len(data) <= size) |
397 |
for item in info: |
398 |
for i in data: |
399 |
if 'subdir' in i.keys(): |
400 |
continue
|
401 |
self.assertTrue(item in i.keys()) |
402 |
|
403 |
def _assert_xml(self, data, type, size): |
404 |
convert = lambda s: s.lower()
|
405 |
info = [convert(elem) for elem in details[type]] |
406 |
try:
|
407 |
info.remove('content_encoding')
|
408 |
except ValueError: |
409 |
pass
|
410 |
xml = data |
411 |
entities = xml.getElementsByTagName(type)
|
412 |
self.assertTrue(len(entities) <= size) |
413 |
for e in entities: |
414 |
for item in info: |
415 |
self.assertTrue(e.getElementsByTagName(item))
|
416 |
|
417 |
|
418 |
class AssertMappingInvariant(object): |
419 |
def __init__(self, callable, *args, **kwargs): |
420 |
self.callable = callable |
421 |
self.args = args
|
422 |
self.kwargs = kwargs
|
423 |
|
424 |
def __enter__(self): |
425 |
self.map = self.callable(*self.args, **self.kwargs) |
426 |
return self.map |
427 |
|
428 |
def __exit__(self, type, value, tb): |
429 |
map = self.callable(*self.args, **self.kwargs) |
430 |
for k, v in self.map.items(): |
431 |
if is_date(v):
|
432 |
continue
|
433 |
|
434 |
assert(k in map), '%s not in map' % k |
435 |
assert v == map[k] |
436 |
|
437 |
|
438 |
class AssertUUidInvariant(object): |
439 |
def __init__(self, callable, *args, **kwargs): |
440 |
self.callable = callable |
441 |
self.args = args
|
442 |
self.kwargs = kwargs
|
443 |
|
444 |
def __enter__(self): |
445 |
self.map = self.callable(*self.args, **self.kwargs) |
446 |
assert('x-object-uuid' in self.map) |
447 |
self.uuid = self.map['x-object-uuid'] |
448 |
return self.map |
449 |
|
450 |
def __exit__(self, type, value, tb): |
451 |
map = self.callable(*self.args, **self.kwargs) |
452 |
assert('x-object-uuid' in self.map) |
453 |
uuid = map['x-object-uuid'] |
454 |
assert(uuid == self.uuid) |
455 |
|
456 |
def test_concurrently(times=2): |
457 |
"""
|
458 |
Add this decorator to small pieces of code that you want to test
|
459 |
concurrently to make sure they don't raise exceptions when run at the
|
460 |
same time. E.g., some Django views that do a SELECT and then a subsequent
|
461 |
INSERT might fail when the INSERT assumes that the data has not changed
|
462 |
since the SELECT.
|
463 |
"""
|
464 |
def test_concurrently_decorator(test_func): |
465 |
def wrapper(*args, **kwargs): |
466 |
exceptions = [] |
467 |
|
468 |
def call_test_func(): |
469 |
try:
|
470 |
test_func(*args, **kwargs) |
471 |
except Exception, e: |
472 |
exceptions.append(e) |
473 |
raise
|
474 |
|
475 |
threads = [] |
476 |
for i in range(times): |
477 |
threads.append(threading.Thread()) |
478 |
for t in threads: |
479 |
t.start() |
480 |
for t in threads: |
481 |
t.join() |
482 |
if exceptions:
|
483 |
raise Exception( |
484 |
('test_concurrently intercepted %s',
|
485 |
'exceptions: %s') % (len(exceptions), exceptions)) |
486 |
return wrapper
|
487 |
return test_concurrently_decorator
|