Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / test / __init__.py @ 6ee6677e

History | View | Annotate | Download (19.6 kB)

1
#!/usr/bin/env python
2
#coding=utf8
3

    
4
# Copyright 2011-2013 GRNET S.A. All rights reserved.
5
#
6
# Redistribution and use in source and binary forms, with or
7
# without modification, are permitted provided that the following
8
# conditions are met:
9
#
10
#   1. Redistributions of source code must retain the above
11
#      copyright notice, this list of conditions and the following
12
#      disclaimer.
13
#
14
#   2. Redistributions in binary form must reproduce the above
15
#      copyright notice, this list of conditions and the following
16
#      disclaimer in the documentation and/or other materials
17
#      provided with the distribution.
18
#
19
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
20
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
23
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
26
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
29
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30
# POSSIBILITY OF SUCH DAMAGE.
31
#
32
# The views and conclusions contained in the software and
33
# documentation are those of the authors and should not be
34
# interpreted as representing official policies, either expressed
35
# or implied, of GRNET S.A.
36

    
37
from urlparse import urlunsplit, urlsplit
38
from xml.dom import minidom
39
from urllib import quote, unquote
40

    
41
from snf_django.utils.testing import with_settings, astakos_user
42

    
43
from pithos.api import settings as pithos_settings
44
from pithos.api.test.util import is_date, get_random_data, get_random_name
45
from pithos.backends.migrate import initialize_db
46

    
47
from synnefo.lib.services import get_service_path
48
from synnefo.lib import join_urls
49

    
50
from django.test import TestCase
51
from django.test.simple import DjangoTestSuiteRunner
52
from django.conf import settings
53
from django.utils.http import urlencode
54
from django.db.backends.creation import TEST_DATABASE_PREFIX
55

    
56
import django.utils.simplejson as json
57

    
58
import random
59
import functools
60

    
61

    
62
pithos_test_settings = functools.partial(with_settings, pithos_settings)
63

    
64
DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
65
                "%A, %d-%b-%y %H:%M:%S GMT",
66
                "%a, %d %b %Y %H:%M:%S GMT"]
67

    
68
o_names = ['kate.jpg',
69
           'kate_beckinsale.jpg',
70
           'How To Win Friends And Influence People.pdf',
71
           'moms_birthday.jpg',
72
           'poodle_strut.mov',
73
           'Disturbed - Down With The Sickness.mp3',
74
           'army_of_darkness.avi',
75
           'the_mad.avi',
76
           'photos/animals/dogs/poodle.jpg',
77
           'photos/animals/dogs/terrier.jpg',
78
           'photos/animals/cats/persian.jpg',
79
           'photos/animals/cats/siamese.jpg',
80
           'photos/plants/fern.jpg',
81
           'photos/plants/rose.jpg',
82
           'photos/me.jpg']
83

    
84
details = {'container': ('name', 'count', 'bytes', 'last_modified',
85
                         'x_container_policy'),
86
           'object': ('name', 'hash', 'bytes', 'content_type',
87
                      'content_encoding', 'last_modified',)}
88

    
89
TEST_BLOCK_SIZE = 1024
90
TEST_HASH_ALGORITHM = 'sha256'
91

    
92
print 'backend module:', pithos_settings.BACKEND_DB_MODULE
93
print 'backend database engine:', settings.DATABASES['default']['ENGINE']
94
print 'update md5:', pithos_settings.UPDATE_MD5
95

    
96

    
97
django_sqlalchemy_engines = {
98
    'django.db.backends.postgresql_psycopg2': 'postgresql+psycopg2',
99
    'django.db.backends.postgresql': 'postgresql',
100
    'django.db.backends.mysql': '',
101
    'django.db.backends.sqlite3': 'mssql',
102
    'django.db.backends.oracle': 'oracle'}
103

    
104

    
105
def prepate_db_connection():
106
    """Build pithos backend connection string from django default database"""
107

    
108
    db = settings.DATABASES['default']
109
    name = db.get('TEST_NAME', TEST_DATABASE_PREFIX + db['NAME'])
110

    
111
    if (pithos_settings.BACKEND_DB_MODULE == 'pithos.backends.lib.sqlalchemy'):
112
        if db['ENGINE'] == 'django.db.backends.sqlite3':
113
            db_connection = 'sqlite:///%s' % name
114
        else:
115
            d = dict(scheme=django_sqlalchemy_engines.get(db['ENGINE']),
116
                     user=db['USER'],
117
                     pwd=db['PASSWORD'],
118
                     host=db['HOST'].lower(),
119
                     port=int(db['PORT']) if db['PORT'] != '' else '',
120
                     name=name)
121
            db_connection = (
122
                '%(scheme)s://%(user)s:%(pwd)s@%(host)s:%(port)s/%(name)s' % d)
123

    
124
            # initialize pithos database
125
            initialize_db(db_connection)
126
    else:
127
        db_connection = name
128
    pithos_settings.BACKEND_DB_CONNECTION = db_connection
129

    
130

    
131
def filter_headers(headers, prefix):
132
    meta = {}
133
    for k, v in headers.iteritems():
134
        if not k.startswith(prefix):
135
            continue
136
        meta[unquote(k[len(prefix):])] = unquote(v)
137
    return meta
138

    
139

    
140
class PithosTestSuiteRunner(DjangoTestSuiteRunner):
141
    def setup_databases(self, **kwargs):
142
        old_names, mirrors = super(PithosTestSuiteRunner,
143
                                   self).setup_databases(**kwargs)
144
        prepate_db_connection()
145
        return old_names, mirrors
146

    
147

    
148
class PithosAPITest(TestCase):
149
    def setUp(self):
150
        # Override default block size to spead up tests
151
        pithos_settings.BACKEND_BLOCK_SIZE = TEST_BLOCK_SIZE
152
        pithos_settings.BACKEND_HASH_ALGORITHM = TEST_HASH_ALGORITHM
153

    
154
        self.user = 'user'
155
        self.pithos_path = join_urls(get_service_path(
156
            pithos_settings.pithos_services, 'object-store'))
157

    
158
    def tearDown(self):
159
        #delete additionally created metadata
160
        meta = self.get_account_meta()
161
        self.delete_account_meta(meta)
162

    
163
        #delete additionally created groups
164
        groups = self.get_account_groups()
165
        self.delete_account_groups(groups)
166

    
167
        self._clean_account()
168

    
169
    def _clean_account(self):
170
        for c in self.list_containers():
171
            self.delete_container_content(c['name'])
172
            self.delete_container(c['name'])
173

    
174
    def head(self, url, user='user', data={}, follow=False, **extra):
175
        with astakos_user(user):
176
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
177
            response = self.client.head(url, data, follow, **extra)
178
        return response
179

    
180
    def get(self, url, user='user', data={}, follow=False, **extra):
181
        with astakos_user(user):
182
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
183
            response = self.client.get(url, data, follow, **extra)
184
        return response
185

    
186
    def delete(self, url, user='user', data={}, follow=False, **extra):
187
        with astakos_user(user):
188
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
189
            response = self.client.delete(url, data, follow, **extra)
190
        return response
191

    
192
    def post(self, url, user='user', data={},
193
             content_type='application/octet-stream', follow=False, **extra):
194
        with astakos_user(user):
195
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
196
            response = self.client.post(url, data, content_type, follow,
197
                                        **extra)
198
        return response
199

    
200
    def put(self, url, user='user', data={},
201
            content_type='application/octet-stream', follow=False, **extra):
202
        with astakos_user(user):
203
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
204
            response = self.client.put(url, data, content_type, follow,
205
                                       **extra)
206
        return response
207

    
208
    def update_account_meta(self, meta, user=None):
209
        user = user or self.user
210
        kwargs = dict(
211
            ('HTTP_X_ACCOUNT_META_%s' % k, str(v)) for k, v in meta.items())
212
        url = join_urls(self.pithos_path, user)
213
        r = self.post('%s?update=' % url, user=user, **kwargs)
214
        self.assertEqual(r.status_code, 202)
215
        account_meta = self.get_account_meta()
216
        (self.assertTrue(k in account_meta) for k in meta.keys())
217
        (self.assertEqual(account_meta[k], v) for k, v in meta.items())
218

    
219
    def delete_account_meta(self, meta, user=None):
220
        user = user or self.user
221
        transform = lambda k: 'HTTP_X_ACCOUNT_META_%s' % k.replace('-', '_').upper()
222
        kwargs = dict((transform(k), '') for k, v in meta.items())
223
        url = join_urls(self.pithos_path, user)
224
        r = self.post('%s?update=' % url, user=user, **kwargs)
225
        self.assertEqual(r.status_code, 202)
226
        account_meta = self.get_account_meta()
227
        (self.assertTrue(k not in account_meta) for k in meta.keys())
228
        return r
229

    
230
    def delete_account_groups(self, groups, user=None):
231
        user = user or self.user
232
        url = join_urls(self.pithos_path, user)
233
        transform = lambda k: 'HTTP_X_ACCOUNT_GROUP_%s' % k.replace('-', '_').upper()
234
        kwargs = dict((transform(k), '') for k, v in groups.items())
235
        r = self.post('%s?update=' % url, user=user, **kwargs)
236
        self.assertEqual(r.status_code, 202)
237
        account_groups = self.get_account_groups()
238
        (self.assertTrue(k not in account_groups) for k in groups.keys())
239
        return r
240

    
241
    def get_account_info(self, until=None, user=None):
242
        user = user or self.user
243
        url = join_urls(self.pithos_path, user)
244
        if until is not None:
245
            parts = list(urlsplit(url))
246
            parts[3] = urlencode({
247
                'until': until
248
            })
249
            url = urlunsplit(parts)
250
        r = self.head(url, user=user)
251
        self.assertEqual(r.status_code, 204)
252
        return r
253

    
254
    def get_account_meta(self, until=None, user=None):
255
        prefix = 'X-Account-Meta-'
256
        r = self.get_account_info(until=until, user=user)
257
        headers = dict(r._headers.values())
258
        return filter_headers(headers, prefix)
259

    
260
    def get_account_groups(self, until=None, user=None):
261
        prefix = 'X-Account-Group-'
262
        r = self.get_account_info(until=until, user=user)
263
        headers = dict(r._headers.values())
264
        return filter_headers(headers, prefix)
265

    
266
    def get_container_info(self, container, until=None, user=None):
267
        user = user or self.user
268
        url = join_urls(self.pithos_path, user, container)
269
        if until is not None:
270
            parts = list(urlsplit(url))
271
            parts[3] = urlencode({
272
                'until': until
273
            })
274
            url = urlunsplit(parts)
275
        r = self.head(url, user=user)
276
        self.assertEqual(r.status_code, 204)
277
        return r
278

    
279
    def get_container_meta(self, container, until=None, user=None):
280
        prefix = 'X-Container-Meta-'
281
        r = self.get_container_info(container, until=until, user=user)
282
        headers = dict(r._headers.values())
283
        return filter_headers(headers, prefix)
284

    
285
    def update_container_meta(self, container, meta, user=None):
286
        user = user or self.user
287
        kwargs = dict(
288
            ('HTTP_X_CONTAINER_META_%s' % k, str(v)) for k, v in meta.items())
289
        url = join_urls(self.pithos_path, user, container)
290
        r = self.post('%s?update=' % url, user=user, **kwargs)
291
        self.assertEqual(r.status_code, 202)
292
        container_meta = self.get_container_meta(container)
293
        (self.assertTrue('X-Container-Meta-%s' % k in container_meta) for
294
            k in meta.keys())
295
        (self.assertEqual(container_meta['X-Container-Meta-%s' % k], v) for
296
            k, v in meta.items())
297

    
298
    def list_containers(self, format='json', headers={}, user=None, **params):
299
        user = user or self.user
300
        _url = join_urls(self.pithos_path, user)
301
        parts = list(urlsplit(_url))
302
        params['format'] = format
303
        parts[3] = urlencode(params)
304
        url = urlunsplit(parts)
305
        _headers = dict(('HTTP_%s' % k.upper(), str(v))
306
                        for k, v in headers.items())
307
        r = self.get(url, user=user, **_headers)
308

    
309
        if format is None:
310
            containers = r.content.split('\n')
311
            if '' in containers:
312
                containers.remove('')
313
            return containers
314
        elif format == 'json':
315
            try:
316
                containers = json.loads(r.content)
317
            except:
318
                self.fail('json format expected')
319
            return containers
320
        elif format == 'xml':
321
            return minidom.parseString(r.content)
322

    
323
    def delete_container_content(self, cname, user=None):
324
        user = user or self.user
325
        url = join_urls(self.pithos_path, user, cname)
326
        r = self.delete('%s?delimiter=/' % url, user=user)
327
        self.assertEqual(r.status_code, 204)
328
        return r
329

    
330
    def delete_container(self, cname, user=None):
331
        user = user or self.user
332
        url = join_urls(self.pithos_path, user, cname)
333
        r = self.delete(url, user=user)
334
        self.assertEqual(r.status_code, 204)
335
        return r
336

    
337
    def create_container(self, cname=None, user=None):
338
        cname = cname or get_random_name()
339
        user = user or self.user
340
        url = join_urls(self.pithos_path, user, cname)
341
        r = self.put(url, user=user, data='')
342
        self.assertTrue(r.status_code in (202, 201))
343
        return cname, r
344

    
345
    def upload_object(self, cname, oname=None, length=None, verify=True,
346
                      user=None, **meta):
347
        oname = oname or get_random_name()
348
        length = length or random.randint(TEST_BLOCK_SIZE, 2 * TEST_BLOCK_SIZE)
349
        user = user or self.user
350
        data = get_random_data(length=length)
351
        headers = dict(('HTTP_X_OBJECT_META_%s' % k.upper(), v)
352
                       for k, v in meta.iteritems())
353
        url = join_urls(self.pithos_path, user, cname, oname)
354
        r = self.put(url, user=user, data=data, **headers)
355
        if verify:
356
            self.assertEqual(r.status_code, 201)
357
        return oname, data, r
358

    
359
    def update_object_data(self, cname, oname=None, length=None,
360
                           content_type=None, content_range=None,
361
                           verify=True, user=None, **meta):
362
        oname = oname or get_random_name()
363
        length = length or random.randint(TEST_BLOCK_SIZE, 2 * TEST_BLOCK_SIZE)
364
        content_type = content_type or 'application/octet-stream'
365
        user = user or self.user
366
        data = get_random_data(length=length)
367
        headers = dict(('HTTP_X_OBJECT_META_%s' % k.upper(), v)
368
                       for k, v in meta.iteritems())
369
        if content_range:
370
            headers['HTTP_CONTENT_RANGE'] = content_range
371
        url = join_urls(self.pithos_path, user, cname, oname)
372
        r = self.post(url, user=user, data=data, content_type=content_type,
373
                      **headers)
374
        if verify:
375
            self.assertEqual(r.status_code, 204)
376
        return oname, data, r
377

    
378
    def append_object_data(self, cname, oname=None, length=None,
379
                           content_type=None, user=None):
380
        return self.update_object_data(cname, oname=oname,
381
                                       length=length,
382
                                       content_type=content_type,
383
                                       content_range='bytes */*',
384
                                       user=user)
385

    
386
    def create_folder(self, cname, oname=None, user=None, **headers):
387
        user = user or self.user
388
        oname = oname or get_random_name()
389
        url = join_urls(self.pithos_path, user, cname, oname)
390
        r = self.put(url, user=user, data='',
391
                     content_type='application/directory', **headers)
392
        self.assertEqual(r.status_code, 201)
393
        return oname, r
394

    
395
    def list_objects(self, cname, prefix=None, user=None):
396
        user = user or self.user
397
        url = join_urls(self.pithos_path, user, cname)
398
        path = '%s?format=json' % url
399
        if prefix is not None:
400
            path = '%s&prefix=%s' % (path, prefix)
401
        r = self.get(path, user=user)
402
        self.assertTrue(r.status_code in (200, 204))
403
        try:
404
            objects = json.loads(r.content)
405
        except:
406
            self.fail('json format expected')
407
        return objects
408

    
409
    def get_object_info(self, container, object, version=None, until=None,
410
                        user=None):
411
        user = user or self.user
412
        url = join_urls(self.pithos_path, user, container, object)
413
        if until is not None:
414
            parts = list(urlsplit(url))
415
            parts[3] = urlencode({
416
                'until': until
417
            })
418
            url = urlunsplit(parts)
419
        if version:
420
            url = '%s?version=%s' % (url, version)
421
        r = self.head(url, user=user)
422
        self.assertEqual(r.status_code, 200)
423
        return r
424

    
425
    def get_object_meta(self, container, object, version=None, until=None,
426
                        user=None):
427
        prefix = 'X-Object-Meta-'
428
        r = self.get_object_info(container, object, version, until=until,
429
                                 user=user)
430
        headers = dict(r._headers.values())
431
        return filter_headers(headers, prefix)
432

    
433
    def update_object_meta(self, container, object, meta, user=None):
434
        user = user or self.user
435
        kwargs = dict(
436
            ('HTTP_X_OBJECT_META_%s' % k, str(v)) for k, v in meta.items())
437
        url = join_urls(self.pithos_path, user, container, object)
438
        r = self.post('%s?update=' % url, user=user, content_type='', **kwargs)
439
        self.assertEqual(r.status_code, 202)
440
        object_meta = self.get_object_meta(container, object)
441
        (self.assertTrue('X-Objecr-Meta-%s' % k in object_meta) for
442
            k in meta.keys())
443
        (self.assertEqual(object_meta['X-Object-Meta-%s' % k], v) for
444
            k, v in meta.items())
445

    
446
    def assert_extended(self, data, format, type, size=10000):
447
        if format == 'xml':
448
            self._assert_xml(data, type, size)
449
        elif format == 'json':
450
            self._assert_json(data, type, size)
451

    
452
    def _assert_json(self, data, type, size):
453
        convert = lambda s: s.lower()
454
        info = [convert(elem) for elem in details[type]]
455
        self.assertTrue(len(data) <= size)
456
        for item in info:
457
            for i in data:
458
                if 'subdir' in i.keys():
459
                    continue
460
                self.assertTrue(item in i.keys())
461

    
462
    def _assert_xml(self, data, type, size):
463
        convert = lambda s: s.lower()
464
        info = [convert(elem) for elem in details[type]]
465
        try:
466
            info.remove('content_encoding')
467
        except ValueError:
468
            pass
469
        xml = data
470
        entities = xml.getElementsByTagName(type)
471
        self.assertTrue(len(entities) <= size)
472
        for e in entities:
473
            for item in info:
474
                self.assertTrue(e.getElementsByTagName(item))
475

    
476

    
477
class AssertMappingInvariant(object):
478
    def __init__(self, callable, *args, **kwargs):
479
        self.callable = callable
480
        self.args = args
481
        self.kwargs = kwargs
482

    
483
    def __enter__(self):
484
        self.map = self.callable(*self.args, **self.kwargs)
485
        return self.map
486

    
487
    def __exit__(self, type, value, tb):
488
        map = self.callable(*self.args, **self.kwargs)
489
        for k, v in self.map.items():
490
            if is_date(v):
491
                continue
492

    
493
            assert(k in map), '%s not in map' % k
494
            assert v == map[k]
495

    
496

    
497
class AssertUUidInvariant(object):
498
    def __init__(self, callable, *args, **kwargs):
499
        self.callable = callable
500
        self.args = args
501
        self.kwargs = kwargs
502

    
503
    def __enter__(self):
504
        self.map = self.callable(*self.args, **self.kwargs)
505
        assert('x-object-uuid' in self.map)
506
        self.uuid = self.map['x-object-uuid']
507
        return self.map
508

    
509
    def __exit__(self, type, value, tb):
510
        map = self.callable(*self.args, **self.kwargs)
511
        assert('x-object-uuid' in self.map)
512
        uuid = map['x-object-uuid']
513
        assert(uuid == self.uuid)