Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / test / __init__.py @ 6e0f3e65

History | View | Annotate | Download (19.9 kB)

1
#!/usr/bin/env python
2
#coding=utf8
3

    
4
# Copyright 2011-2013 GRNET S.A. All rights reserved.
5
#
6
# Redistribution and use in source and binary forms, with or
7
# without modification, are permitted provided that the following
8
# conditions are met:
9
#
10
#   1. Redistributions of source code must retain the above
11
#      copyright notice, this list of conditions and the following
12
#      disclaimer.
13
#
14
#   2. Redistributions in binary form must reproduce the above
15
#      copyright notice, this list of conditions and the following
16
#      disclaimer in the documentation and/or other materials
17
#      provided with the distribution.
18
#
19
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
20
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
23
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
26
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
29
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30
# POSSIBILITY OF SUCH DAMAGE.
31
#
32
# The views and conclusions contained in the software and
33
# documentation are those of the authors and should not be
34
# interpreted as representing official policies, either expressed
35
# or implied, of GRNET S.A.
36

    
37
from urlparse import urlunsplit, urlsplit
38
from xml.dom import minidom
39
from urllib import quote, unquote
40

    
41
from snf_django.utils.testing import with_settings, astakos_user
42

    
43
from pithos.api import settings as pithos_settings
44
from pithos.api.test.util import is_date, get_random_data, get_random_name
45
from pithos.backends.migrate import initialize_db
46

    
47
from synnefo.lib.services import get_service_path
48
from synnefo.lib import join_urls
49

    
50
from django.test import TestCase
51
from django.test.simple import DjangoTestSuiteRunner
52
from django.conf import settings
53
from django.utils.http import urlencode
54
from django.db.backends.creation import TEST_DATABASE_PREFIX
55

    
56
import django.utils.simplejson as json
57

    
58
import random
59
import functools
60

    
61

    
62
pithos_test_settings = functools.partial(with_settings, pithos_settings)
63

    
64
DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
65
                "%A, %d-%b-%y %H:%M:%S GMT",
66
                "%a, %d %b %Y %H:%M:%S GMT"]
67

    
68
o_names = ['kate.jpg',
69
           'kate_beckinsale.jpg',
70
           'How To Win Friends And Influence People.pdf',
71
           'moms_birthday.jpg',
72
           'poodle_strut.mov',
73
           'Disturbed - Down With The Sickness.mp3',
74
           'army_of_darkness.avi',
75
           'the_mad.avi',
76
           'photos/animals/dogs/poodle.jpg',
77
           'photos/animals/dogs/terrier.jpg',
78
           'photos/animals/cats/persian.jpg',
79
           'photos/animals/cats/siamese.jpg',
80
           'photos/plants/fern.jpg',
81
           'photos/plants/rose.jpg',
82
           'photos/me.jpg']
83

    
84
details = {'container': ('name', 'count', 'bytes', 'last_modified',
85
                         'x_container_policy'),
86
           'object': ('name', 'hash', 'bytes', 'content_type',
87
                      'content_encoding', 'last_modified',)}
88

    
89
TEST_BLOCK_SIZE = 1024
90
TEST_HASH_ALGORITHM = 'sha256'
91

    
92
print 'backend module:', pithos_settings.BACKEND_DB_MODULE
93
print 'backend database engine:', settings.DATABASES['default']['ENGINE']
94
print 'update md5:', pithos_settings.UPDATE_MD5
95

    
96

    
97
django_sqlalchemy_engines = {
98
    'django.db.backends.postgresql_psycopg2': 'postgresql+psycopg2',
99
    'django.db.backends.postgresql': 'postgresql',
100
    'django.db.backends.mysql': '',
101
    'django.db.backends.sqlite3': 'mssql',
102
    'django.db.backends.oracle': 'oracle'}
103

    
104

    
105
def prepate_db_connection():
106
    """Build pithos backend connection string from django default database"""
107

    
108
    db = settings.DATABASES['default']
109
    name = db.get('TEST_NAME', TEST_DATABASE_PREFIX + db['NAME'])
110

    
111
    if (pithos_settings.BACKEND_DB_MODULE == 'pithos.backends.lib.sqlalchemy'):
112
        if db['ENGINE'] == 'django.db.backends.sqlite3':
113
            db_connection = 'sqlite:///%s' % name
114
        else:
115
            d = dict(scheme=django_sqlalchemy_engines.get(db['ENGINE']),
116
                     user=db['USER'],
117
                     pwd=db['PASSWORD'],
118
                     host=db['HOST'].lower(),
119
                     port=int(db['PORT']) if db['PORT'] != '' else '',
120
                     name=name)
121
            db_connection = (
122
                '%(scheme)s://%(user)s:%(pwd)s@%(host)s:%(port)s/%(name)s' % d)
123

    
124
            # initialize pithos database
125
            initialize_db(db_connection)
126
    else:
127
        db_connection = name
128
    pithos_settings.BACKEND_DB_CONNECTION = db_connection
129

    
130

    
131
def filter_headers(headers, prefix):
132
    meta = {}
133
    for k, v in headers.iteritems():
134
        if not k.startswith(prefix):
135
            continue
136
        meta[unquote(k[len(prefix):])] = unquote(v)
137
    return meta
138

    
139

    
140
class PithosTestSuiteRunner(DjangoTestSuiteRunner):
141
    def setup_databases(self, **kwargs):
142
        old_names, mirrors = super(PithosTestSuiteRunner,
143
                                   self).setup_databases(**kwargs)
144
        prepate_db_connection()
145
        return old_names, mirrors
146

    
147

    
148
class PithosAPITest(TestCase):
149
    def setUp(self):
150
        # Override default block size to spead up tests
151
        pithos_settings.BACKEND_BLOCK_SIZE = TEST_BLOCK_SIZE
152
        pithos_settings.BACKEND_HASH_ALGORITHM = TEST_HASH_ALGORITHM
153

    
154
        self.user = 'user'
155
        self.pithos_path = join_urls(get_service_path(
156
            pithos_settings.pithos_services, 'object-store'))
157

    
158
    def tearDown(self):
159
        #delete additionally created metadata
160
        meta = self.get_account_meta()
161
        self.delete_account_meta(meta)
162

    
163
        #delete additionally created groups
164
        groups = self.get_account_groups()
165
        self.delete_account_groups(groups)
166

    
167
        self._clean_account()
168

    
169
    def _clean_account(self):
170
        for c in self.list_containers():
171
            self.delete_container_content(c['name'])
172
            self.delete_container(c['name'])
173

    
174
    def head(self, url, user='user', data={}, follow=False, **extra):
175
        with astakos_user(user):
176
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
177
            extra.setdefault('HTTP_X_AUTH_TOKEN', 'token')
178
            response = self.client.head(url, data, follow, **extra)
179
        return response
180

    
181
    def get(self, url, user='user', data={}, follow=False, **extra):
182
        with astakos_user(user):
183
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
184
            extra.setdefault('HTTP_X_AUTH_TOKEN', 'token')
185
            response = self.client.get(url, data, follow, **extra)
186
        return response
187

    
188
    def delete(self, url, user='user', data={}, follow=False, **extra):
189
        with astakos_user(user):
190
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
191
            extra.setdefault('HTTP_X_AUTH_TOKEN', 'token')
192
            response = self.client.delete(url, data, follow, **extra)
193
        return response
194

    
195
    def post(self, url, user='user', data={},
196
             content_type='application/octet-stream', follow=False, **extra):
197
        with astakos_user(user):
198
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
199
            extra.setdefault('HTTP_X_AUTH_TOKEN', 'token')
200
            response = self.client.post(url, data, content_type, follow,
201
                                        **extra)
202
        return response
203

    
204
    def put(self, url, user='user', data={},
205
            content_type='application/octet-stream', follow=False, **extra):
206
        with astakos_user(user):
207
            extra = dict((quote(k), quote(v)) for k, v in extra.items())
208
            extra.setdefault('HTTP_X_AUTH_TOKEN', 'token')
209
            response = self.client.put(url, data, content_type, follow,
210
                                       **extra)
211
        return response
212

    
213
    def update_account_meta(self, meta, user=None):
214
        user = user or self.user
215
        kwargs = dict(
216
            ('HTTP_X_ACCOUNT_META_%s' % k, str(v)) for k, v in meta.items())
217
        url = join_urls(self.pithos_path, user)
218
        r = self.post('%s?update=' % url, user=user, **kwargs)
219
        self.assertEqual(r.status_code, 202)
220
        account_meta = self.get_account_meta()
221
        (self.assertTrue(k in account_meta) for k in meta.keys())
222
        (self.assertEqual(account_meta[k], v) for k, v in meta.items())
223

    
224
    def delete_account_meta(self, meta, user=None):
225
        user = user or self.user
226
        transform = (lambda k: 'HTTP_X_ACCOUNT_META_%s' %
227
                     k.replace('-', '_').upper())
228
        kwargs = dict((transform(k), '') for k, v in meta.items())
229
        url = join_urls(self.pithos_path, user)
230
        r = self.post('%s?update=' % url, user=user, **kwargs)
231
        self.assertEqual(r.status_code, 202)
232
        account_meta = self.get_account_meta()
233
        (self.assertTrue(k not in account_meta) for k in meta.keys())
234
        return r
235

    
236
    def delete_account_groups(self, groups, user=None):
237
        user = user or self.user
238
        url = join_urls(self.pithos_path, user)
239
        transform = (lambda k: 'HTTP_X_ACCOUNT_GROUP_%s' %
240
                     k.replace('-', '_').upper())
241
        kwargs = dict((transform(k), '') for k, v in groups.items())
242
        r = self.post('%s?update=' % url, user=user, **kwargs)
243
        self.assertEqual(r.status_code, 202)
244
        account_groups = self.get_account_groups()
245
        (self.assertTrue(k not in account_groups) for k in groups.keys())
246
        return r
247

    
248
    def get_account_info(self, until=None, user=None):
249
        user = user or self.user
250
        url = join_urls(self.pithos_path, user)
251
        if until is not None:
252
            parts = list(urlsplit(url))
253
            parts[3] = urlencode({
254
                'until': until
255
            })
256
            url = urlunsplit(parts)
257
        r = self.head(url, user=user)
258
        self.assertEqual(r.status_code, 204)
259
        return r
260

    
261
    def get_account_meta(self, until=None, user=None):
262
        prefix = 'X-Account-Meta-'
263
        r = self.get_account_info(until=until, user=user)
264
        headers = dict(r._headers.values())
265
        return filter_headers(headers, prefix)
266

    
267
    def get_account_groups(self, until=None, user=None):
268
        prefix = 'X-Account-Group-'
269
        r = self.get_account_info(until=until, user=user)
270
        headers = dict(r._headers.values())
271
        return filter_headers(headers, prefix)
272

    
273
    def get_container_info(self, container, until=None, user=None):
274
        user = user or self.user
275
        url = join_urls(self.pithos_path, user, container)
276
        if until is not None:
277
            parts = list(urlsplit(url))
278
            parts[3] = urlencode({
279
                'until': until
280
            })
281
            url = urlunsplit(parts)
282
        r = self.head(url, user=user)
283
        self.assertEqual(r.status_code, 204)
284
        return r
285

    
286
    def get_container_meta(self, container, until=None, user=None):
287
        prefix = 'X-Container-Meta-'
288
        r = self.get_container_info(container, until=until, user=user)
289
        headers = dict(r._headers.values())
290
        return filter_headers(headers, prefix)
291

    
292
    def update_container_meta(self, container, meta, user=None):
293
        user = user or self.user
294
        kwargs = dict(
295
            ('HTTP_X_CONTAINER_META_%s' % k, str(v)) for k, v in meta.items())
296
        url = join_urls(self.pithos_path, user, container)
297
        r = self.post('%s?update=' % url, user=user, **kwargs)
298
        self.assertEqual(r.status_code, 202)
299
        container_meta = self.get_container_meta(container, user=user)
300
        (self.assertTrue('X-Container-Meta-%s' % k in container_meta) for
301
            k in meta.keys())
302
        (self.assertEqual(container_meta['X-Container-Meta-%s' % k], v) for
303
            k, v in meta.items())
304

    
305
    def list_containers(self, format='json', headers={}, user=None, **params):
306
        user = user or self.user
307
        _url = join_urls(self.pithos_path, user)
308
        parts = list(urlsplit(_url))
309
        params['format'] = format
310
        parts[3] = urlencode(params)
311
        url = urlunsplit(parts)
312
        _headers = dict(('HTTP_%s' % k.upper(), str(v))
313
                        for k, v in headers.items())
314
        r = self.get(url, user=user, **_headers)
315

    
316
        if format is None:
317
            containers = r.content.split('\n')
318
            if '' in containers:
319
                containers.remove('')
320
            return containers
321
        elif format == 'json':
322
            try:
323
                containers = json.loads(r.content)
324
            except:
325
                self.fail('json format expected')
326
            return containers
327
        elif format == 'xml':
328
            return minidom.parseString(r.content)
329

    
330
    def delete_container_content(self, cname, user=None):
331
        user = user or self.user
332
        url = join_urls(self.pithos_path, user, cname)
333
        r = self.delete('%s?delimiter=/' % url, user=user)
334
        self.assertEqual(r.status_code, 204)
335
        return r
336

    
337
    def delete_container(self, cname, user=None):
338
        user = user or self.user
339
        url = join_urls(self.pithos_path, user, cname)
340
        r = self.delete(url, user=user)
341
        self.assertEqual(r.status_code, 204)
342
        return r
343

    
344
    def create_container(self, cname=None, user=None):
345
        cname = cname or get_random_name()
346
        user = user or self.user
347
        url = join_urls(self.pithos_path, user, cname)
348
        r = self.put(url, user=user, data='')
349
        self.assertTrue(r.status_code in (202, 201))
350
        return cname, r
351

    
352
    def upload_object(self, cname, oname=None, length=None, verify=True,
353
                      user=None, **meta):
354
        oname = oname or get_random_name()
355
        length = length or random.randint(TEST_BLOCK_SIZE, 2 * TEST_BLOCK_SIZE)
356
        user = user or self.user
357
        data = get_random_data(length=length)
358
        headers = dict(('HTTP_X_OBJECT_META_%s' % k.upper(), v)
359
                       for k, v in meta.iteritems())
360
        url = join_urls(self.pithos_path, user, cname, oname)
361
        r = self.put(url, user=user, data=data, **headers)
362
        if verify:
363
            self.assertEqual(r.status_code, 201)
364
        return oname, data, r
365

    
366
    def update_object_data(self, cname, oname=None, length=None,
367
                           content_type=None, content_range=None,
368
                           verify=True, user=None, **meta):
369
        oname = oname or get_random_name()
370
        length = length or random.randint(TEST_BLOCK_SIZE, 2 * TEST_BLOCK_SIZE)
371
        content_type = content_type or 'application/octet-stream'
372
        user = user or self.user
373
        data = get_random_data(length=length)
374
        headers = dict(('HTTP_X_OBJECT_META_%s' % k.upper(), v)
375
                       for k, v in meta.iteritems())
376
        if content_range:
377
            headers['HTTP_CONTENT_RANGE'] = content_range
378
        url = join_urls(self.pithos_path, user, cname, oname)
379
        r = self.post(url, user=user, data=data, content_type=content_type,
380
                      **headers)
381
        if verify:
382
            self.assertEqual(r.status_code, 204)
383
        return oname, data, r
384

    
385
    def append_object_data(self, cname, oname=None, length=None,
386
                           content_type=None, user=None):
387
        return self.update_object_data(cname, oname=oname,
388
                                       length=length,
389
                                       content_type=content_type,
390
                                       content_range='bytes */*',
391
                                       user=user)
392

    
393
    def create_folder(self, cname, oname=None, user=None, **headers):
394
        user = user or self.user
395
        oname = oname or get_random_name()
396
        url = join_urls(self.pithos_path, user, cname, oname)
397
        r = self.put(url, user=user, data='',
398
                     content_type='application/directory', **headers)
399
        self.assertEqual(r.status_code, 201)
400
        return oname, r
401

    
402
    def list_objects(self, cname, prefix=None, user=None):
403
        user = user or self.user
404
        url = join_urls(self.pithos_path, user, cname)
405
        path = '%s?format=json' % url
406
        if prefix is not None:
407
            path = '%s&prefix=%s' % (path, prefix)
408
        r = self.get(path, user=user)
409
        self.assertTrue(r.status_code in (200, 204))
410
        try:
411
            objects = json.loads(r.content)
412
        except:
413
            self.fail('json format expected')
414
        return objects
415

    
416
    def get_object_info(self, container, object, version=None, until=None,
417
                        user=None):
418
        user = user or self.user
419
        url = join_urls(self.pithos_path, user, container, object)
420
        if until is not None:
421
            parts = list(urlsplit(url))
422
            parts[3] = urlencode({
423
                'until': until
424
            })
425
            url = urlunsplit(parts)
426
        if version:
427
            url = '%s?version=%s' % (url, version)
428
        r = self.head(url, user=user)
429
        self.assertEqual(r.status_code, 200)
430
        return r
431

    
432
    def get_object_meta(self, container, object, version=None, until=None,
433
                        user=None):
434
        prefix = 'X-Object-Meta-'
435
        r = self.get_object_info(container, object, version, until=until,
436
                                 user=user)
437
        headers = dict(r._headers.values())
438
        return filter_headers(headers, prefix)
439

    
440
    def update_object_meta(self, container, object, meta, user=None):
441
        user = user or self.user
442
        kwargs = dict(
443
            ('HTTP_X_OBJECT_META_%s' % k, str(v)) for k, v in meta.items())
444
        url = join_urls(self.pithos_path, user, container, object)
445
        r = self.post('%s?update=' % url, user=user, content_type='', **kwargs)
446
        self.assertEqual(r.status_code, 202)
447
        object_meta = self.get_object_meta(container, object, user=user)
448
        (self.assertTrue('X-Objecr-Meta-%s' % k in object_meta) for
449
            k in meta.keys())
450
        (self.assertEqual(object_meta['X-Object-Meta-%s' % k], v) for
451
            k, v in meta.items())
452

    
453
    def assert_extended(self, data, format, type, size=10000):
454
        if format == 'xml':
455
            self._assert_xml(data, type, size)
456
        elif format == 'json':
457
            self._assert_json(data, type, size)
458

    
459
    def _assert_json(self, data, type, size):
460
        convert = lambda s: s.lower()
461
        info = [convert(elem) for elem in details[type]]
462
        self.assertTrue(len(data) <= size)
463
        for item in info:
464
            for i in data:
465
                if 'subdir' in i.keys():
466
                    continue
467
                self.assertTrue(item in i.keys())
468

    
469
    def _assert_xml(self, data, type, size):
470
        convert = lambda s: s.lower()
471
        info = [convert(elem) for elem in details[type]]
472
        try:
473
            info.remove('content_encoding')
474
        except ValueError:
475
            pass
476
        xml = data
477
        entities = xml.getElementsByTagName(type)
478
        self.assertTrue(len(entities) <= size)
479
        for e in entities:
480
            for item in info:
481
                self.assertTrue(e.getElementsByTagName(item))
482

    
483

    
484
class AssertMappingInvariant(object):
485
    def __init__(self, callable, *args, **kwargs):
486
        self.callable = callable
487
        self.args = args
488
        self.kwargs = kwargs
489

    
490
    def __enter__(self):
491
        self.map = self.callable(*self.args, **self.kwargs)
492
        return self.map
493

    
494
    def __exit__(self, type, value, tb):
495
        map = self.callable(*self.args, **self.kwargs)
496
        for k, v in self.map.items():
497
            if is_date(v):
498
                continue
499

    
500
            assert(k in map), '%s not in map' % k
501
            assert v == map[k]
502

    
503

    
504
class AssertUUidInvariant(object):
505
    def __init__(self, callable, *args, **kwargs):
506
        self.callable = callable
507
        self.args = args
508
        self.kwargs = kwargs
509

    
510
    def __enter__(self):
511
        self.map = self.callable(*self.args, **self.kwargs)
512
        assert('x-object-uuid' in self.map)
513
        self.uuid = self.map['x-object-uuid']
514
        return self.map
515

    
516
    def __exit__(self, type, value, tb):
517
        map = self.callable(*self.args, **self.kwargs)
518
        assert('x-object-uuid' in self.map)
519
        uuid = map['x-object-uuid']
520
        assert(uuid == self.uuid)