Statistics
| Branch: | Tag: | Revision:

root / contrib / snf-pithos-tools / pithos / tools / test.py @ 0a92ff85

History | View | Annotate | Download (143.3 kB)

1
#!/usr/bin/env python
2
#coding=utf8
3

    
4
# Copyright 2011-2012 GRNET S.A. All rights reserved.
5
#
6
# Redistribution and use in source and binary forms, with or
7
# without modification, are permitted provided that the following
8
# conditions are met:
9
#
10
#   1. Redistributions of source code must retain the above
11
#      copyright notice, this list of conditions and the following
12
#      disclaimer.
13
#
14
#   2. Redistributions in binary form must reproduce the above
15
#      copyright notice, this list of conditions and the following
16
#      disclaimer in the documentation and/or other materials
17
#      provided with the distribution.
18
#
19
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
20
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
23
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
26
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
29
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30
# POSSIBILITY OF SUCH DAMAGE.
31
#
32
# The views and conclusions contained in the software and
33
# documentation are those of the authors and should not be
34
# interpreted as representing official policies, either expressed
35
# or implied, of GRNET S.A.
36

    
37
from pithos.tools.lib.client import Pithos_Client, Fault
38
from pithos.tools.lib.util import get_user, get_auth, get_url
39

    
40
from xml.dom import minidom
41
from StringIO import StringIO
42
from hashlib import new as newhasher
43
from binascii import hexlify
44
from httplib import HTTPConnection, HTTPSConnection
45
from urlparse import urlparse
46

    
47
import json
48
import unittest
49
import time as _time
50
import types
51
import hashlib
52
import mimetypes
53
import random
54
import datetime
55
import string
56
import re
57

    
58
DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
59
                "%A, %d-%b-%y %H:%M:%S GMT",
60
                "%a, %d %b %Y %H:%M:%S GMT"]
61

    
62
from pithos.api.settings import AUTHENTICATION_USERS
63
AUTHENTICATION_USERS = AUTHENTICATION_USERS or {}
64
OTHER_ACCOUNTS = AUTHENTICATION_USERS.copy()
65
try:
66
    OTHER_ACCOUNTS.pop(get_auth())
67
except:
68
    pass
69

    
70
class BaseTestCase(unittest.TestCase):
71
    #TODO unauthorized request
72
    def setUp(self):
73
        self.client = Pithos_Client(get_url(), get_auth(), get_user())
74

    
75
        #keep track of initial containers
76
        self.initial_containers = self.client.list_containers()
77
        if self.initial_containers == '':
78
            self.initial_containers = []
79

    
80
        self._clean_account()
81
        self.invalid_client = Pithos_Client(get_url(), get_auth(), 'invalid')
82

    
83
        #keep track of initial account groups
84
        self.initial_groups = self.client.retrieve_account_groups()
85

    
86
        #keep track of initial account meta
87
        self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
88

    
89
        self.extended = {
90
            'container':(
91
                'name',
92
                'count',
93
                'bytes',
94
                'last_modified',
95
                'x_container_policy'),
96
            'object':(
97
                'name',
98
                'hash',
99
                'bytes',
100
                'content_type',
101
                'content_encoding',
102
                'last_modified',)}
103
        self.return_codes = (400, 401, 403, 404, 503,)
104

    
105
    def tearDown(self):
106
        #delete additionally created meta
107
        l = []
108
        for m in self.client.retrieve_account_metadata(restricted=True):
109
            if m not in self.initial_meta:
110
                l.append(m)
111
        self.client.delete_account_metadata(l)
112

    
113
        #delete additionally created groups
114
        l = []
115
        for g in self.client.retrieve_account_groups():
116
            if g not in self.initial_groups:
117
                l.append(g)
118
        self.client.unset_account_groups(l)
119
        self._clean_account()
120

    
121
    def _clean_account(self):
122
        for c in self.client.list_containers():
123
#             if c not in self.initial_containers:
124
                self.client.delete_container(c, delimiter='/')
125
                self.client.delete_container(c)
126

    
127
    def assert_status(self, status, codes):
128
        l = [elem for elem in self.return_codes]
129
        if type(codes) == types.ListType:
130
            l.extend(codes)
131
        else:
132
            l.append(codes)
133
        self.assertTrue(status in l)
134

    
135
    def assert_extended(self, data, format, type, size=10000):
136
        if format == 'xml':
137
            self._assert_xml(data, type, size)
138
        elif format == 'json':
139
            self._assert_json(data, type, size)
140

    
141
    def _assert_json(self, data, type, size):
142
        convert = lambda s: s.lower()
143
        info = [convert(elem) for elem in self.extended[type]]
144
        self.assertTrue(len(data) <= size)
145
        for item in info:
146
            for i in data:
147
                if 'subdir' in i.keys():
148
                    continue
149
                self.assertTrue(item in i.keys())
150

    
151
    def _assert_xml(self, data, type, size):
152
        convert = lambda s: s.lower()
153
        info = [convert(elem) for elem in self.extended[type]]
154
        try:
155
            info.remove('content_encoding')
156
        except ValueError:
157
            pass
158
        xml = data
159
        entities = xml.getElementsByTagName(type)
160
        self.assertTrue(len(entities) <= size)
161
        for e in entities:
162
            for item in info:
163
                self.assertTrue(e.getElementsByTagName(item))
164

    
165
    def assert_raises_fault(self, status, callableObj, *args, **kwargs):
166
        """
167
        asserts that a Fault with a specific status is raised
168
        when callableObj is called with the specific arguments
169
        """
170
        try:
171
            r = callableObj(*args, **kwargs)
172
            self.fail('Should never reach here')
173
        except Fault, f:
174
            if type(status) == types.ListType:
175
                self.failUnless(f.status in status)
176
            else:
177
                self.failUnless(f.status == status)
178

    
179
    def assert_not_raises_fault(self, status, callableObj, *args, **kwargs):
180
        """
181
        asserts that a Fault with a specific status is not raised
182
        when callableObj is called with the specific arguments
183
        """
184
        try:
185
            r = callableObj(*args, **kwargs)
186
        except Fault, f:
187
            self.failIfEqual(f.status, status)
188

    
189
    def assert_container_exists(self, container):
190
        """
191
        asserts the existence of a container
192
        """
193
        try:
194
            self.client.retrieve_container_metadata(container)
195
        except Fault, f:
196
            self.failIf(f.status == 404)
197

    
198
    def assert_container_not_exists(self, container):
199
        """
200
        asserts there is no such a container
201
        """
202
        self.assert_raises_fault(404, self.client.retrieve_container_metadata,
203
                                 container)
204

    
205
    def assert_object_exists(self, container, object):
206
        """
207
        asserts the existence of an object
208
        """
209
        try:
210
            self.client.retrieve_object_metadata(container, object)
211
        except Fault, f:
212
            self.failIf(f.status == 404)
213

    
214
    def assert_object_not_exists(self, container, object):
215
        """
216
        asserts there is no such an object
217
        """
218
        self.assert_raises_fault(404, self.client.retrieve_object_metadata,
219
                                 container, object)
220

    
221
    def assert_versionlist_structure(self, versionlist):
222
        self.assertTrue(type(versionlist) == types.ListType)
223
        for elem in versionlist:
224
            self.assertTrue(type(elem) == types.ListType)
225
            self.assertEqual(len(elem), 2)
226

    
227
    def upload_random_data(self, container, name, length=1024, type=None,
228
                           enc=None, **meta):
229
        data = get_random_data(length)
230
        return self.upload_data(container, name, data, type, enc, **meta)
231

    
232
    def upload_data(self, container, name, data, type=None, enc=None, etag=None,
233
                    **meta):
234
        obj = {}
235
        obj['name'] = name
236
        try:
237
            obj['data'] = data
238
            obj['hash'] = compute_md5_hash(obj['data'])
239

    
240
            args = {}
241
            args['etag'] = etag if etag else obj['hash']
242

    
243
            try:
244
                guess = mimetypes.guess_type(name)
245
                type = type if type else guess[0]
246
                enc = enc if enc else guess[1]
247
            except:
248
                pass
249
            args['content_type'] = type if type else 'plain/text'
250
            args['content_encoding'] = enc if enc else None
251

    
252
            obj['meta'] = args
253

    
254
            path = '/%s/%s' % (container, name)
255
            self.client.create_object(container, name, f=StringIO(obj['data']),
256
                                      meta=meta, **args)
257

    
258
            return obj
259
        except IOError:
260
            return
261

    
262
class AccountHead(BaseTestCase):
263
    def setUp(self):
264
        BaseTestCase.setUp(self)
265
        self.containers = list(set(self.initial_containers + ['apples', 'bananas', 'kiwis', 'oranges', 'pears']))
266
        self.containers.sort()
267

    
268
        for item in self.containers:
269
            self.client.create_container(item)
270

    
271
        meta = {'foo':'bar'}
272
        self.client.update_account_metadata(**meta)
273
        #self.updated_meta = self.initial_meta.update(meta)
274

    
275
    def test_get_account_meta(self):
276
        meta = self.client.retrieve_account_metadata()
277

    
278
        containers = self.client.list_containers()
279
        l = str(len(containers))
280
        self.assertEqual(meta['x-account-container-count'], l)
281
        size1 = 0
282
        size2 = 0
283
        for c in containers:
284
            m = self.client.retrieve_container_metadata(c)
285
            csum = sum([o['bytes'] for o in self.client.list_objects(c, format='json')])
286
            self.assertEqual(int(m['x-container-bytes-used']), csum)
287
            size1 += int(m['x-container-bytes-used'])
288
            size2 += csum
289
        self.assertEqual(meta['x-account-bytes-used'], str(size1))
290
        self.assertEqual(meta['x-account-bytes-used'], str(size2))
291

    
292
    def test_get_account_403(self):
293
        self.assert_raises_fault(403,
294
                                 self.invalid_client.retrieve_account_metadata)
295

    
296
    def test_get_account_meta_until(self):
297
        t = datetime.datetime.utcnow()
298
        past = t - datetime.timedelta(minutes=15)
299
        past = int(_time.mktime(past.timetuple()))
300

    
301
        meta = {'premium':True}
302
        self.client.update_account_metadata(**meta)
303
        meta = self.client.retrieve_account_metadata(restricted=True,
304
                                                     until=past)
305
        self.assertTrue('premium' not in meta)
306

    
307
        meta = self.client.retrieve_account_metadata(restricted=True)
308
        self.assertTrue('premium' in meta)
309

    
310
    def test_get_account_meta_until_invalid_date(self):
311
        meta = {'premium':True}
312
        self.client.update_account_metadata(**meta)
313
        meta = self.client.retrieve_account_metadata(restricted=True,
314
                                                     until='kshfksfh')
315
        self.assertTrue('premium' in meta)
316

    
317
class AccountGet(BaseTestCase):
318
    def setUp(self):
319
        BaseTestCase.setUp(self)
320
        #create some containers
321
        self.containers = list(set(self.initial_containers + ['apples', 'bananas', 'kiwis', 'oranges', 'pears']))
322
        self.containers.sort()
323

    
324
        for item in self.containers:
325
            self.client.create_container(item)
326

    
327
    def test_list(self):
328
        #list containers
329
        containers = self.client.list_containers()
330
        self.assertEquals(self.containers, containers)
331

    
332
    def test_list_403(self):
333
        self.assert_raises_fault(403, self.invalid_client.list_containers)
334

    
335
    def test_list_with_limit(self):
336
        limit = 2
337
        containers = self.client.list_containers(limit=limit)
338
        self.assertEquals(len(containers), limit)
339
        self.assertEquals(self.containers[:2], containers)
340

    
341
    def test_list_with_marker(self):
342
        l = 2
343
        m = 'bananas'
344
        containers = self.client.list_containers(limit=l, marker=m)
345
        i = self.containers.index(m) + 1
346
        self.assertEquals(self.containers[i:(i+l)], containers)
347

    
348
        m = 'oranges'
349
        containers = self.client.list_containers(limit=l, marker=m)
350
        i = self.containers.index(m) + 1
351
        self.assertEquals(self.containers[i:(i+l)], containers)
352

    
353
    def test_list_json_with_marker(self):
354
        l = 2
355
        m = 'bananas'
356
        containers = self.client.list_containers(limit=l, marker=m, format='json')
357
        self.assert_extended(containers, 'json', 'container', l)
358
        self.assertEqual(containers[0]['name'], 'kiwis')
359
        self.assertEqual(containers[1]['name'], 'oranges')
360

    
361
    def test_list_xml_with_marker(self):
362
        l = 2
363
        m = 'oranges'
364
        xml = self.client.list_containers(limit=l, marker=m, format='xml')
365
        self.assert_extended(xml, 'xml', 'container', l)
366
        nodes = xml.getElementsByTagName('name')
367
        self.assertTrue(len(nodes) <= l)
368
        names = [n.childNodes[0].data for n in nodes]
369
        self.assertTrue('pears' in names or 'pears' > name for name in names)
370

    
371
    def test_if_modified_since(self):
372
        t = datetime.datetime.utcnow()
373
        t2 = t - datetime.timedelta(minutes=10)
374

    
375
        #add a new container
376
        self.client.create_container('dummy')
377

    
378
        for f in DATE_FORMATS:
379
            past = t2.strftime(f)
380
            try:
381
                c = self.client.list_containers(if_modified_since=past)
382
                self.assertEqual(len(c), len(self.containers) + 1)
383
            except Fault, f:
384
                self.failIf(f.status == 304) #fail if not modified
385

    
386
    def test_if_modified_since_invalid_date(self):
387
        c = self.client.list_containers(if_modified_since='')
388
        self.assertEqual(len(c), len(self.containers))
389

    
390
    def test_if_not_modified_since(self):
391
        now = datetime.datetime.utcnow()
392
        since = now + datetime.timedelta(1)
393

    
394
        for f in DATE_FORMATS:
395
            args = {'if_modified_since':'%s' %since.strftime(f)}
396

    
397
            #assert not modified
398
            self.assert_raises_fault(304, self.client.list_containers, **args)
399

    
400
    def test_if_unmodified_since(self):
401
        now = datetime.datetime.utcnow()
402
        since = now + datetime.timedelta(1)
403

    
404
        for f in DATE_FORMATS:
405
            c = self.client.list_containers(if_unmodified_since=since.strftime(f))
406

    
407
            #assert success
408
            self.assertEqual(self.containers, c)
409

    
410
    def test_if_unmodified_since_precondition_failed(self):
411
        t = datetime.datetime.utcnow()
412
        t2 = t - datetime.timedelta(minutes=10)
413

    
414
        #add a new container
415
        self.client.create_container('dummy')
416

    
417
        for f in DATE_FORMATS:
418
            past = t2.strftime(f)
419

    
420
            args = {'if_unmodified_since':'%s' %past}
421

    
422
            #assert precondition failed
423
            self.assert_raises_fault(412, self.client.list_containers, **args)
424

    
425
class AccountPost(BaseTestCase):
426
    def setUp(self):
427
        BaseTestCase.setUp(self)
428
        self.containers = list(set(self.initial_containers + ['apples', 'bananas', 'kiwis', 'oranges', 'pears']))
429
        self.containers.sort()
430

    
431
        for item in self.containers:
432
            self.client.create_container(item)
433

    
434
        meta = {'foo':'bar'}
435
        self.client.update_account_metadata(**meta)
436
        self.updated_meta = self.initial_meta.update(meta)
437

    
438
    def test_update_meta(self):
439
        with AssertMappingInvariant(self.client.retrieve_account_groups):
440
            meta = {'test':'test', 'tost':'tost'}
441
            self.client.update_account_metadata(**meta)
442

    
443
            meta.update(self.initial_meta)
444
            self.assertEqual(meta,
445
                             self.client.retrieve_account_metadata(
446
                                restricted=True))
447

    
448
    def test_invalid_account_update_meta(self):
449
        meta = {'test':'test', 'tost':'tost'}
450
        self.assert_raises_fault(403,
451
                                 self.invalid_client.update_account_metadata,
452
                                 **meta)
453

    
454
    def test_reset_meta(self):
455
        with AssertMappingInvariant(self.client.retrieve_account_groups):
456
            meta = {'test':'test', 'tost':'tost'}
457
            self.client.update_account_metadata(**meta)
458

    
459
            meta = {'test':'test33'}
460
            self.client.reset_account_metadata(**meta)
461

    
462
            self.assertEqual(meta, self.client.retrieve_account_metadata(restricted=True))
463

    
464
    def test_delete_meta(self):
465
        with AssertMappingInvariant(self.client.retrieve_account_groups):
466
            meta = {'test':'test', 'tost':'tost'}
467
            self.client.update_account_metadata(**meta)
468

    
469
            self.client.delete_account_metadata(meta.keys())
470

    
471
            account_meta = self.client.retrieve_account_metadata(restricted=True)
472
            for m in meta:
473
                self.assertTrue(m not in account_meta.keys())
474

    
475
    def test_set_account_groups(self):
476
        with AssertMappingInvariant(self.client.retrieve_account_metadata):
477
            groups = {'pithosdev':'verigak,gtsouk,chazapis'}
478
            self.client.set_account_groups(**groups)
479

    
480
            self.assertEqual(set(groups['pithosdev']),
481
                             set(self.client.retrieve_account_groups()['pithosdev']))
482

    
483
            more_groups = {'clientsdev':'pkanavos,mvasilak'}
484
            self.client.set_account_groups(**more_groups)
485

    
486
            groups.update(more_groups)
487
            self.assertEqual(set(groups['clientsdev']),
488
                             set(self.client.retrieve_account_groups()['clientsdev']))
489

    
490
    def test_reset_account_groups(self):
491
        with AssertMappingInvariant(self.client.retrieve_account_metadata):
492
            groups = {'pithosdev':'verigak,gtsouk,chazapis',
493
                      'clientsdev':'pkanavos,mvasilak'}
494
            self.client.set_account_groups(**groups)
495

    
496
            self.assertEqual(set(groups['pithosdev'].split(',')),
497
                             set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
498
            self.assertEqual(set(groups['clientsdev'].split(',')),
499
                             set(self.client.retrieve_account_groups()['clientsdev'].split(',')))
500

    
501
            groups = {'pithosdev':'verigak,gtsouk,chazapis,papagian'}
502
            self.client.reset_account_groups(**groups)
503

    
504
            self.assertEqual(set(groups['pithosdev'].split(',')),
505
                             set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
506

    
507
    def test_delete_account_groups(self):
508
        with AssertMappingInvariant(self.client.retrieve_account_metadata):
509
            groups = {'pithosdev':'verigak,gtsouk,chazapis',
510
                      'clientsdev':'pkanavos,mvasilak'}
511
            self.client.set_account_groups(**groups)
512

    
513
            self.client.unset_account_groups(groups.keys())
514

    
515
            self.assertEqual({}, self.client.retrieve_account_groups())
516

    
517
class ContainerHead(BaseTestCase):
518
    def setUp(self):
519
        BaseTestCase.setUp(self)
520
        self.container = 'apples'
521
        self.client.create_container(self.container)
522

    
523
    def test_get_meta(self):
524
        meta = {'trash':'true'}
525
        t1 = datetime.datetime.utcnow()
526
        o = self.upload_random_data(self.container, o_names[0], **meta)
527
        if o:
528
            headers = self.client.retrieve_container_metadata(self.container)
529
            self.assertEqual(headers['x-container-object-count'], '1')
530
            self.assertEqual(headers['x-container-bytes-used'], str(len(o['data'])))
531
            t2 = datetime.datetime.strptime(headers['last-modified'], DATE_FORMATS[2])
532
            delta = (t2 - t1)
533
            threashold = datetime.timedelta(seconds=1)
534
            self.assertTrue(delta < threashold)
535
            self.assertTrue(headers['x-container-object-meta'])
536
            self.assertTrue('Trash' in headers['x-container-object-meta'])
537

    
538
class ContainerGet(BaseTestCase):
539
    def setUp(self):
540
        BaseTestCase.setUp(self)
541
        self.container = ['pears', 'apples']
542
        for c in self.container:
543
            self.client.create_container(c)
544
        self.obj = []
545
        for o in o_names[:8]:
546
            self.obj.append(self.upload_random_data(self.container[0], o))
547
        for o in o_names[8:]:
548
            self.obj.append(self.upload_random_data(self.container[1], o))
549

    
550
    def test_list_shared(self):
551
        self.client.share_object(self.container[0], self.obj[0]['name'], ('*',))
552
        objs = self.client.list_objects(self.container[0], shared=True)
553
        self.assertEqual(objs, [self.obj[0]['name']])
554

    
555
        # create child object
556
        self.upload_random_data(self.container[0], strnextling(self.obj[0]['name']))
557
        objs = self.client.list_objects(self.container[0], shared=True)
558
        self.assertEqual(objs, [self.obj[0]['name']])
559

    
560
        # test inheritance
561
        self.client.create_folder(self.container[1], 'folder')
562
        self.client.share_object(self.container[1], 'folder', ('*',))
563
        self.upload_random_data(self.container[1], 'folder/object')
564
        objs = self.client.list_objects(self.container[1], shared=True)
565
        self.assertEqual(objs, ['folder', 'folder/object'])
566

    
567
    def test_list_public(self):
568
        self.client.publish_object(self.container[0], self.obj[0]['name'])
569
        objs = self.client.list_objects(self.container[0], public=True)
570
        self.assertEqual(objs, [self.obj[0]['name']])
571

    
572
        # create child object
573
        self.upload_random_data(self.container[0], strnextling(self.obj[0]['name']))
574
        objs = self.client.list_objects(self.container[0], public=True)
575
        self.assertEqual(objs, [self.obj[0]['name']])
576

    
577
        # test inheritance
578
        self.client.create_folder(self.container[1], 'folder')
579
        self.client.publish_object(self.container[1], 'folder')
580
        self.upload_random_data(self.container[1], 'folder/object')
581
        objs = self.client.list_objects(self.container[1], public=True)
582
        self.assertEqual(objs, ['folder'])
583

    
584
    def test_list_shared_public(self):
585
        self.client.share_object(self.container[0], self.obj[0]['name'], ('*',))
586
        self.client.publish_object(self.container[0], self.obj[1]['name'])
587
        objs = self.client.list_objects(self.container[0], shared=True, public=True)
588
        self.assertEqual(objs, [self.obj[0]['name'], self.obj[1]['name']])
589

    
590
        # create child object
591
        self.upload_random_data(self.container[0], strnextling(self.obj[0]['name']))
592
        self.upload_random_data(self.container[0], strnextling(self.obj[1]['name']))
593
        objs = self.client.list_objects(self.container[0], shared=True, public=True)
594
        self.assertEqual(objs, [self.obj[0]['name'], self.obj[1]['name']])
595

    
596
        # test inheritance
597
        self.client.create_folder(self.container[1], 'folder1')
598
        self.client.share_object(self.container[1], 'folder1', ('*',))
599
        self.upload_random_data(self.container[1], 'folder1/object')
600
        self.client.create_folder(self.container[1], 'folder2')
601
        self.client.publish_object(self.container[1], 'folder2')
602
        o = self.upload_random_data(self.container[1], 'folder2/object')
603
        objs = self.client.list_objects(self.container[1], shared=True, public=True)
604
        self.assertEqual(objs, ['folder1', 'folder1/object', 'folder2'])
605

    
606
    def test_list_objects(self):
607
        objects = self.client.list_objects(self.container[0])
608
        l = [elem['name'] for elem in self.obj[:8]]
609
        l.sort()
610
        self.assertEqual(objects, l)
611

    
612
    def test_list_objects_containing_slash(self):
613
        self.client.create_container('test')
614
        self.upload_random_data('test', '/objectname')
615

    
616
        objects = self.client.list_objects('test')
617
        self.assertEqual(objects, ['/objectname'])
618

    
619
        objects = self.client.list_objects('test', format='json')
620
        self.assertEqual(objects[0]['name'], '/objectname')
621

    
622
        objects = self.client.list_objects('test', format='xml')
623
        self.assert_extended(objects, 'xml', 'object')
624
        node_name = objects.getElementsByTagName('name')[0]
625
        self.assertEqual(node_name.firstChild.data, '/objectname')
626

    
627
    def test_list_objects_with_limit_marker(self):
628
        objects = self.client.list_objects(self.container[0], limit=2)
629
        l = [elem['name'] for elem in self.obj[:8]]
630
        l.sort()
631
        self.assertEqual(objects, l[:2])
632

    
633
        markers = ['How To Win Friends And Influence People.pdf',
634
                   'moms_birthday.jpg']
635
        limit = 4
636
        for m in markers:
637
            objects = self.client.list_objects(self.container[0], limit=limit,
638
                                               marker=m)
639
            l = [elem['name'] for elem in self.obj[:8]]
640
            l.sort()
641
            start = l.index(m) + 1
642
            end = start + limit
643
            end = end if len(l) >= end else len(l)
644
            self.assertEqual(objects, l[start:end])
645

    
646
    #takes too long
647
    def _test_list_limit_exceeds(self):
648
        self.client.create_container('pithos')
649

    
650
        for i in range(10001):
651
            self.client.create_zero_length_object('pithos', i)
652

    
653
        self.assertEqual(10000, len(self.client.list_objects('pithos')))
654

    
655
    def test_list_empty_params(self):
656
        objects = self.client.get('/%s/%s' % (get_user(), self.container[0]))[2]
657
        if objects:
658
            objects = objects.strip().split('\n')
659
        self.assertEqual(objects,
660
                         self.client.list_objects(self.container[0]))
661

    
662
    def test_list_pseudo_hierarchical_folders(self):
663
        objects = self.client.list_objects(self.container[1], prefix='photos',
664
                                           delimiter='/')
665
        self.assertEquals(['photos/animals/', 'photos/me.jpg',
666
                           'photos/plants/'], objects)
667

    
668
        objects = self.client.list_objects(self.container[1],
669
                                           prefix='photos/animals',
670
                                           delimiter='/')
671
        l = ['photos/animals/cats/', 'photos/animals/dogs/']
672
        self.assertEquals(l, objects)
673

    
674
        objects = self.client.list_objects(self.container[1], path='photos')
675
        self.assertEquals(['photos/me.jpg'], objects)
676

    
677
    def test_extended_list_json(self):
678
        objects = self.client.list_objects(self.container[1], format='json',
679
                                           limit=2, prefix='photos/animals',
680
                                           delimiter='/')
681
        self.assertEqual(objects[0]['subdir'], 'photos/animals/cats/')
682
        self.assertEqual(objects[1]['subdir'], 'photos/animals/dogs/')
683

    
684
    def test_extended_list_xml(self):
685
        xml = self.client.list_objects(self.container[1], format='xml', limit=4,
686
                                       prefix='photos', delimiter='/')
687
        self.assert_extended(xml, 'xml', 'object', size=4)
688
        dirs = xml.getElementsByTagName('subdir')
689
        self.assertEqual(len(dirs), 2)
690
        self.assertEqual(dirs[0].attributes['name'].value, 'photos/animals/')
691
        self.assertEqual(dirs[1].attributes['name'].value, 'photos/plants/')
692

    
693
        objects = xml.getElementsByTagName('name')
694
        self.assertEqual(len(objects), 1)
695
        self.assertEqual(objects[0].childNodes[0].data, 'photos/me.jpg')
696

    
697
    def test_list_meta_double_matching(self):
698
        meta = {'quality':'aaa', 'stock':'true'}
699
        self.client.update_object_metadata(self.container[0],
700
                                           self.obj[0]['name'], **meta)
701
        obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
702
        self.assertEqual(len(obj), 1)
703
        self.assertTrue(obj, self.obj[0]['name'])
704

    
705
    def test_list_using_meta(self):
706
        meta = {'quality':'aaa'}
707
        for o in self.obj[:2]:
708
            self.client.update_object_metadata(self.container[0], o['name'],
709
                                               **meta)
710
        meta = {'stock':'true'}
711
        for o in self.obj[3:5]:
712
            self.client.update_object_metadata(self.container[0], o['name'],
713
                                               **meta)
714

    
715
        obj = self.client.list_objects(self.container[0], meta='Quality')
716
        self.assertEqual(len(obj), 2)
717
        self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
718

    
719
        # test case insensitive
720
        obj = self.client.list_objects(self.container[0], meta='quality')
721
        self.assertEqual(len(obj), 2)
722
        self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
723

    
724
        # test multiple matches
725
        obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
726
        self.assertEqual(len(obj), 4)
727
        self.assertTrue(obj, [o['name'] for o in self.obj[:4]])
728

    
729
        # test non 1-1 multiple match
730
        obj = self.client.list_objects(self.container[0], meta='Quality,aaaa')
731
        self.assertEqual(len(obj), 2)
732
        self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
733

    
734
    def test_if_modified_since(self):
735
        t = datetime.datetime.utcnow()
736
        t2 = t - datetime.timedelta(minutes=10)
737

    
738
        #add a new object
739
        self.upload_random_data(self.container[0], o_names[0])
740

    
741
        for f in DATE_FORMATS:
742
            past = t2.strftime(f)
743
            try:
744
                o = self.client.list_objects(self.container[0],
745
                                            if_modified_since=past)
746
                self.assertEqual(o,
747
                                 self.client.list_objects(self.container[0]))
748
            except Fault, f:
749
                self.failIf(f.status == 304) #fail if not modified
750

    
751
    def test_if_modified_since_invalid_date(self):
752
        headers = {'if-modified-since':''}
753
        o = self.client.list_objects(self.container[0], if_modified_since='')
754
        self.assertEqual(o, self.client.list_objects(self.container[0]))
755

    
756
    def test_if_not_modified_since(self):
757
        now = datetime.datetime.utcnow()
758
        since = now + datetime.timedelta(1)
759

    
760
        for f in DATE_FORMATS:
761
            args = {'if_modified_since':'%s' %since.strftime(f)}
762

    
763
            #assert not modified
764
            self.assert_raises_fault(304, self.client.list_objects,
765
                                     self.container[0], **args)
766

    
767
    def test_if_unmodified_since(self):
768
        now = datetime.datetime.utcnow()
769
        since = now + datetime.timedelta(1)
770

    
771
        for f in DATE_FORMATS:
772
            obj = self.client.list_objects(self.container[0],
773
                                           if_unmodified_since=since.strftime(f))
774

    
775
            #assert unmodified
776
            self.assertEqual(obj, self.client.list_objects(self.container[0]))
777

    
778
    def test_if_unmodified_since_precondition_failed(self):
779
        t = datetime.datetime.utcnow()
780
        t2 = t - datetime.timedelta(minutes=10)
781

    
782
        #add a new container
783
        self.client.create_container('dummy')
784

    
785
        for f in DATE_FORMATS:
786
            past = t2.strftime(f)
787

    
788
            args = {'if_unmodified_since':'%s' %past}
789

    
790
            #assert precondition failed
791
            self.assert_raises_fault(412, self.client.list_objects,
792
                                     self.container[0], **args)
793

    
794
class ContainerPut(BaseTestCase):
795
    def setUp(self):
796
        BaseTestCase.setUp(self)
797
        self.containers = list(set(self.initial_containers + ['c1', 'c2']))
798
        self.containers.sort()
799

    
800
    def test_create(self):
801
        self.client.create_container(self.containers[0])
802
        containers = self.client.list_containers()
803
        self.assertTrue(self.containers[0] in containers)
804
        self.assert_container_exists(self.containers[0])
805

    
806
    def test_create_twice(self):
807
        self.client.create_container(self.containers[0])
808
        self.assertTrue(not self.client.create_container(self.containers[0]))
809

    
810
    def test_quota(self):
811
        self.client.create_container(self.containers[0])
812

    
813
        policy = {'quota':100}
814
        self.client.set_container_policies(self.containers[0], **policy)
815

    
816
        meta = self.client.retrieve_container_metadata(self.containers[0])
817
        self.assertTrue('x-container-policy-quota' in meta)
818
        self.assertEqual(meta['x-container-policy-quota'], '100')
819

    
820
        args = [self.containers[0], 'o1']
821
        kwargs = {'length':101}
822
        self.assert_raises_fault(413, self.upload_random_data, *args, **kwargs)
823

    
824
        #reset quota
825
        policy = {'quota':0}
826
        self.client.set_container_policies(self.containers[0], **policy)
827

    
828
class ContainerPost(BaseTestCase):
829
    def setUp(self):
830
        BaseTestCase.setUp(self)
831
        self.container = 'apples'
832
        self.client.create_container(self.container)
833

    
834
    def test_update_meta(self):
835
        meta = {'test':'test33',
836
                'tost':'tost22'}
837
        self.client.update_container_metadata(self.container, **meta)
838
        headers = self.client.retrieve_container_metadata(self.container)
839
        for k,v in meta.items():
840
            k = 'x-container-meta-%s' % k
841
            self.assertTrue(headers[k])
842
            self.assertEqual(headers[k], v)
843

    
844
class ContainerDelete(BaseTestCase):
845
    def setUp(self):
846
        BaseTestCase.setUp(self)
847
        self.containers = list(set(self.initial_containers + ['c1', 'c2']))
848
        self.containers.sort()
849

    
850
        for c in self.containers:
851
            self.client.create_container(c)
852

    
853
    def test_delete(self):
854
        status = self.client.delete_container(self.containers[0])[0]
855
        self.assertEqual(status, 204)
856

    
857
    def test_delete_non_empty(self):
858
        self.upload_random_data(self.containers[1], o_names[0])
859
        self.assert_raises_fault(409, self.client.delete_container,
860
                                 self.containers[1])
861

    
862
    def test_delete_invalid(self):
863
        self.assert_raises_fault(404, self.client.delete_container, 'c3')
864

    
865
    def test_delete_contents(self):
866
        self.client.create_folder(self.containers[0], 'folder-1')
867
        self.upload_random_data(self.containers[1], 'folder-1/%s' % o_names[0])
868
        self.client.create_folder(self.containers[0], 'folder-1/subfolder')
869
        self.client.create_folder(self.containers[0], 'folder-2/%s' % o_names[1])
870

    
871
        objects = self.client.list_objects(self.containers[0])
872
        self.client.delete_container(self.containers[0], delimiter='/')
873
        for o in objects:
874
            self.assert_object_not_exists(self.containers[0], o)
875
        self.assert_container_exists(self.containers[0])
876

    
877
class ObjectGet(BaseTestCase):
878
    def setUp(self):
879
        BaseTestCase.setUp(self)
880
        self.containers = list(set(self.initial_containers + ['c1', 'c2']))
881
        self.containers.sort()
882

    
883
        #create some containers
884
        for c in self.containers:
885
            self.client.create_container(c)
886

    
887
        #upload a file
888
        names = ('obj1', 'obj2')
889
        self.objects = []
890
        for n in names:
891
            self.objects.append(self.upload_random_data(self.containers[1], n))
892

    
893
    def test_versions(self):
894
        c = self.containers[1]
895
        o = self.objects[0]
896
        b = self.client.retrieve_object_versionlist(c, o['name'])['versions']
897
        self.assert_versionlist_structure(b)
898

    
899
        #update meta
900
        meta = {'quality':'AAA', 'stock':True}
901
        self.client.update_object_metadata(c, o['name'], **meta)
902

    
903
        a = self.client.retrieve_object_versionlist(c, o['name'])['versions']
904
        self.assert_versionlist_structure(a)
905
        self.assertEqual(len(b)+1, len(a))
906
        self.assertEqual(b, a[:-1])
907

    
908
        #get exact previous version metadata
909
        v = a[-2][0]
910
        v_meta = self.client.retrieve_object_metadata(c, o['name'],
911
                                                      restricted=True,
912
                                                      version=v)
913
        (self.assertTrue(k not in v_meta) for k in meta.keys())
914

    
915
        #update obejct
916
        data = get_random_data()
917
        self.client.update_object(c, o['name'], StringIO(data))
918

    
919
        aa = self.client.retrieve_object_versionlist(c, o['name'])['versions']
920
        self.assert_versionlist_structure(aa)
921
        self.assertEqual(len(a)+1, len(aa))
922
        self.assertEqual(a, aa[:-1])
923

    
924
        #get exact previous version
925
        v = aa[-3][0]
926
        v_data = self.client.retrieve_object_version(c, o['name'], version=v)
927
        self.assertEqual(o['data'], v_data)
928
        self.assertEqual(self.client.retrieve_object(c, o['name']),
929
                         '%s%s' %(v_data, data))
930

    
931
    def test_get(self):
932
        #perform get
933
        o = self.client.retrieve_object(self.containers[1],
934
                                        self.objects[0]['name'],
935
                                        self.objects[0]['meta'])
936
        self.assertEqual(o, self.objects[0]['data'])
937

    
938
    def test_objects_with_trailing_spaces(self):
939
        self.client.create_container('test')
940
        #create 'a' object
941
        self.upload_random_data('test', 'a')
942
        #look for 'a ' object
943
        self.assert_raises_fault(404, self.client.retrieve_object,
944
                                 'test', 'a ')
945

    
946
        #delete 'a' object
947
        self.client.delete_object('test', 'a')
948
        self.assert_raises_fault(404, self.client.retrieve_object,
949
                                 'test', 'a')
950

    
951
        #create 'a ' object
952
        self.upload_random_data('test', 'a ')
953
        #look for 'a' object
954
        self.assert_raises_fault(404, self.client.retrieve_object,
955
                                 'test', 'a')
956

    
957
    def test_get_invalid(self):
958
        self.assert_raises_fault(404, self.client.retrieve_object,
959
                                 self.containers[0], self.objects[0]['name'])
960

    
961
    def test_get_partial(self):
962
        #perform get with range
963
        status, headers, data = self.client.request_object(self.containers[1],
964
                                                            self.objects[0]['name'],
965
                                                            range='bytes=0-499')
966

    
967
        #assert successful partial content
968
        self.assertEqual(status, 206)
969

    
970
        #assert content-type
971
        self.assertEqual(headers['content-type'],
972
                         self.objects[0]['meta']['content_type'])
973

    
974
        #assert content length
975
        self.assertEqual(int(headers['content-length']), 500)
976

    
977
        #assert content
978
        self.assertEqual(self.objects[0]['data'][:500], data)
979

    
980
    def test_get_final_500(self):
981
        #perform get with range
982
        headers = {'range':'bytes=-500'}
983
        status, headers, data = self.client.request_object(self.containers[1],
984
                                                            self.objects[0]['name'],
985
                                                            range='bytes=-500')
986

    
987
        #assert successful partial content
988
        self.assertEqual(status, 206)
989

    
990
        #assert content-type
991
        self.assertEqual(headers['content-type'],
992
                         self.objects[0]['meta']['content_type'])
993

    
994
        #assert content length
995
        self.assertEqual(int(headers['content-length']), 500)
996

    
997
        #assert content
998
        self.assertTrue(self.objects[0]['data'][-500:], data)
999

    
1000
    def test_get_rest(self):
1001
        #perform get with range
1002
        offset = len(self.objects[0]['data']) - 500
1003
        status, headers, data = self.client.request_object(self.containers[1],
1004
                                                self.objects[0]['name'],
1005
                                                range='bytes=%s-' %offset)
1006

    
1007
        #assert successful partial content
1008
        self.assertEqual(status, 206)
1009

    
1010
        #assert content-type
1011
        self.assertEqual(headers['content-type'],
1012
                         self.objects[0]['meta']['content_type'])
1013

    
1014
        #assert content length
1015
        self.assertEqual(int(headers['content-length']), 500)
1016

    
1017
        #assert content
1018
        self.assertTrue(self.objects[0]['data'][-500:], data)
1019

    
1020
    def test_get_range_not_satisfiable(self):
1021
        #perform get with range
1022
        offset = len(self.objects[0]['data']) + 1
1023

    
1024
        #assert range not satisfiable
1025
        self.assert_raises_fault(416, self.client.retrieve_object,
1026
                                 self.containers[1], self.objects[0]['name'],
1027
                                 range='bytes=0-%s' %offset)
1028

    
1029
    def test_multiple_range(self):
1030
        #perform get with multiple range
1031
        ranges = ['0-499', '-500', '1000-']
1032
        bytes = 'bytes=%s' % ','.join(ranges)
1033
        status, headers, data = self.client.request_object(self.containers[1],
1034
                                                           self.objects[0]['name'],
1035
                                                           range=bytes)
1036

    
1037
        # assert partial content
1038
        self.assertEqual(status, 206)
1039

    
1040
        # assert Content-Type of the reply will be multipart/byteranges
1041
        self.assertTrue(headers['content-type'])
1042
        content_type_parts = headers['content-type'].split()
1043
        self.assertEqual(content_type_parts[0], ('multipart/byteranges;'))
1044

    
1045
        boundary = '--%s' %content_type_parts[1].split('=')[-1:][0]
1046
        cparts = data.split(boundary)[1:-1]
1047

    
1048
        # assert content parts are exactly 2
1049
        self.assertEqual(len(cparts), len(ranges))
1050

    
1051
        # for each content part assert headers
1052
        i = 0
1053
        for cpart in cparts:
1054
            content = cpart.split('\r\n')
1055
            headers = content[1:3]
1056
            content_range = headers[0].split(': ')
1057
            self.assertEqual(content_range[0], 'Content-Range')
1058

    
1059
            r = ranges[i].split('-')
1060
            if not r[0] and not r[1]:
1061
                pass
1062
            elif not r[0]:
1063
                start = len(self.objects[0]['data']) - int(r[1])
1064
                end = len(self.objects[0]['data'])
1065
            elif not r[1]:
1066
                start = int(r[0])
1067
                end = len(self.objects[0]['data'])
1068
            else:
1069
                start = int(r[0])
1070
                end = int(r[1]) + 1
1071
            fdata = self.objects[0]['data'][start:end]
1072
            sdata = '\r\n'.join(content[4:-1])
1073
            self.assertEqual(len(fdata), len(sdata))
1074
            self.assertEquals(fdata, sdata)
1075
            i+=1
1076

    
1077
    def test_multiple_range_not_satisfiable(self):
1078
        #perform get with multiple range
1079
        out_of_range = len(self.objects[0]['data']) + 1
1080
        ranges = ['0-499', '-500', '%d-' %out_of_range]
1081
        bytes = 'bytes=%s' % ','.join(ranges)
1082

    
1083
        # assert partial content
1084
        self.assert_raises_fault(416, self.client.retrieve_object,
1085
                                 self.containers[1],
1086
                                 self.objects[0]['name'], range=bytes)
1087

    
1088
    def test_get_with_if_match(self):
1089
        #perform get with If-Match
1090
        etag = self.objects[0]['hash']
1091
        status, headers, data = self.client.request_object(self.containers[1],
1092
                                                           self.objects[0]['name'],
1093
                                                           if_match=etag)
1094
        #assert get success
1095
        self.assertEqual(status, 200)
1096

    
1097
        #assert content-type
1098
        self.assertEqual(headers['content-type'],
1099
                         self.objects[0]['meta']['content_type'])
1100

    
1101
        #assert response content
1102
        self.assertEqual(self.objects[0]['data'], data)
1103

    
1104
    def test_get_with_if_match_star(self):
1105
        #perform get with If-Match *
1106
        headers = {'if-match':'*'}
1107
        status, headers, data = self.client.request_object(self.containers[1],
1108
                                                self.objects[0]['name'],
1109
                                                **headers)
1110
        #assert get success
1111
        self.assertEqual(status, 200)
1112

    
1113
        #assert content-type
1114
        self.assertEqual(headers['content-type'],
1115
                         self.objects[0]['meta']['content_type'])
1116

    
1117
        #assert response content
1118
        self.assertEqual(self.objects[0]['data'], data)
1119

    
1120
    def test_get_with_multiple_if_match(self):
1121
        #perform get with If-Match
1122
        etags = [i['hash'] for i in self.objects if i]
1123
        etags = ','.join('"%s"' % etag for etag in etags)
1124
        status, headers, data = self.client.request_object(self.containers[1],
1125
                                                           self.objects[0]['name'],
1126
                                                           if_match=etags)
1127
        #assert get success
1128
        self.assertEqual(status, 200)
1129

    
1130
        #assert content-type
1131
        self.assertEqual(headers['content-type'],
1132
                         self.objects[0]['meta']['content_type'])
1133

    
1134
        #assert content-type
1135
        self.assertEqual(headers['content-type'],
1136
                         self.objects[0]['meta']['content_type'])
1137

    
1138
        #assert response content
1139
        self.assertEqual(self.objects[0]['data'], data)
1140

    
1141
    def test_if_match_precondition_failed(self):
1142
        #assert precondition failed
1143
        self.assert_raises_fault(412, self.client.retrieve_object,
1144
                                 self.containers[1],
1145
                                 self.objects[0]['name'], if_match='123')
1146

    
1147
    def test_if_none_match(self):
1148
        #perform get with If-None-Match
1149
        status, headers, data = self.client.request_object(self.containers[1],
1150
                                                           self.objects[0]['name'],
1151
                                                           if_none_match='123')
1152

    
1153
        #assert get success
1154
        self.assertEqual(status, 200)
1155

    
1156
        #assert content-type
1157
        self.assertEqual(headers['content_type'],
1158
                         self.objects[0]['meta']['content_type'])
1159

    
1160
    def test_if_none_match(self):
1161
        #perform get with If-None-Match * and assert not modified
1162
        self.assert_raises_fault(304, self.client.retrieve_object,
1163
                                 self.containers[1],
1164
                                 self.objects[0]['name'],
1165
                                 if_none_match='*')
1166

    
1167
    def test_if_none_match_not_modified(self):
1168
        #perform get with If-None-Match and assert not modified
1169
        self.assert_raises_fault(304, self.client.retrieve_object,
1170
                                 self.containers[1],
1171
                                 self.objects[0]['name'],
1172
                                 if_none_match=self.objects[0]['hash'])
1173

    
1174
        meta = self.client.retrieve_object_metadata(self.containers[1],
1175
                                                    self.objects[0]['name'])
1176
        self.assertEqual(meta['etag'], self.objects[0]['hash'])
1177

    
1178
    def test_if_modified_since(self):
1179
        t = datetime.datetime.utcnow()
1180
        t2 = t - datetime.timedelta(minutes=10)
1181

    
1182
        #modify the object
1183
        self.upload_data(self.containers[1],
1184
                           self.objects[0]['name'],
1185
                           self.objects[0]['data'][:200])
1186

    
1187
        for f in DATE_FORMATS:
1188
            past = t2.strftime(f)
1189

    
1190
            headers = {'if-modified-since':'%s' %past}
1191
            try:
1192
                o = self.client.retrieve_object(self.containers[1],
1193
                                                self.objects[0]['name'],
1194
                                                if_modified_since=past)
1195
                self.assertEqual(o,
1196
                                 self.client.retrieve_object(self.containers[1],
1197
                                                             self.objects[0]['name']))
1198
            except Fault, f:
1199
                self.failIf(f.status == 304)
1200

    
1201
    def test_if_modified_since_invalid_date(self):
1202
        o = self.client.retrieve_object(self.containers[1],
1203
                                        self.objects[0]['name'],
1204
                                        if_modified_since='')
1205
        self.assertEqual(o, self.client.retrieve_object(self.containers[1],
1206
                                                        self.objects[0]['name']))
1207

    
1208
    def test_if_not_modified_since(self):
1209
        now = datetime.datetime.utcnow()
1210
        since = now + datetime.timedelta(1)
1211

    
1212
        for f in DATE_FORMATS:
1213
            #assert not modified
1214
            self.assert_raises_fault(304, self.client.retrieve_object,
1215
                                     self.containers[1], self.objects[0]['name'],
1216
                                     if_modified_since=since.strftime(f))
1217

    
1218
    def test_if_unmodified_since(self):
1219
        now = datetime.datetime.utcnow()
1220
        since = now + datetime.timedelta(1)
1221

    
1222
        for f in DATE_FORMATS:
1223
            t = since.strftime(f)
1224
            status, headers, data = self.client.request_object(self.containers[1],
1225
                                                               self.objects[0]['name'],
1226
                                                               if_unmodified_since=t)
1227
            #assert success
1228
            self.assertEqual(status, 200)
1229
            self.assertEqual(self.objects[0]['data'], data)
1230

    
1231
            #assert content-type
1232
            self.assertEqual(headers['content-type'],
1233
                             self.objects[0]['meta']['content_type'])
1234

    
1235
    def test_if_unmodified_since_precondition_failed(self):
1236
        t = datetime.datetime.utcnow()
1237
        t2 = t - datetime.timedelta(minutes=10)
1238

    
1239
        #modify the object
1240
        self.upload_data(self.containers[1],
1241
                           self.objects[0]['name'],
1242
                           self.objects[0]['data'][:200])
1243

    
1244
        for f in DATE_FORMATS:
1245
            past = t2.strftime(f)
1246
            #assert precondition failed
1247
            self.assert_raises_fault(412, self.client.retrieve_object,
1248
                                     self.containers[1], self.objects[0]['name'],
1249
                                     if_unmodified_since=past)
1250

    
1251
    def test_hashes(self):
1252
        l = 8388609
1253
        fname = 'largefile'
1254
        o = self.upload_random_data(self.containers[1], fname, l)
1255
        if o:
1256
            body = self.client.retrieve_object(self.containers[1], fname,
1257
                                               format='json')
1258
            hashes = body['hashes']
1259
            block_size = body['block_size']
1260
            block_hash = body['block_hash']
1261
            block_num = l/block_size if l/block_size == 0 else l/block_size + 1
1262
            self.assertTrue(len(hashes), block_num)
1263
            i = 0
1264
            for h in hashes:
1265
                start = i * block_size
1266
                end = (i + 1) * block_size
1267
                hash = compute_block_hash(o['data'][start:end], block_hash)
1268
                self.assertEqual(h, hash)
1269
                i += 1
1270

    
1271
class ObjectPut(BaseTestCase):
1272
    def setUp(self):
1273
        BaseTestCase.setUp(self)
1274
        self.container = 'c1'
1275
        self.client.create_container(self.container)
1276

    
1277
    def test_upload(self):
1278
        name = o_names[0]
1279
        meta = {'test':'test1'}
1280
        o = self.upload_random_data(self.container, name, **meta)
1281

    
1282
        headers = self.client.retrieve_object_metadata(self.container,
1283
                                                       name,
1284
                                                       restricted=True)
1285
        self.assertTrue('test' in headers.keys())
1286
        self.assertEqual(headers['test'], meta['test'])
1287

    
1288
        #assert uploaded content
1289
        status, h, data = self.client.request_object(self.container, name)
1290
        self.assertEqual(len(o['data']), int(h['content-length']))
1291
        self.assertEqual(o['data'], data)
1292

    
1293
        #assert content-type
1294
        self.assertEqual(h['content-type'], o['meta']['content_type'])
1295

    
1296
    def _test_maximum_upload_size_exceeds(self):
1297
        name = o_names[0]
1298
        meta = {'test':'test1'}
1299
        #upload 5GB
1300
        length= 5 * (1024 * 1024 * 1024) + 1
1301
        self.assert_raises_fault(400, self.upload_random_data, self.container,
1302
                                 name, length, **meta)
1303

    
1304
    def test_upload_with_name_containing_slash(self):
1305
        name = '/%s' % o_names[0]
1306
        meta = {'test':'test1'}
1307
        o = self.upload_random_data(self.container, name, **meta)
1308

    
1309
        self.assertEqual(o['data'],
1310
                         self.client.retrieve_object(self.container, name))
1311

    
1312
        self.assertTrue(name in self.client.list_objects(self.container))
1313

    
1314
    def test_create_directory_marker(self):
1315
        self.client.create_directory_marker(self.container, 'foo')
1316
        meta = self.client.retrieve_object_metadata(self.container, 'foo')
1317
        self.assertEqual(meta['content-length'], '0')
1318
        self.assertEqual(meta['content-type'], 'application/directory')
1319

    
1320
    def test_upload_unprocessable_entity(self):
1321
        meta={'etag':'123', 'test':'test1'}
1322

    
1323
        #assert unprocessable entity
1324
        self.assert_raises_fault(422, self.upload_random_data, self.container,
1325
                                 o_names[0], **meta)
1326

    
1327
    def test_chunked_transfer(self):
1328
        data = get_random_data()
1329
        objname = 'object'
1330
        self.client.create_object_using_chunks(self.container, objname,
1331
                                               StringIO(data))
1332

    
1333
        uploaded_data = self.client.retrieve_object(self.container, objname)
1334
        self.assertEqual(data, uploaded_data)
1335

    
1336
    def test_manifestation(self):
1337
        prefix = 'myobject/'
1338
        data = ''
1339
        for i in range(5):
1340
            part = '%s%d' %(prefix, i)
1341
            o = self.upload_random_data(self.container, part)
1342
            data += o['data']
1343

    
1344
        manifest = '%s/%s' %(self.container, prefix)
1345
        self.client.create_manifestation(self.container, 'large-object', manifest)
1346

    
1347
        self.assert_object_exists(self.container, 'large-object')
1348
        self.assertEqual(data, self.client.retrieve_object(self.container,
1349
                                                           'large-object'))
1350

    
1351
        r = self.client.retrieve_object_hashmap(self.container,'large-object')
1352
        hashes = r['hashes']
1353
        block_size = int(r['block_size'])
1354
        block_hash = r['block_hash']
1355
        l = len(data)
1356
        block_num = l/block_size if l/block_size != 0 else l/block_size + 1
1357
        self.assertEqual(block_num, len(hashes))
1358

    
1359
        #wrong manifestation
1360
        self.client.create_manifestation(self.container, 'large-object',
1361
                                         '%s/invalid' % self.container)
1362
        self.assertEqual('', self.client.retrieve_object(self.container,
1363
                                                         'large-object'))
1364

    
1365
    def test_create_zero_length_object(self):
1366
        c = self.container
1367
        o = 'object'
1368
        zero = self.client.create_zero_length_object(c, o)
1369
        zero_meta = self.client.retrieve_object_metadata(c, o)
1370
        zero_hash = self.client.retrieve_object_hashmap(c, o)["hashes"]
1371
        zero_data = self.client.retrieve_object(c, o)
1372

    
1373
        self.assertEqual(int(zero_meta['content-length']), 0)
1374
        hasher = newhasher('sha256')
1375
        hasher.update("")
1376
        emptyhash = hasher.digest()
1377
        self.assertEqual(zero_hash, [hexlify(emptyhash)])
1378
        self.assertEqual(zero_data, '')
1379

    
1380
    def test_create_object_by_hashmap(self):
1381
        c = self.container
1382
        o = 'object'
1383
        self.upload_random_data(c, o)
1384
        hashmap = self.client.retrieve_object(c, o, format='json')
1385
        o2 = 'object-copy'
1386
        self.client.create_object_by_hashmap(c, o2, hashmap)
1387
        self.assertEqual(self.client.retrieve_object(c, o),
1388
                         self.client.retrieve_object(c, o))
1389

    
1390
class ObjectCopy(BaseTestCase):
1391
    def setUp(self):
1392
        BaseTestCase.setUp(self)
1393
        self.containers = list(set(self.initial_containers + ['c1', 'c2']))
1394
        self.containers.sort()
1395

    
1396
        for c in self.containers:
1397
            self.client.create_container(c)
1398
        self.obj = self.upload_random_data(self.containers[0], o_names[0])
1399

    
1400
    def test_copy(self):
1401
        with AssertMappingInvariant(self.client.retrieve_object_metadata,
1402
                             self.containers[0], self.obj['name']):
1403
            #perform copy
1404
            meta = {'test':'testcopy'}
1405
            status = self.client.copy_object(self.containers[0],
1406
                                              self.obj['name'],
1407
                                              self.containers[0],
1408
                                              'testcopy',
1409
                                              meta)[0]
1410

    
1411
            #assert copy success
1412
            self.assertEqual(status, 201)
1413

    
1414
            #assert access the new object
1415
            headers = self.client.retrieve_object_metadata(self.containers[0],
1416
                                                           'testcopy')
1417
            self.assertTrue('x-object-meta-test' in headers.keys())
1418
            self.assertTrue(headers['x-object-meta-test'], 'testcopy')
1419

    
1420
            #assert etag is the same
1421
            self.assertEqual(headers['etag'], self.obj['hash'])
1422

    
1423
            #assert src object still exists
1424
            self.assert_object_exists(self.containers[0], self.obj['name'])
1425

    
1426
    def test_copy_from_different_container(self):
1427
        with AssertMappingInvariant(self.client.retrieve_object_metadata,
1428
                             self.containers[0], self.obj['name']):
1429
            meta = {'test':'testcopy'}
1430
            status = self.client.copy_object(self.containers[0],
1431
                                             self.obj['name'],
1432
                                             self.containers[1],
1433
                                             'testcopy',
1434
                                             meta)[0]
1435
            self.assertEqual(status, 201)
1436

    
1437
            # assert updated metadata
1438
            meta = self.client.retrieve_object_metadata(self.containers[1],
1439
                                                           'testcopy',
1440
                                                           restricted=True)
1441
            self.assertTrue('test' in meta.keys())
1442
            self.assertTrue(meta['test'], 'testcopy')
1443

    
1444
            #assert src object still exists
1445
            self.assert_object_exists(self.containers[0], self.obj['name'])
1446

    
1447
    def test_copy_invalid(self):
1448
        #copy from invalid object
1449
        meta = {'test':'testcopy'}
1450
        self.assert_raises_fault(404, self.client.copy_object, self.containers[0],
1451
                                 'test.py', self.containers[1], 'testcopy', meta)
1452

    
1453
        #copy from invalid container
1454
        meta = {'test':'testcopy'}
1455
        self.assert_raises_fault(404, self.client.copy_object, self.containers[1],
1456
                                 self.obj['name'], self.containers[1],
1457
                                 'testcopy', meta)
1458

    
1459
    def test_copy_dir(self):
1460
        self.client.create_folder(self.containers[0], 'dir')
1461
        self.client.create_folder(self.containers[0], 'dir/subdir')
1462
        self.upload_random_data(self.containers[0], 'dir/object1.jpg', length=1024)
1463
        self.upload_random_data(self.containers[0], 'dir/subdir/object2.pdf', length=2*1024)
1464
        self.client.create_folder(self.containers[0], 'dirs')
1465

    
1466
        objects = self.client.list_objects(self.containers[0], prefix='dir')
1467
        self.client.copy_object(self.containers[0], 'dir', self.containers[1], 'dir-backup', delimiter='/')
1468
        for object in objects[:-1]:
1469
            self.assert_object_exists(self.containers[0], object)
1470
            self.assert_object_exists(self.containers[1], object.replace('dir', 'dir-backup', 1))
1471
            meta0 = self.client.retrieve_object_metadata(self.containers[0], object)
1472
            meta1 = self.client.retrieve_object_metadata(self.containers[1], object.replace('dir', 'dir-backup', 1))
1473
            t = ('content-length', 'x-object-hash', 'content-type')
1474
            (self.assertEqual(meta0[elem], meta1[elem]) for elem in t)
1475
        self.assert_object_not_exists(self.containers[1], objects[-1])
1476

    
1477
class ObjectMove(BaseTestCase):
1478
    def setUp(self):
1479
        BaseTestCase.setUp(self)
1480
        self.containers = list(set(self.initial_containers + ['c1', 'c2']))
1481
        self.containers.sort()
1482

    
1483
        for c in self.containers:
1484
            self.client.create_container(c)
1485
        self.obj = self.upload_random_data(self.containers[0], o_names[0])
1486

    
1487
    def test_move(self):
1488
        meta = self.client.retrieve_object_metadata(self.containers[0],
1489
                                                    self.obj['name'])
1490
        self.assertTrue('x-object-uuid' in meta)
1491
        uuid = meta['x-object-uuid']
1492

    
1493
        #perform move
1494
        meta = {'test':'testcopy'}
1495
        src_path = '/'.join(('/', self.containers[0], self.obj['name']))
1496
        status = self.client.move_object(self.containers[0], self.obj['name'],
1497
                                         self.containers[0], 'testcopy',
1498
                                         meta)[0]
1499

    
1500
        #assert successful move
1501
        self.assertEqual(status, 201)
1502

    
1503
        #assert updated metadata
1504
        meta = self.client.retrieve_object_metadata(self.containers[0],
1505
                                                    'testcopy')
1506
        self.assertTrue('x-object-meta-test' in meta.keys())
1507
        self.assertTrue(meta['x-object-meta-test'], 'testcopy')
1508

    
1509
        #assert same uuid
1510
        self.assertTrue(meta['x-object-uuid'], uuid)
1511

    
1512
        #assert src object no more exists
1513
        self.assert_object_not_exists(self.containers[0], self.obj['name'])
1514

    
1515

    
1516
    def test_move_dir(self):
1517
        meta = {}
1518
        self.client.create_folder(self.containers[0], 'dir')
1519
        meta['dir'] = self.client.retrieve_object_metadata(self.containers[0], 'dir')
1520
        self.client.create_folder(self.containers[0], 'dir/subdir')
1521
        meta['dir/subdir'] = self.client.retrieve_object_metadata(self.containers[0], 'dir/subdir')
1522
        self.upload_random_data(self.containers[0], 'dir/object1.jpg', length=1024)
1523
        meta['dir/object1.jpg'] = self.client.retrieve_object_metadata(self.containers[0], 'dir/object1.jpg')
1524
        self.upload_random_data(self.containers[0], 'dir/subdir/object2.pdf', length=2*1024)
1525
        meta['dir/subdir/object2.pdf'] = self.client.retrieve_object_metadata(self.containers[0], 'dir/subdir/object2.pdf')
1526
        self.client.create_folder(self.containers[0], 'dirs')
1527
        meta['dirs'] = self.client.retrieve_object_metadata(self.containers[0], 'dirs')
1528

    
1529
        objects = self.client.list_objects(self.containers[0], prefix='dir')
1530
        self.client.move_object(self.containers[0], 'dir', self.containers[1], 'dir-backup', delimiter='/')
1531
        for object in objects[:-1]:
1532
            self.assert_object_not_exists(self.containers[0], object)
1533
            self.assert_object_exists(self.containers[1], object.replace('dir', 'dir-backup', 1))
1534
            meta1 = self.client.retrieve_object_metadata(self.containers[1], object.replace('dir', 'dir-backup', 1))
1535
            t = ('content-length', 'x-object-hash', 'content-type')
1536
            (self.assertEqual(meta0[elem], meta1[elem]) for elem in t)
1537
        self.assert_object_exists(self.containers[0], objects[-1])
1538
        self.assert_object_not_exists(self.containers[1], objects[-1])
1539

    
1540
class ObjectPost(BaseTestCase):
1541
    def setUp(self):
1542
        BaseTestCase.setUp(self)
1543
        self.containers = list(set(self.initial_containers + ['c1', 'c2']))
1544
        self.containers.sort()
1545

    
1546
        for c in self.containers:
1547
            self.client.create_container(c)
1548
        self.obj = []
1549
        for i in range(2):
1550
            self.obj.append(self.upload_random_data(self.containers[0], o_names[i]))
1551

    
1552
    def test_update_meta(self):
1553
        with AssertUUidInvariant(self.client.retrieve_object_metadata,
1554
                                 self.containers[0],
1555
                                 self.obj[0]['name']):
1556
            #perform update metadata
1557
            more = {'foo': 'foo', 'bar': 'bar', 'f' * 114: 'b' * 256}
1558
            status = self.client.update_object_metadata(self.containers[0],
1559
                                                        self.obj[0]['name'],
1560
                                                        **more)[0]
1561
            #assert request accepted
1562
            self.assertEqual(status, 202)
1563

    
1564
            #assert old metadata are still there
1565
            headers = self.client.retrieve_object_metadata(self.containers[0],
1566
                                                           self.obj[0]['name'],
1567
                                                           restricted=True)
1568
            #assert new metadata have been updated
1569
            for k,v in more.items():
1570
                self.assertTrue(k in headers.keys())
1571
                self.assertTrue(headers[k], v)
1572

    
1573
            #out of limits
1574
            more = {'f' * 114: 'b' * 257}
1575
            self.assert_raises_fault(400, self.client.update_object_metadata,
1576
                                                        self.containers[0],
1577
                                                        self.obj[0]['name'],
1578
                                                        **more)
1579

    
1580
            #perform update metadata
1581
            more = {'α': 'β' * 256}
1582
            status = self.client.update_object_metadata(self.containers[0],
1583
                                                        self.obj[0]['name'],
1584
                                                        **more)[0]
1585
            #assert request accepted
1586
            self.assertEqual(status, 202)
1587

    
1588
            #assert old metadata are still there
1589
            headers = self.client.retrieve_object_metadata(self.containers[0],
1590
                                                           self.obj[0]['name'],
1591
                                                           restricted=True)
1592
            #assert new metadata have been updated
1593
            for k,v in more.items():
1594
                self.assertTrue(k in headers.keys())
1595
                self.assertTrue(headers[k], v)
1596

    
1597
            #out of limits
1598
            more = {'α': 'β' * 257}
1599
            self.assert_raises_fault(400, self.client.update_object_metadata,
1600
                                                        self.containers[0],
1601
                                                        self.obj[0]['name'],
1602
                                                        **more)
1603

    
1604
    def test_update_object(self,
1605
                           first_byte_pos=0,
1606
                           last_byte_pos=499,
1607
                           instance_length = True,
1608
                           content_length = 500):
1609
        with AssertUUidInvariant(self.client.retrieve_object_metadata,
1610
                                 self.containers[0],
1611
                                 self.obj[0]['name']):
1612
            l = len(self.obj[0]['data'])
1613
            range = 'bytes %d-%d/%s' %(first_byte_pos,
1614
                                           last_byte_pos,
1615
                                            l if instance_length else '*')
1616
            partial = last_byte_pos - first_byte_pos + 1
1617
            length = first_byte_pos + partial
1618
            data = get_random_data(partial)
1619
            args = {'content_type':'application/octet-stream',
1620
                    'content_range':'%s' %range}
1621
            if content_length:
1622
                args['content_length'] = content_length
1623

    
1624
            r = self.client.update_object(self.containers[0], self.obj[0]['name'],
1625
                                      StringIO(data), **args)
1626
            status = r[0]
1627
            etag = r[1]['etag']
1628
            if partial < 0 or (instance_length and l <= last_byte_pos):
1629
                self.assertEqual(status, 202)
1630
            else:
1631
                self.assertEqual(status, 204)
1632
                #check modified object
1633
                content = self.client.retrieve_object(self.containers[0],
1634
                                                  self.obj[0]['name'])
1635
                self.assertEqual(content[:first_byte_pos], self.obj[0]['data'][:first_byte_pos])
1636
                self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1637
                self.assertEqual(content[last_byte_pos+1:], self.obj[0]['data'][last_byte_pos+1:])
1638
                self.assertEqual(etag, compute_md5_hash(content))
1639

    
1640
    def test_update_object_lt_blocksize(self):
1641
        self.test_update_object(10, 20, content_length=None)
1642

    
1643
    def test_update_object_gt_blocksize(self):
1644
        o = self.upload_random_data(self.containers[0], o_names[1],
1645
                                length=4*1024*1024+5)
1646
        c = self.containers[0]
1647
        o_name = o['name']
1648
        o_data = o['data']
1649
        first_byte_pos = 4*1024*1024+1
1650
        last_byte_pos = 4*1024*1024+4
1651
        l = last_byte_pos - first_byte_pos + 1
1652
        data = get_random_data(l)
1653
        range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1654
        self.client.update_object(c, o_name, StringIO(data), content_range=range)
1655
        content = self.client.retrieve_object(c, o_name)
1656
        self.assertEqual(content[:first_byte_pos], o_data[:first_byte_pos])
1657
        self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1658
        self.assertEqual(content[last_byte_pos+1:], o_data[last_byte_pos+1:])
1659

    
1660
    def test_update_object_divided_by_blocksize(self):
1661
        o = self.upload_random_data(self.containers[0], o_names[1],
1662
                                length=4*1024*1024+5)
1663
        c = self.containers[0]
1664
        o_name = o['name']
1665
        o_data = o['data']
1666
        first_byte_pos = 4*1024*1024
1667
        last_byte_pos = 5*1024*1024
1668
        l = last_byte_pos - first_byte_pos + 1
1669
        data = get_random_data(l)
1670
        range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1671
        self.client.update_object(c, o_name, StringIO(data), content_range=range)
1672
        content = self.client.retrieve_object(c, o_name)
1673
        self.assertEqual(content[:first_byte_pos], o_data[:first_byte_pos])
1674
        self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1675
        self.assertEqual(content[last_byte_pos+1:], o_data[last_byte_pos+1:])
1676

    
1677
    def test_update_object_no_content_length(self):
1678
        self.test_update_object(content_length = None)
1679

    
1680
    def test_update_object_invalid_content_length(self):
1681
        with AssertContentInvariant(self.client.retrieve_object,
1682
                                    self.containers[0], self.obj[0]['name']):
1683
            self.assert_raises_fault(400, self.test_update_object,
1684
                                     content_length = 1000)
1685

    
1686
    def test_update_object_invalid_range(self):
1687
        with AssertContentInvariant(self.client.retrieve_object,
1688
                                    self.containers[0], self.obj[0]['name']):
1689
            self.assert_raises_fault(416, self.test_update_object, 499, 0, True)
1690

    
1691
    def test_update_object_invalid_range_and_length(self):
1692
        with AssertContentInvariant(self.client.retrieve_object,
1693
                                    self.containers[0], self.obj[0]['name']):
1694
            self.assert_raises_fault([400, 416], self.test_update_object, 499, 0, True,
1695
                                     -1)
1696

    
1697
    def test_update_object_invalid_range_with_no_content_length(self):
1698
        with AssertContentInvariant(self.client.retrieve_object,
1699
                                    self.containers[0], self.obj[0]['name']):
1700
            self.assert_raises_fault(416, self.test_update_object, 499, 0, True,
1701
                                     content_length = None)
1702

    
1703
    def test_update_object_out_of_limits(self):
1704
        with AssertContentInvariant(self.client.retrieve_object,
1705
                                    self.containers[0], self.obj[0]['name']):
1706
            l = len(self.obj[0]['data'])
1707
            self.assert_raises_fault(416, self.test_update_object, 0, l+1, True)
1708

    
1709
    def test_append(self):
1710
        data = get_random_data(500)
1711
        headers = {}
1712
        self.client.update_object(self.containers[0], self.obj[0]['name'],
1713
                                  StringIO(data), content_length=500,
1714
                                  content_type='application/octet-stream')
1715

    
1716
        content = self.client.retrieve_object(self.containers[0],
1717
                                              self.obj[0]['name'])
1718
        self.assertEqual(len(content), len(self.obj[0]['data']) + 500)
1719
        self.assertEqual(content[:-500], self.obj[0]['data'])
1720

    
1721
    def test_update_with_chunked_transfer(self):
1722
        data = get_random_data(500)
1723
        dl = len(data)
1724
        fl = len(self.obj[0]['data'])
1725

    
1726
        self.client.update_object_using_chunks(self.containers[0],
1727
                                               self.obj[0]['name'],
1728
                                               StringIO(data),
1729
                                               offset=0,
1730
                                               content_type='application/octet-stream')
1731

    
1732
        #check modified object
1733
        content = self.client.retrieve_object(self.containers[0],
1734
                                              self.obj[0]['name'])
1735
        self.assertEqual(content[0:dl], data)
1736
        self.assertEqual(content[dl:fl], self.obj[0]['data'][dl:fl])
1737

    
1738
    def test_update_from_other_object(self):
1739
        c = self.containers[0]
1740
        src = o_names[0]
1741
        dest = 'object'
1742

    
1743
        source_data = self.client.retrieve_object(c, src)
1744
        source_meta = self.client.retrieve_object_metadata(c, src)
1745
        source_hash = self.client.retrieve_object_hashmap(c, src)["hashes"]
1746

    
1747
        #update zero length object
1748
        self.client.create_zero_length_object(c, dest)
1749
        source_object = '/%s/%s' % (c, src)
1750
        self.client.update_from_other_source(c, dest, source_object)
1751
        dest_data = self.client.retrieve_object(c, src)
1752
        dest_meta = self.client.retrieve_object_metadata(c, dest)
1753
        dest_hash = self.client.retrieve_object_hashmap(c, src)["hashes"]
1754
        self.assertEqual(source_data, dest_data)
1755
        self.assertEqual(source_hash, dest_hash)
1756

    
1757
        #test append
1758
        self.client.update_from_other_source(c, dest, source_object)
1759
        content = self.client.retrieve_object(c, dest)
1760
        self.assertEqual(source_data * 2, content)
1761

    
1762
    def test_update_range_from_other_object(self):
1763
        c = self.containers[0]
1764
        dest = 'object'
1765

    
1766
        #test update range
1767
        src = self.obj[1]['name']
1768
        src_data = self.client.retrieve_object(c, src)
1769

    
1770
        #update zero length object
1771
        prev_data = self.upload_random_data(c, dest, length=4*1024*1024+10)['data']
1772
        source_object = '/%s/%s' % (c, src)
1773
        first_byte_pos = 4*1024*1024+1
1774
        last_byte_pos = 4*1024*1024+4
1775
        range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1776
        self.client.update_from_other_source(c, dest, source_object,
1777
                                             content_range=range)
1778
        content = self.client.retrieve_object(c, dest)
1779
        self.assertEqual(content[:first_byte_pos], prev_data[:first_byte_pos])
1780
        self.assertEqual(content[first_byte_pos:last_byte_pos+1], src_data[:last_byte_pos - first_byte_pos + 1])
1781
        self.assertEqual(content[last_byte_pos+1:], prev_data[last_byte_pos+1:])
1782

    
1783
    def test_update_hashes_from_other_object(self):
1784
        c = self.containers[0]
1785
        dest = 'object'
1786

    
1787
        #test update range
1788
        src_data = self.upload_random_data(c, o_names[0], length=1024*1024+10)['data']
1789

    
1790
        #update zero length object
1791
        prev_data = self.upload_random_data(c, dest, length=5*1024*1024+10)['data']
1792
        source_object = '/%s/%s' % (c, o_names[0])
1793
        first_byte_pos = 4*1024*1024
1794
        last_byte_pos = 5*1024*1024
1795
        range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1796
        self.client.update_from_other_source(c, dest, source_object,
1797
                                             content_range=range)
1798
        content = self.client.retrieve_object(c, dest)
1799
        self.assertEqual(content[:first_byte_pos], prev_data[:first_byte_pos])
1800
        self.assertEqual(content[first_byte_pos:last_byte_pos+1], src_data[:last_byte_pos - first_byte_pos + 1])
1801
        self.assertEqual(content[last_byte_pos+1:], prev_data[last_byte_pos+1:])
1802

    
1803

    
1804
    def test_update_zero_length_object(self):
1805
        c = self.containers[0]
1806
        o = 'object'
1807
        other = 'other'
1808
        zero = self.client.create_zero_length_object(c, o)
1809

    
1810
        data = get_random_data()
1811
        self.client.update_object(c, o, StringIO(data))
1812
        self.client.create_object(c, other, StringIO(data))
1813

    
1814
        self.assertEqual(self.client.retrieve_object(c, o),
1815
                         self.client.retrieve_object(c, other))
1816

    
1817
        self.assertEqual(self.client.retrieve_object_hashmap(c, o)["hashes"],
1818
                         self.client.retrieve_object_hashmap(c, other)["hashes"])
1819

    
1820
class ObjectDelete(BaseTestCase):
1821
    def setUp(self):
1822
        BaseTestCase.setUp(self)
1823
        self.containers = ['c1', 'c2']
1824
        self.containers.extend(self.initial_containers)
1825

    
1826
        for c in self.containers:
1827
            self.client.create_container(c)
1828
        self.obj = self.upload_random_data(self.containers[0], o_names[0])
1829

    
1830
    def test_delete(self):
1831
        #perform delete object
1832
        self.client.delete_object(self.containers[0], self.obj['name'])[0]
1833

    
1834
    def test_delete_invalid(self):
1835
        #assert item not found
1836
        self.assert_raises_fault(404, self.client.delete_object, self.containers[1],
1837
                                 self.obj['name'])
1838

    
1839
    def test_delete_dir(self):
1840
        self.client.create_folder(self.containers[0], 'dir')
1841
        self.client.create_folder(self.containers[0], 'dir/subdir')
1842
        self.upload_random_data(self.containers[0], 'dir/object1.jpg', length=1024)
1843
        self.upload_random_data(self.containers[0], 'dir/subdir/object2.pdf', length=2*1024)
1844
        self.client.create_folder(self.containers[0], 'dirs')
1845

    
1846
        objects = self.client.list_objects(self.containers[0], prefix='dir')
1847
        self.client.delete_object(self.containers[0], 'dir', delimiter='/')
1848
        for object in objects[:-1]:
1849
            self.assert_object_not_exists(self.containers[0], object)
1850
        self.assert_object_exists(self.containers[0], objects[-1])
1851

    
1852
class ListSharing(BaseTestCase):
1853
    def setUp(self):
1854
        BaseTestCase.setUp(self)
1855
        for i in range(2):
1856
            self.client.create_container('c%s' %i)
1857
        self.client.create_container('c')
1858
        for i in range(2):
1859
            self.upload_random_data('c1', 'o%s' %i)
1860
        if not OTHER_ACCOUNTS:
1861
            raise Warning('No other accounts avalaible for running this test.')
1862
        for token, account in OTHER_ACCOUNTS.items():
1863
            self.o1_sharing = token, account
1864
            self.client.share_object('c1', 'o1', (account,), read=True)
1865
            break
1866

    
1867
    def test_list_other_shared(self):
1868
        self.other = Pithos_Client(get_url(),
1869
                              self.o1_sharing[0],
1870
                              self.o1_sharing[1])
1871
        self.assertTrue(get_user() in self.other.list_shared_with_me())
1872

    
1873
    def test_list_my_shared(self):
1874
        my_shared_containers = self.client.list_containers(shared=True)
1875
        self.assertTrue('c1' in my_shared_containers)
1876
        self.assertTrue('c2' not in my_shared_containers)
1877

    
1878
        my_shared_objects = self.client.list_objects('c1', shared=True)
1879
        self.assertTrue('o1' in my_shared_objects)
1880
        self.assertTrue('o2' not in my_shared_objects)
1881

    
1882
class List(BaseTestCase):
1883
    def setUp(self):
1884
        BaseTestCase.setUp(self)
1885
        for i in range(1, 5):
1886
            c = 'c%s' % i
1887
            self.client.create_container(c)
1888
            for j in range(1, 3):
1889
                o = 'o%s' % j
1890
                self.upload_random_data(c, o)
1891
            if i < 3:
1892
                self.client.share_object(c, 'o1', ['papagian'], read=True)
1893
            if i%2 != 0:
1894
                self.client.publish_object(c, 'o2')
1895

    
1896
    def test_shared_public(self):
1897
        diff = lambda l: set(l) - set(self.initial_containers)
1898

    
1899
        func, kwargs = self.client.list_containers, {'shared':True}
1900
        l = func(**kwargs)
1901
        self.assertEqual(set(['c1', 'c2']), diff(l))
1902
        self.assertEqual(l, [e['name'] for e in func(format='json', **kwargs)])
1903

    
1904
        func, kwargs = self.client.list_containers, {'public':True}
1905
        l = func(**kwargs)
1906
        self.assertEqual(set(['c1', 'c3']), diff(l))
1907
        self.assertEqual(l, [e['name'] for e in func(format='json', **kwargs)])
1908

    
1909
        func, kwargs = self.client.list_containers, {'shared':True, 'public':True}
1910
        l = func(**kwargs)
1911
        self.assertEqual(set(['c1', 'c2', 'c3']), diff(l))
1912
        self.assertEqual(l, [e['name'] for e in func(format='json', **kwargs)])
1913

    
1914
        func, args, kwargs = self.client.list_objects, ['c1'], {'shared':True}
1915
        l = func(*args, **kwargs)
1916
        self.assertEqual(l, ['o1'])
1917
        self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1918

    
1919
        func, args, kwargs = self.client.list_objects, ['c1'], {'public':True}
1920
        l = func(*args, **kwargs)
1921
        self.assertEqual(l, ['o2'])
1922
        self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1923

    
1924
        func, args, kwargs = self.client.list_objects, ['c1'], {'shared':True, 'public':True}
1925
        l = func(*args, **kwargs)
1926
        self.assertEqual(l, ['o1', 'o2'])
1927
        self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1928

    
1929
        func, args, kwargs = self.client.list_objects, ['c2'], {'shared':True}
1930
        l = func(*args, **kwargs)
1931
        self.assertEqual(l, ['o1'])
1932
        self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1933

    
1934
        func, args, kwargs = self.client.list_objects, ['c2'], {'public':True}
1935
        l = func(*args, **kwargs)
1936
        self.assertEqual(l, '')
1937
        self.assertEqual([], func(*args, format='json', **kwargs))
1938

    
1939
        func, args, kwargs = self.client.list_objects, ['c2'], {'shared':True, 'public':True}
1940
        l = func(*args, **kwargs)
1941
        self.assertEqual(l, ['o1'])
1942
        self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1943

    
1944
        func, args, kwargs = self.client.list_objects, ['c3'], {'shared':True}
1945
        l = func(*args, **kwargs)
1946
        self.assertEqual(l, '')
1947
        self.assertEqual([], func(*args, format='json', **kwargs))
1948

    
1949
        func, args, kwargs = self.client.list_objects, ['c3'], {'public':True}
1950
        l = func(*args, **kwargs)
1951
        self.assertEqual(l, ['o2'])
1952
        self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1953

    
1954
        func, args, kwargs = self.client.list_objects, ['c3'], {'shared':True, 'public':True}
1955
        l = func(*args, **kwargs)
1956
        self.assertEqual(l, ['o2'])
1957
        self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1958

    
1959
        func, args, kwargs = self.client.list_objects, ['c4'], {'shared':True}
1960
        l = func(*args, **kwargs)
1961
        self.assertEqual(l, '')
1962
        self.assertEqual([], func(*args, format='json', **kwargs))
1963

    
1964
        func, args, kwargs = self.client.list_objects, ['c4'], {'public':True}
1965
        l = func(*args, **kwargs)
1966
        self.assertEqual(l, '')
1967
        self.assertEqual([], func(*args, format='json', **kwargs))
1968

    
1969
        func, args, kwargs = self.client.list_objects, ['c4'], {'shared':True, 'public':True}
1970
        l = func(*args, **kwargs)
1971
        self.assertEqual(l, '')
1972
        self.assertEqual([], func(*args, format='json', **kwargs))
1973

    
1974
class TestUTF8(BaseTestCase):
1975
    def test_create_container(self):
1976
        self.client.create_container('φάκελος')
1977
        self.assert_container_exists('φάκελος')
1978

    
1979
        self.assertTrue('φάκελος' in self.client.list_containers())
1980

    
1981
    def test_create_object(self):
1982
        self.client.create_container('φάκελος')
1983
        self.upload_random_data('φάκελος', 'αντικείμενο')
1984

    
1985
        self.assert_object_exists('φάκελος', 'αντικείμενο')
1986
        self.assertTrue('αντικείμενο' in self.client.list_objects('φάκελος'))
1987

    
1988
    def test_copy_object(self):
1989
        src_container = 'φάκελος'
1990
        src_object = 'αντικείμενο'
1991
        dest_container = 'αντίγραφα'
1992
        dest_object = 'ασφαλές-αντίγραφο'
1993

    
1994
        self.client.create_container(src_container)
1995
        self.upload_random_data(src_container, src_object)
1996

    
1997
        self.client.create_container(dest_container)
1998
        self.client.copy_object(src_container, src_object, dest_container,
1999
                                dest_object)
2000

    
2001
        self.assert_object_exists(src_container, src_object)
2002
        self.assert_object_exists(dest_container, dest_object)
2003
        self.assertTrue(dest_object in self.client.list_objects(dest_container))
2004

    
2005
    def test_move_object(self):
2006
        src_container = 'φάκελος'
2007
        src_object = 'αντικείμενο'
2008
        dest_container = 'αντίγραφα'
2009
        dest_object = 'ασφαλές-αντίγραφο'
2010

    
2011
        self.client.create_container(src_container)
2012
        self.upload_random_data(src_container, src_object)
2013

    
2014
        self.client.create_container(dest_container)
2015
        self.client.move_object(src_container, src_object, dest_container,
2016
                                dest_object)
2017

    
2018
        self.assert_object_not_exists(src_container, src_object)
2019
        self.assert_object_exists(dest_container, dest_object)
2020
        self.assertTrue(dest_object in self.client.list_objects(dest_container))
2021

    
2022
    def test_delete_object(self):
2023
        self.client.create_container('φάκελος')
2024
        self.upload_random_data('φάκελος', 'αντικείμενο')
2025
        self.assert_object_exists('φάκελος', 'αντικείμενο')
2026

    
2027
        self.client.delete_object('φάκελος', 'αντικείμενο')
2028
        self.assert_object_not_exists('φάκελος', 'αντικείμενο')
2029
        self.assertTrue('αντικείμενο' not in self.client.list_objects('φάκελος'))
2030

    
2031
    def test_delete_container(self):
2032
        self.client.create_container('φάκελος')
2033
        self.assert_container_exists('φάκελος')
2034

    
2035
        self.client.delete_container('φάκελος')
2036
        self.assert_container_not_exists('φάκελος')
2037
        self.assertTrue('φάκελος' not in self.client.list_containers())
2038

    
2039
    def test_account_meta(self):
2040
        meta = {'ποιότητα':'ΑΑΑ'}
2041
        self.client.update_account_metadata(**meta)
2042
        meta = self.client.retrieve_account_metadata(restricted=True)
2043
        self.assertTrue('ποιότητα' in meta.keys())
2044
        self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
2045

    
2046
    def test_container_meta(self):
2047
        meta = {'ποιότητα':'ΑΑΑ'}
2048
        self.client.create_container('φάκελος', meta=meta)
2049

    
2050
        meta = self.client.retrieve_container_metadata('φάκελος', restricted=True)
2051
        self.assertTrue('ποιότητα' in meta.keys())
2052
        self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
2053

    
2054
    def test_object_meta(self):
2055
        self.client.create_container('φάκελος')
2056
        meta = {'ποιότητα':'ΑΑΑ'}
2057
        self.upload_random_data('φάκελος', 'αντικείμενο', **meta)
2058

    
2059
        meta = self.client.retrieve_object_metadata('φάκελος', 'αντικείμενο',
2060
                                                    restricted=True)
2061
        self.assertTrue('ποιότητα' in meta.keys())
2062
        self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
2063

    
2064
    def test_list_meta_filtering(self):
2065
        self.client.create_container('φάκελος')
2066
        meta = {'ποιότητα':'ΑΑΑ'}
2067
        self.upload_random_data('φάκελος', 'ο1', **meta)
2068
        self.upload_random_data('φάκελος', 'ο2')
2069
        self.upload_random_data('φάκελος', 'ο3')
2070

    
2071
        meta = {'ποσότητα':'μεγάλη'}
2072
        self.client.update_object_metadata('φάκελος', 'ο2', **meta)
2073
        objects = self.client.list_objects('φάκελος', meta='ποιότητα, ποσότητα')
2074
        self.assertEquals(objects, ['ο1', 'ο2'])
2075

    
2076
        objects = self.client.list_objects('φάκελος', meta='!ποιότητα')
2077
        self.assertEquals(objects, ['ο2', 'ο3'])
2078

    
2079
        objects = self.client.list_objects('φάκελος', meta='!ποιότητα, !ποσότητα')
2080
        self.assertEquals(objects, ['ο3'])
2081

    
2082
        meta = {'ποιότητα':'ΑΒ'}
2083
        self.client.update_object_metadata('φάκελος', 'ο2', **meta)
2084
        objects = self.client.list_objects('φάκελος', meta='ποιότητα=ΑΑΑ')
2085
        self.assertEquals(objects, ['ο1'])
2086
        objects = self.client.list_objects('φάκελος', meta='ποιότητα!=ΑΑΑ')
2087
        self.assertEquals(objects, ['ο2'])
2088

    
2089
        meta = {'έτος':'2011'}
2090
        self.client.update_object_metadata('φάκελος', 'ο3', **meta)
2091
        meta = {'έτος':'2012'}
2092
        self.client.update_object_metadata('φάκελος', 'ο2', **meta)
2093
        objects = self.client.list_objects('φάκελος', meta='έτος<2012')
2094
        self.assertEquals(objects, ['ο3'])
2095
        objects = self.client.list_objects('φάκελος', meta='έτος<=2012')
2096
        self.assertEquals(objects, ['ο2', 'ο3'])
2097
        objects = self.client.list_objects('φάκελος', meta='έτος<2012,έτος!=2011')
2098
        self.assertEquals(objects, '')
2099

    
2100
    def test_groups(self):
2101
        #create a group
2102
        groups = {'γκρουπ':'chazapis,διογένης'}
2103
        self.client.set_account_groups(**groups)
2104
        groups.update(self.initial_groups)
2105
        self.assertEqual(groups['γκρουπ'],
2106
                         self.client.retrieve_account_groups()['γκρουπ'])
2107

    
2108
        #check read access
2109
        self.client.create_container('φάκελος')
2110
        o = self.upload_random_data('φάκελος', 'ο1')
2111
        self.client.share_object('φάκελος', 'ο1', ['%s:γκρουπ' % get_user()])
2112
        if 'διογένης' not in OTHER_ACCOUNTS.values():
2113
            raise Warning('No such an account exists for running this test.')
2114
        chef = Pithos_Client(get_url(),
2115
                            '0009',
2116
                            'διογένης')
2117
        self.assert_not_raises_fault(403, chef.retrieve_object_metadata,
2118
                                     'φάκελος', 'ο1', account=get_user())
2119

    
2120
        #check write access
2121
        self.client.share_object('φάκελος', 'ο1', ['διογένης'], read=False)
2122
        new_data = get_random_data()
2123
        self.assert_not_raises_fault(403, chef.update_object,
2124
                                     'φάκελος', 'ο1', StringIO(new_data),
2125
                                     account=get_user())
2126

    
2127
        server_data = self.client.retrieve_object('φάκελος', 'ο1')
2128
        self.assertEqual(server_data[:len(o['data'])], o['data'])
2129
        self.assertEqual(server_data[len(o['data']):], new_data)
2130

    
2131
    def test_manifestation(self):
2132
        self.client.create_container('κουβάς')
2133
        prefix = 'μέρη/'
2134
        data = ''
2135
        for i in range(5):
2136
            part = '%s%d' %(prefix, i)
2137
            o = self.upload_random_data('κουβάς', part)
2138
            data += o['data']
2139

    
2140
        self.client.create_container('φάκελος')
2141
        manifest = '%s/%s' %('κουβάς', prefix)
2142
        self.client.create_manifestation('φάκελος', 'άπαντα', manifest)
2143

    
2144
        self.assert_object_exists('φάκελος', 'άπαντα')
2145
        self.assertEqual(data, self.client.retrieve_object('φάκελος',
2146
                                                           'άπαντα'))
2147

    
2148
        #wrong manifestation
2149
        self.client.create_manifestation('φάκελος', 'άπαντα', 'κουβάς/άκυρο')
2150
        self.assertEqual('', self.client.retrieve_object('φάκελος', 'άπαντα'))
2151

    
2152
    def test_update_from_another_object(self):
2153
        self.client.create_container('κουβάς')
2154
        src_data = self.upload_random_data('κουβάς', 'πηγή')['data']
2155
        initial_data = self.upload_random_data('κουβάς', 'νέο')['data']
2156
        source_object = '/%s/%s' % ('κουβάς', 'πηγή')
2157
        self.client.update_from_other_source('κουβάς', 'νέο', source_object)
2158

    
2159
        self.assertEqual(
2160
            self.client.retrieve_object('κουβάς', 'νέο'),
2161
            '%s%s' % (initial_data, self.client.retrieve_object('κουβάς', 'πηγή')))
2162

    
2163
class TestPermissions(BaseTestCase):
2164
    def setUp(self):
2165
        BaseTestCase.setUp(self)
2166

    
2167
        if not OTHER_ACCOUNTS:
2168
            raise Warning('No other accounts avalaible for running this test.')
2169

    
2170
        #create a group
2171
        self.authorized = ['chazapis', 'verigak', 'gtsouk']
2172
        groups = {'pithosdev':','.join(self.authorized)}
2173
        self.client.set_account_groups(**groups)
2174

    
2175
        self.container = 'c'
2176
        self.object = 'o'
2177
        self.client.create_container(self.container)
2178
        self.upload_random_data(self.container, self.object)
2179
        self.upload_random_data(self.container, self.object+'/')
2180
        self.upload_random_data(self.container, self.object+'/a')
2181
        self.upload_random_data(self.container, self.object+'a')
2182
        self.upload_random_data(self.container, self.object+'a/')
2183
        self.dir_content_types = ('application/directory', 'application/folder')
2184

    
2185
    def assert_read(self, authorized=[], any=False, depth=0):
2186
        for token, account in OTHER_ACCOUNTS.items():
2187
            cl = Pithos_Client(get_url(), token, account)
2188
            if account in authorized or any:
2189
                self.assert_not_raises_fault(403, cl.retrieve_object_metadata,
2190
                                             self.container, self.object,
2191
                                             account=get_user())
2192
            else:
2193
                self.assert_raises_fault(403, cl.retrieve_object_metadata,
2194
                                         self.container, self.object,
2195
                                         account=get_user())
2196

    
2197
        #check inheritance
2198
        meta = self.client.retrieve_object_metadata(self.container, self.object)
2199
        type = meta['content-type']
2200
        derivatives = self.client.list_objects(self.container, prefix=self.object)
2201
        #exclude the self.object
2202
        del derivatives[derivatives.index(self.object)]
2203
        for o in derivatives:
2204
            for token, account in OTHER_ACCOUNTS.items():
2205
                cl = Pithos_Client(get_url(), token, account)
2206
                prefix = self.object if self.object.endswith('/') else self.object+'/'
2207
                if (account in authorized or any) and \
2208
                (type in self.dir_content_types) and \
2209
                o.startswith(prefix):
2210
                    self.assert_not_raises_fault(403, cl.retrieve_object_metadata,
2211
                                             self.container, o, account=get_user())
2212
                else:
2213
                    self.assert_raises_fault(403, cl.retrieve_object_metadata,
2214
                                         self.container, o, account=get_user())
2215

    
2216
    def assert_write(self, authorized=[], any=False):
2217
        o_data = self.client.retrieve_object(self.container, self.object)
2218
        for token, account in OTHER_ACCOUNTS.items():
2219
            cl = Pithos_Client(get_url(), token, account)
2220
            new_data = get_random_data()
2221
            if account in authorized or any:
2222
                # test write access
2223
                self.assert_not_raises_fault(403, cl.update_object,
2224
                                             self.container, self.object, StringIO(new_data),
2225
                                             account=get_user())
2226
                try:
2227
                    # test read access
2228
                    server_data = cl.retrieve_object(self.container, self.object, account=get_user())
2229
                    self.assertEqual(o_data, server_data[:len(o_data)])
2230
                    self.assertEqual(new_data, server_data[len(o_data):])
2231
                    o_data = server_data
2232
                except Fault, f:
2233
                    self.failIf(f.status == 403)
2234
            else:
2235
                self.assert_raises_fault(403, cl.update_object,
2236
                                             self.container, self.object, StringIO(new_data),
2237
                                             account=get_user())
2238
        #check inheritance
2239
        meta = self.client.retrieve_object_metadata(self.container, self.object)
2240
        type = meta['content-type']
2241
        derivatives = self.client.list_objects(self.container, prefix=self.object)
2242
        #exclude the object
2243
        del derivatives[derivatives.index(self.object)]
2244
        for o in derivatives:
2245
            for token, account in OTHER_ACCOUNTS.items():
2246
                prefix = self.object if self.object.endswith('/') else self.object+'/'
2247
                cl = Pithos_Client(get_url(), token, account)
2248
                new_data = get_random_data()
2249
                if (account in authorized or any) and \
2250
                (type in self.dir_content_types) and \
2251
                o.startswith(prefix):
2252
                    # test write access
2253
                    self.assert_not_raises_fault(403, cl.update_object,
2254
                                                 self.container, o,
2255
                                                 StringIO(new_data),
2256
                                                 account=get_user())
2257
                    try:
2258
                        server_data = cl.retrieve_object(self.container, o, account=get_user())
2259
                        self.assertEqual(new_data, server_data[-len(new_data):])
2260
                    except Fault, f:
2261
                        self.failIf(f.status == 403)
2262
                else:
2263
                    self.assert_raises_fault(403, cl.update_object,
2264
                                                 self.container, o,
2265
                                                 StringIO(new_data),
2266
                                                 account=get_user())
2267

    
2268
    def test_group_read(self):
2269
        self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()])
2270
        self.assert_read(authorized=self.authorized)
2271

    
2272
    def test_read_many(self):
2273
        self.client.share_object(self.container, self.object, self.authorized)
2274
        self.assert_read(authorized=self.authorized)
2275

    
2276
    def test_read_by_everyone(self):
2277
        self.client.share_object(self.container, self.object, ['*'])
2278
        self.assert_read(any=True)
2279

    
2280
    def test_read_directory(self):
2281
        for type in self.dir_content_types:
2282
            #change content type
2283
            self.client.move_object(self.container, self.object, self.container, self.object, content_type=type)
2284
            self.client.share_object(self.container, self.object, ['*'])
2285
            self.assert_read(any=True)
2286
            self.client.share_object(self.container, self.object, self.authorized)
2287
            self.assert_read(authorized=self.authorized)
2288
            self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()])
2289
            self.assert_read(authorized=self.authorized)
2290

    
2291
    def test_group_write(self):
2292
        self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()], read=False)
2293
        self.assert_write(authorized=self.authorized)
2294

    
2295
    def test_write_many(self):
2296
        self.client.share_object(self.container, self.object, self.authorized, read=False)
2297
        self.assert_write(authorized=self.authorized)
2298

    
2299
    def test_write_by_everyone(self):
2300
        self.client.share_object(self.container, self.object, ['*'], read=False)
2301
        self.assert_write(any=True)
2302

    
2303
    def test_write_directory(self):
2304
        dir_content_types = ('application/directory', 'application/foler')
2305
        for type in dir_content_types:
2306
            #change content type
2307
            self.client.move_object(self.container, self.object, self.container, self.object, content_type='application/folder')
2308
            self.client.share_object(self.container, self.object, ['*'], read=False)
2309
            self.assert_write(any=True)
2310
            self.client.share_object(self.container, self.object, self.authorized, read=False)
2311
            self.assert_write(authorized=self.authorized)
2312
            self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()], read=False)
2313
            self.assert_write(authorized=self.authorized)
2314

    
2315
    def test_shared_listing(self):
2316
        self.client.share_object(self.container, self.object, self.authorized)
2317

    
2318
        my_shared_containers = self.client.list_containers(shared=True)
2319
        self.assertTrue('c' in my_shared_containers)
2320
        my_shared_objects = self.client.list_objects('c', shared=True)
2321
        self.assertEqual(['o'], my_shared_objects)
2322

    
2323
        dir_content_types = ('application/directory', 'application/foler')
2324
        for type in dir_content_types:
2325
            #change content type
2326
            self.client.move_object(self.container, self.object, self.container, self.object, content_type='application/folder')
2327
            my_shared_objects = self.client.list_objects('c', shared=True)
2328
            self.assertEqual(['o', 'o/', 'o/a'], my_shared_objects)
2329

    
2330
        for token, account in OTHER_ACCOUNTS.items():
2331
            if account in self.authorized:
2332
                self.other = Pithos_Client(get_url(), token, account)
2333
                self.assertTrue(get_user() in self.other.list_shared_with_me())
2334

    
2335
class TestPublish(BaseTestCase):
2336
    def test_publish(self):
2337
        self.client.create_container('c')
2338
        o_data = self.upload_random_data('c', 'o')['data']
2339
        self.client.publish_object('c', 'o')
2340
        meta = self.client.retrieve_object_metadata('c', 'o')
2341
        self.assertTrue('x-object-public' in meta)
2342
        url = meta['x-object-public']
2343

    
2344
        p = urlparse(get_url())
2345
        if p.scheme == 'http':
2346
            conn = HTTPConnection(p.netloc)
2347
        elif p.scheme == 'https':
2348
            conn = HTTPSConnection(p.netloc)
2349
        else:
2350
            raise Exception('Unknown URL scheme')
2351

    
2352
        conn.request('GET', url)
2353
        resp = conn.getresponse()
2354
        length = resp.getheader('content-length', None)
2355
        data = resp.read(length)
2356
        self.assertEqual(o_data, data)
2357

    
2358
class TestPolicies(BaseTestCase):
2359
    def test_none_versioning(self):
2360
        self.client.create_container('c', policies={'versioning':'none'})
2361
        o = self.upload_random_data('c', 'o')
2362
        meta = self.client.retrieve_object_metadata('c', 'o')
2363
        v = meta['x-object-version']
2364
        more_data = get_random_data()
2365
        self.client.update_object('c', 'o', StringIO(more_data))
2366
        vlist = self.client.retrieve_object_versionlist('c', 'o')
2367
        self.assert_raises_fault(404, self.client.retrieve_object_version,
2368
                                 'c', 'o', v)
2369
        data = self.client.retrieve_object('c', 'o')
2370
        end = len(o['data'])
2371
        self.assertEqual(data[:end], o['data'])
2372
        self.assertEqual(data[end:], more_data)
2373

    
2374
    def test_quota(self):
2375
        self.client.create_container('c', policies={'quota':'1'})
2376
        meta = self.client.retrieve_container_metadata('c')
2377
        self.assertEqual(meta['x-container-policy-quota'], '1')
2378
        self.assert_raises_fault(413, self.upload_random_data, 'c', 'o',
2379
                                 length=1024*1024+1)
2380

    
2381
    def test_quota_none(self):
2382
        self.client.create_container('c', policies={'quota':'0'})
2383
        meta = self.client.retrieve_container_metadata('c')
2384
        self.assertEqual(meta['x-container-policy-quota'], '0')
2385
        self.assert_not_raises_fault(413, self.upload_random_data, 'c', 'o',
2386
                                 length=1024*1024+1)
2387

    
2388
class TestUsageFreeVersioningAutoContainerPolicy(BaseTestCase):
2389
    """ Challenge free version accounting
2390
       in a container with auto versioning policy
2391

2392
       In case of unknown server version accounting policy or
2393
       debit version accounting policy
2394
       enforce all test cases to return immediately
2395
    """
2396
    def setUp(self):
2397
        BaseTestCase.setUp(self)
2398

    
2399
        # TODO Only works if tests are running in the same host with the server. Add support for remote server
2400
        self.stop_execution = False
2401
        try:
2402
            from pithos.api.settings import BACKEND_FREE_VERSIONING
2403
        except ImportError:
2404
            print 'Unable to execute the test: unknown version accounting policy'
2405
            self.stop_execution = True
2406
            return
2407
        else:
2408
            if not BACKEND_FREE_VERSIONING:
2409
                print 'Unable to execute the test: no free version accounting policy'
2410
                self.stop_execution = True
2411
                return
2412

    
2413
        self.create_container('container')
2414
        meta = self.client.retrieve_account_metadata()
2415
        self.initial_usage = int(meta['x-account-bytes-used'])
2416
        print 'Current account usage: %d' % self.initial_usage
2417

    
2418
        self.usage = self.initial_usage
2419
        #self.mtime = {}
2420
        l =  (50, 100, 150, 70, 80)
2421
        self.curr_acc_usage = self.initial_usage
2422
        for i, length in list(enumerate(l)):
2423
            self.upload_random_data('container', 'object', length=length)
2424
            meta = self.client.retrieve_account_metadata()
2425
            self.usage = int(meta['x-account-bytes-used'])
2426
            print 'Current account usage: %d' % self.usage
2427
            self.curr_acc_usage = self.initial_usage + length
2428
            self.assertEqual(self.usage, self.curr_acc_usage)
2429

    
2430
            #t = datetime.datetime.utcnow()
2431
            #self.mtime[i] = int(_time.mktime(t.timetuple()))
2432
            _time.sleep(1)
2433

    
2434
        versions = self.client.retrieve_object_versionlist(
2435
            'container', 'object'
2436
        )['versions']
2437
        self.mtime = [int(i[1]) for i in versions]
2438

    
2439
    def create_container(self, cname):
2440
        self.client.create_container(cname)
2441

    
2442
    def test_delete_object_container(self):
2443
        """
2444
            assume account usage = 0
2445
            scenario:
2446
                upload object (length=50)
2447
                update object (length=100)
2448
                update object (length=150)
2449
                update object (length=70)
2450
                update object (length=80)
2451
                delete object
2452
                delete container
2453
            account usage sequel: 50|100|150|70|80|0|0
2454
        """
2455
        if self.stop_execution:
2456
            return
2457

    
2458
        self.client.delete_object('container', 'object')
2459
        meta = self.client.retrieve_account_metadata()
2460
        self.usage = int(meta['x-account-bytes-used'])
2461
        print 'Current account usage: %d' % self.usage
2462
        self.assertEqual(self.usage, self.initial_usage)
2463

    
2464
        self.client.delete_container('container')
2465
        meta = self.client.retrieve_account_metadata()
2466
        self.usage = int(meta['x-account-bytes-used'])
2467
        print 'Current account usage: %d' % self.usage
2468
        self.assertEqual(self.usage, self.initial_usage)
2469

    
2470
    def test_purge_delete_object(self):
2471
        """
2472
            assume account usage = 0
2473
            scenario:
2474
                upload object (length=50)
2475
                update object (length=100)
2476
                update object (length=150)
2477
                update object (length=70)
2478
                update object (length=80)
2479
                delete object history
2480
                delete object
2481
                delete container
2482
            account usage sequel: 50|100|150|70|80|80|0|0
2483
        """
2484
        if self.stop_execution:
2485
            return
2486

    
2487
        # purge some object history
2488
        i = random.randrange(len(self.mtime))
2489
        self.client.delete_object('container', 'object', until=self.mtime[i])
2490
        meta = self.client.retrieve_account_metadata()
2491
        self.usage = int(meta['x-account-bytes-used'])
2492
        print 'Current account usage: %d' % self.usage
2493
        self.assertEqual(self.usage, self.curr_acc_usage)
2494

    
2495
        self.client.delete_object('container', 'object')
2496
        meta = self.client.retrieve_account_metadata()
2497
        self.usage = int(meta['x-account-bytes-used'])
2498
        print 'Current account usage: %d' % self.usage
2499
        self.assertEqual(self.usage, self.initial_usage)
2500

    
2501
        self.client.delete_container('container')
2502
        meta = self.client.retrieve_account_metadata()
2503
        self.usage = int(meta['x-account-bytes-used'])
2504
        print 'Current account usage: %d' % self.usage
2505
        self.assertEqual(self.usage, self.initial_usage)
2506

    
2507
    def test_delete_object_purge_container_history(self):
2508
        """
2509
            assume account usage = 0
2510
            scenario:
2511
                upload object (length=50)
2512
                update object (length=100)
2513
                update object (length=150)
2514
                update object (length=70)
2515
                update object (length=80)
2516
                delete object
2517
                delete container history
2518
                delete container
2519
            account usage sequel: 50|100|150|70|80|0|0|0
2520
        """
2521
        if self.stop_execution:
2522
            return
2523

    
2524
        self.client.delete_object('container', 'object')
2525
        meta = self.client.retrieve_account_metadata()
2526
        self.usage = int(meta['x-account-bytes-used'])
2527
        print 'Current account usage: %d' % self.usage
2528
        self.assertEqual(self.usage,
2529
                         self.initial_usage)
2530

    
2531
        # purge some container history
2532
        i = random.randrange(len(self.mtime))
2533
        self.client.delete_container('container', until=self.mtime[i])
2534
        meta = self.client.retrieve_account_metadata()
2535
        self.usage = int(meta['x-account-bytes-used'])
2536
        print 'Current account usage: %d' % self.usage
2537
        self.assertEqual(self.usage, self.initial_usage)
2538

    
2539
        self.client.delete_container('container')
2540
        meta = self.client.retrieve_account_metadata()
2541
        self.usage = int(meta['x-account-bytes-used'])
2542
        print 'Current account usage: %d' % self.usage
2543
        self.assertEqual(self.usage, self.initial_usage)
2544

    
2545
    def test_purge_container_delete_object(self):
2546
        """
2547
            assume account usage = 0
2548
            scenario:
2549
                upload object (length=50)
2550
                update object (length=100)
2551
                update object (length=150)
2552
                update object (length=70)
2553
                update object (length=80)
2554
                delete container history
2555
                delete object
2556
                delete container
2557
            account usage sequel: 50|100|150|70|80|80|0|0
2558
        """
2559
        if self.stop_execution:
2560
            return
2561

    
2562
        # purge some container history
2563
        i = random.randrange(len(self.mtime))
2564
        self.client.delete_container('container', until=self.mtime[i])
2565
        meta = self.client.retrieve_account_metadata()
2566
        self.usage = int(meta['x-account-bytes-used'])
2567
        print 'Current account usage: %d' % self.usage
2568
        self.assertEqual( self.usage, self.curr_acc_usage)
2569

    
2570
        self.client.delete_object('container', 'object')
2571
        meta = self.client.retrieve_account_metadata()
2572
        self.usage = int(meta['x-account-bytes-used'])
2573
        print 'Current account usage: %d' % self.usage
2574
        self.assertEqual(self.usage, self.initial_usage)
2575

    
2576
        self.client.delete_container('container')
2577
        meta = self.client.retrieve_account_metadata()
2578
        self.usage = int(meta['x-account-bytes-used'])
2579
        print 'Current account usage: %d' % self.usage
2580
        self.assertEqual(self.usage, self.initial_usage)
2581

    
2582
    def test_successive_purging(self):
2583
        """
2584
            assume account usage = 0
2585
            scenario:
2586
                upload object (length=50)
2587
                update object (length=100)
2588
                update object (length=150)
2589
                update object (length=70)
2590
                update object (length=80)
2591
                delete earlier 2 object history versions
2592
                delete next earlier object history version
2593
                delete rest object history and current version
2594
                delete object (404 status)
2595
                delete container
2596
            account usage sequel: 50|100|150|70|80|80|80|0|0|0
2597
        """
2598
        if self.stop_execution:
2599
            return
2600

    
2601
        # purge some object history
2602
        i = random.randrange(len(self.mtime))
2603
        self.client.delete_object('container', 'object', until=self.mtime[i])
2604
        meta = self.client.retrieve_account_metadata()
2605
        self.usage = int(meta['x-account-bytes-used'])
2606
        print 'Current account usage: %d' % self.usage
2607
        self.assertEqual(self.usage, self.curr_acc_usage)
2608

    
2609
        # purge some container history
2610
        i = random.randrange(len(self.mtime))
2611
        self.client.delete_container('container', until=self.mtime[i])
2612
        meta = self.client.retrieve_account_metadata()
2613
        self.usage = int(meta['x-account-bytes-used'])
2614
        print 'Current account usage: %d' % self.usage
2615
        self.assertEqual(self.usage, self.curr_acc_usage)
2616

    
2617
        # purge some more object history
2618
        i = random.randrange(len(self.mtime))
2619
        self.client.delete_object('container', 'object', until=self.mtime[i])
2620
        meta = self.client.retrieve_account_metadata()
2621
        self.usage = int(meta['x-account-bytes-used'])
2622
        print 'Current account usage: %d' % self.usage
2623
        self.assertEqual(self.usage, self.curr_acc_usage)
2624

    
2625
        # purge some object history and current
2626
        t = datetime.datetime.utcnow()
2627
        now = int(_time.mktime(t.timetuple()))
2628
        self.client.delete_object('container', 'object', until=now)
2629
        meta = self.client.retrieve_account_metadata()
2630
        self.usage = int(meta['x-account-bytes-used'])
2631
        print 'Current account usage: %d' % self.usage
2632
        self.assertEqual(self.usage, self.initial_usage)
2633

    
2634
        # try to delete object and assert object is not found
2635
        self.assert_raises_fault(
2636
            404, self.client.delete_object, 'container', 'object'
2637
        )
2638

    
2639
        meta = self.client.retrieve_account_metadata()
2640
        self.usage = int(meta['x-account-bytes-used'])
2641
        print 'Current account usage: %d' % self.usage
2642
        self.assertEqual(self.usage, self.initial_usage)
2643

    
2644
        self.client.delete_container('container')
2645
        meta = self.client.retrieve_account_metadata()
2646
        self.usage = int(meta['x-account-bytes-used'])
2647
        print 'Current account usage: %d' % self.usage
2648
        self.assertEqual(self.usage, self.initial_usage)
2649

    
2650
    def test_delete_object_empty_container_content(self):
2651
        """
2652
            assume account usage = 0
2653
            scenario:
2654
                upload object (length=50)
2655
                update object (length=100)
2656
                update object (length=150)
2657
                update object (length=70)
2658
                update object (length=80)
2659
                delete object
2660
                delete container contents
2661
                delete container
2662
            account usage sequel: 50|100|150|70|80|0|0|0
2663
        """
2664
        if self.stop_execution:
2665
            return
2666

    
2667
        self.client.delete_object('container', 'object')
2668

    
2669
        meta = self.client.retrieve_account_metadata()
2670
        self.usage = int(meta['x-account-bytes-used'])
2671
        print 'Current account usage: %d' % self.usage
2672
        self.assertEqual(self.usage, self.initial_usage)
2673

    
2674
        self.client.delete_container('container', delimiter='/')
2675
        meta = self.client.retrieve_account_metadata()
2676
        self.usage = int(meta['x-account-bytes-used'])
2677
        print 'Current account usage: %d' % self.usage
2678
        self.assertEqual(self.usage, self.initial_usage)
2679

    
2680
        self.client.delete_container('container')
2681
        meta = self.client.retrieve_account_metadata()
2682
        self.usage = int(meta['x-account-bytes-used'])
2683
        print 'Current account usage: %d' % self.usage
2684
        self.assertEqual(self.usage, self.initial_usage)
2685

    
2686
    def test_delete_container_content(self):
2687
        """
2688
            assume account usage = 0
2689
            scenario:
2690
                upload object (length=50)
2691
                update object (length=100)
2692
                update object (length=150)
2693
                update object (length=70)
2694
                update object (length=80)
2695
                delete container contents
2696
                delete container
2697
            account usage sequel: 50|100|150|70|80|0|0
2698
        """
2699
        if self.stop_execution:
2700
            return
2701

    
2702
        self.client.delete_container('container', delimiter='/')
2703
        meta = self.client.retrieve_account_metadata()
2704
        self.usage = int(meta['x-account-bytes-used'])
2705
        print 'Current account usage: %d' % self.usage
2706
        self.assertEqual(self.usage, self.initial_usage)
2707

    
2708
        self.client.delete_container('container')
2709
        meta = self.client.retrieve_account_metadata()
2710
        self.usage = int(meta['x-account-bytes-used'])
2711
        print 'Current account usage: %d' % self.usage
2712
        self.assertEqual(self.usage, self.initial_usage)
2713

    
2714

    
2715
class TestUsageFreeVersioningNoneContainerPolicy(
2716
    TestUsageFreeVersioningAutoContainerPolicy):
2717
    """ Challenge free version accounting
2718
       in a container with none versioning policy
2719

2720
       In case of unknown server version accounting policy or
2721
       debit version accounting policy
2722
       enforce all test cases to return immediately
2723
    """
2724
    def create_container(self, cname):
2725
        self.client.create_container(cname,
2726
                                     policies={'versioning':'none'})
2727

    
2728
class TestUsageDebitVersioningAutoContainerPolicy(BaseTestCase):
2729
    """ Challenge debit version accounting
2730
       in a container with auto versioning policy
2731

2732
       In case of unknown server version accounting policy or
2733
       free version accounting policy
2734
       enforce all test cases to return immediately
2735
    """
2736
    def setUp(self):
2737
        BaseTestCase.setUp(self)
2738

    
2739
        self.stop_execution = False
2740
        try:
2741
            from pithos.api.settings import BACKEND_FREE_VERSIONING
2742
        except ImportError:
2743
            print 'Unable to execute the test: unknown version accounting policy'
2744
            self.stop_execution = True
2745
            return
2746
        else:
2747
            if BACKEND_FREE_VERSIONING:
2748
                print 'Unable to execute the test: free version accounting policy'
2749
                self.stop_execution = True
2750
                return
2751

    
2752
        self.create_container('container')
2753
        meta = self.client.retrieve_account_metadata()
2754
        self.initial_usage = int(meta['x-account-bytes-used'])
2755
        print 'Current account usage: %d' % self.initial_usage
2756

    
2757
        self.usage = self.initial_usage
2758
        self.l = (50, 100, 150, 70, 80)
2759
        self.curr_acc_usage = self.initial_usage
2760
        for i, length in list(enumerate(self.l)):
2761
            self.upload_random_data('container', 'object', length=length)
2762
            meta = self.client.retrieve_account_metadata()
2763
            self.usage = int(meta['x-account-bytes-used'])
2764
            print 'Current account usage: %d' % self.usage
2765
            self.curr_acc_usage += length
2766
            self.assertEqual(self.usage, self.curr_acc_usage)
2767
            _time.sleep(1)
2768

    
2769
        versions = self.client.retrieve_object_versionlist(
2770
            'container', 'object'
2771
        )['versions']
2772
        self.mtime = [int(i[1]) for i in versions]
2773

    
2774
    def create_container(self, cname):
2775
        self.client.create_container(cname)
2776

    
2777
    def test_delete_object_container(self):
2778
        """
2779
            assume account usage = 0
2780
            scenario:
2781
                upload object (length=50)
2782
                update object (length=100)
2783
                update object (length=150)
2784
                update object (length=70)
2785
                update object (length=80)
2786
                delete object
2787
                delete container
2788
            account usage sequel: 50|150|300|370|450|450|0
2789
        """
2790
        if self.stop_execution:
2791
            return
2792

    
2793
        self.client.delete_object('container', 'object')
2794
        meta = self.client.retrieve_account_metadata()
2795
        self.usage = int(meta['x-account-bytes-used'])
2796
        print 'Current account usage: %d' % self.usage
2797
        self.assertEqual(self.usage, self.initial_usage + sum(self.l))
2798

    
2799
        self.client.delete_container('container')
2800
        meta = self.client.retrieve_account_metadata()
2801
        self.usage = int(meta['x-account-bytes-used'])
2802
        print 'Current account usage: %d' % self.usage
2803
        self.assertEqual(self.usage, self.initial_usage)
2804

    
2805
    def test_purge_delete_object(self):
2806
        """
2807
            assume account usage = 0
2808
            scenario:
2809
                upload object (length=50)
2810
                update object (length=100)
2811
                update object (length=150)
2812
                update object (length=70)
2813
                update object (length=80)
2814
                delete 3 earlier object versions
2815
                delete object
2816
                delete container
2817
            account usage sequel: 50|150|300|370|450|150|150|0
2818
        """
2819
        if self.stop_execution:
2820
            return
2821

    
2822
        # purge some object history
2823
        i = 3
2824
        self.client.delete_object('container', 'object', until=self.mtime[i])
2825
        meta = self.client.retrieve_account_metadata()
2826
        self.usage = int(meta['x-account-bytes-used'])
2827
        print 'Current account usage: %d' % self.usage
2828
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2829

    
2830
        self.client.delete_object('container', 'object')
2831
        meta = self.client.retrieve_account_metadata()
2832
        self.usage = int(meta['x-account-bytes-used'])
2833
        print 'Current account usage: %d' % self.usage
2834
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2835

    
2836
        self.client.delete_container('container')
2837
        meta = self.client.retrieve_account_metadata()
2838
        self.usage = int(meta['x-account-bytes-used'])
2839
        print 'Current account usage: %d' % self.usage
2840
        self.assertEqual(self.usage, self.initial_usage)
2841

    
2842
    def test_delete_object_purge_container_history(self):
2843
        """
2844
            assume account usage = 0
2845
            scenario:
2846
                upload object (length=50)
2847
                update object (length=100)
2848
                update object (length=150)
2849
                update object (length=70)
2850
                update object (length=80)
2851
                delete object
2852
                delete container history (3 earlier object versions)
2853
                delete container
2854
            account usage sequel: 50|150|300|370|450|450|150|0
2855
        """
2856
        if self.stop_execution:
2857
            return
2858

    
2859
        self.client.delete_object('container', 'object')
2860
        meta = self.client.retrieve_account_metadata()
2861
        self.usage = int(meta['x-account-bytes-used'])
2862
        print 'Current account usage: %d' % self.usage
2863
        self.assertEqual(self.usage, self.curr_acc_usage)
2864

    
2865
        # purge some container history
2866
        i = 3
2867
        self.client.delete_container('container', until=self.mtime[i])
2868
        meta = self.client.retrieve_account_metadata()
2869
        self.usage = int(meta['x-account-bytes-used'])
2870
        print 'Current account usage: %d' % self.usage
2871
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2872

    
2873
        self.client.delete_container('container')
2874
        meta = self.client.retrieve_account_metadata()
2875
        self.usage = int(meta['x-account-bytes-used'])
2876
        print 'Current account usage: %d' % self.usage
2877
        self.assertEqual(self.usage, self.initial_usage)
2878

    
2879
    def test_purge_container_delete_object(self):
2880
        """
2881
            assume account usage = 0
2882
            scenario:
2883
                upload object (length=50)
2884
                update object (length=100)
2885
                update object (length=150)
2886
                update object (length=70)
2887
                update object (length=80)
2888
                delete container history
2889
                delete object
2890
                delete container
2891
            account usage sequel: 50|150|300|370|450|150|150|0
2892
        """
2893
        if self.stop_execution:
2894
            return
2895

    
2896
        versions = self.client.retrieve_object_versionlist(
2897
            'container', 'object'
2898
        )['versions']
2899
        mtime = [i[1] for i in versions]
2900

    
2901
        # purge some container history
2902
        i = 3
2903
        until = int(mtime[i])
2904
        self.client.delete_container('container', until=until)
2905
        meta = self.client.retrieve_account_metadata()
2906
        self.usage = int(meta['x-account-bytes-used'])
2907
        print 'Current account usage: %d' % self.usage
2908
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2909

    
2910
        self.client.delete_object('container', 'object')
2911
        meta = self.client.retrieve_account_metadata()
2912
        self.usage = int(meta['x-account-bytes-used'])
2913
        print 'Current account usage: %d' % self.usage
2914
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2915

    
2916

    
2917
        self.client.delete_container('container')
2918
        meta = self.client.retrieve_account_metadata()
2919
        self.usage = int(meta['x-account-bytes-used'])
2920
        print 'Current account usage: %d' % self.usage
2921
        self.assertEqual(self.usage, self.initial_usage)
2922

    
2923
    def test_successive_purging(self):
2924
        """
2925
            assume account usage = 0
2926
            scenario:
2927
                upload object (length=50)
2928
                update object (length=100)
2929
                update object (length=150)
2930
                update object (length=70)
2931
                update object (length=80)
2932
                delete earlier 2 object history versions
2933
                delete next earlier object history version
2934
                delete rest object history and current version
2935
                delete object (404 status)
2936
                delete container
2937
            account usage sequel: 50|150|300|370|450|400|300|150|0|0
2938
        """
2939
        if self.stop_execution:
2940
            return
2941

    
2942
        versions = self.client.retrieve_object_versionlist(
2943
            'container', 'object'
2944
        )['versions']
2945
        mtime = [i[1] for i in versions]
2946

    
2947
        # purge some object history
2948
        i = 1
2949
        until = int(mtime[i])
2950
        self.client.delete_object('container', 'object', until=until)
2951
        meta = self.client.retrieve_account_metadata()
2952
        self.usage = int(meta['x-account-bytes-used'])
2953
        print 'Current account usage: %d' % self.usage
2954
        self.curr_acc_usage = self.curr_acc_usage - sum(self.l[:i])
2955
        self.assertEqual(self.usage, self.curr_acc_usage)
2956

    
2957
        # purge some container history
2958
        j = 2
2959
        until = int(mtime[j])
2960
        self.client.delete_container('container', until=until)
2961
        meta = self.client.retrieve_account_metadata()
2962
        self.usage = int(meta['x-account-bytes-used'])
2963
        print 'Current account usage: %d' % self.usage
2964
        self.curr_acc_usage = self.curr_acc_usage - sum(self.l[i:j])
2965
        self.assertEqual(self.usage, self.curr_acc_usage)
2966

    
2967
        # purge some more object history
2968
        k = 3
2969
        until = int(mtime[k])
2970
        self.client.delete_object('container', 'object', until=until)
2971
        meta = self.client.retrieve_account_metadata()
2972
        self.usage = int(meta['x-account-bytes-used'])
2973
        print 'Current account usage: %d' % self.usage
2974
        self.curr_acc_usage = self.curr_acc_usage - sum(self.l[j:k])
2975
        self.assertEqual(self.usage, self.curr_acc_usage)
2976

    
2977
        # purge some object history and current
2978
        t = datetime.datetime.utcnow()
2979
        now = int(_time.mktime(t.timetuple()))
2980
        self.client.delete_object('container', 'object', until=now)
2981
        meta = self.client.retrieve_account_metadata()
2982
        self.usage = int(meta['x-account-bytes-used'])
2983
        print 'Current account usage: %d' % self.usage
2984
        self.assertEqual(self.usage, self.initial_usage)
2985

    
2986
        # try to delete object and assert object is not found
2987
        self.assert_raises_fault(
2988
            404, self.client.delete_object, 'container', 'object'
2989
        )
2990

    
2991
        meta = self.client.retrieve_account_metadata()
2992
        self.usage = int(meta['x-account-bytes-used'])
2993
        print 'Current account usage: %d' % self.usage
2994
        self.assertEqual(self.usage, self.initial_usage)
2995

    
2996
        self.client.delete_container('container')
2997
        meta = self.client.retrieve_account_metadata()
2998
        self.usage = int(meta['x-account-bytes-used'])
2999
        print 'Current account usage: %d' % self.usage
3000
        self.assertEqual(self.usage, self.initial_usage)
3001

    
3002
    def test_delete_object_empty_container_content(self):
3003
        """
3004
            assume account usage = 0
3005
            scenario:
3006
                upload object (length=50)
3007
                update object (length=100)
3008
                update object (length=150)
3009
                update object (length=70)
3010
                update object (length=80)
3011
                delete object
3012
                delete container contents
3013
                delete container
3014
            account usage sequel: 50|150|300|370|450|450|450|0
3015
        """
3016
        if self.stop_execution:
3017
            return
3018

    
3019
        self.client.delete_object('container', 'object')
3020

    
3021
        meta = self.client.retrieve_account_metadata()
3022
        self.usage = int(meta['x-account-bytes-used'])
3023
        print 'Current account usage: %d' % self.usage
3024
        self.assertEqual(self.usage, self.curr_acc_usage)
3025

    
3026
        self.client.delete_container('container', delimiter='/')
3027
        meta = self.client.retrieve_account_metadata()
3028
        self.usage = int(meta['x-account-bytes-used'])
3029
        print 'Current account usage: %d' % self.usage
3030
        self.assertEqual(self.usage, self.curr_acc_usage)
3031

    
3032
        self.client.delete_container('container')
3033
        meta = self.client.retrieve_account_metadata()
3034
        self.usage = int(meta['x-account-bytes-used'])
3035
        print 'Current account usage: %d' % self.usage
3036
        self.assertEqual(self.usage, self.initial_usage)
3037

    
3038
    def test_delete_container_content(self):
3039
        """
3040
            assume account usage = 0
3041
            scenario:
3042
                upload object (length=50)
3043
                update object (length=100)
3044
                update object (length=150)
3045
                update object (length=70)
3046
                update object (length=80)
3047
                delete container contents
3048
                delete container
3049
            account usage sequel: 50|150|300|370|450|450|0
3050
        """
3051
        if self.stop_execution:
3052
            return
3053

    
3054
        self.client.delete_container('container', delimiter='/')
3055
        meta = self.client.retrieve_account_metadata()
3056
        self.usage = int(meta['x-account-bytes-used'])
3057
        print 'Current account usage: %d' % self.usage
3058
        self.assertEqual(self.usage, self.curr_acc_usage)
3059

    
3060
        self.client.delete_container('container')
3061
        meta = self.client.retrieve_account_metadata()
3062
        self.usage = int(meta['x-account-bytes-used'])
3063
        print 'Current account usage: %d' % self.usage
3064
        self.assertEqual(self.usage, self.initial_usage)
3065

    
3066
class TestUsageDebitVersioningNoneContainerPolicy(BaseTestCase):
3067
    """ Challenge debit version accounting
3068
       in a container with none versioning policy
3069

3070
       In case of unknown server version accounting policy or
3071
       free version accounting policy
3072
       enforce all test cases to return immediately
3073
    """
3074
    def setUp(self):
3075
        BaseTestCase.setUp(self)
3076

    
3077
        self.stop_execution = False
3078
        try:
3079
            from pithos.api.settings import BACKEND_FREE_VERSIONING
3080
        except ImportError:
3081
            print 'Unable to execute the test: unknown version accounting policy'
3082
            self.stop_execution = True
3083
            return
3084
        else:
3085
            if BACKEND_FREE_VERSIONING:
3086
                print 'Unable to execute the test: free version accounting policy'
3087
                self.stop_execution = True
3088
                return
3089

    
3090
        self.create_container('container')
3091
        meta = self.client.retrieve_account_metadata()
3092
        self.initial_usage = int(meta['x-account-bytes-used'])
3093
        print 'Current account usage: %d' % self.initial_usage
3094

    
3095
        l =  (50, 100, 150, 70, 80)
3096
        self.curr_acc_usage = self.initial_usage
3097
        for i, length in list(enumerate(l)):
3098
            self.upload_random_data('container', 'object', length=length)
3099
            meta = self.client.retrieve_account_metadata()
3100
            self.usage = int(meta['x-account-bytes-used'])
3101
            print 'Current account usage: %d' % self.usage
3102
            self.curr_acc_usage = self.initial_usage + length
3103
            self.assertEqual(self.usage, self.curr_acc_usage)
3104
            _time.sleep(1)
3105

    
3106
        versions = self.client.retrieve_object_versionlist(
3107
            'container', 'object'
3108
        )['versions']
3109
        self.mtime = [int(i[1]) for i in versions]
3110

    
3111
    def create_container(self, cname):
3112
        self.client.create_container(cname,
3113
                                     policies={'versioning':'none'})
3114

    
3115
    def test_delete_object_container(self):
3116
        """
3117
            assume account usage = 0
3118
            scenario:
3119
                upload object (length=50)
3120
                update object (length=100)
3121
                update object (length=150)
3122
                update object (length=70)
3123
                update object (length=80)
3124
                delete object
3125
                delete container
3126
            account usage sequel: 50|100|150|70|80|0|0
3127
        """
3128
        if self.stop_execution:
3129
            return
3130

    
3131
        self.client.delete_object('container', 'object')
3132
        meta = self.client.retrieve_account_metadata()
3133
        self.usage = int(meta['x-account-bytes-used'])
3134
        print 'Current account usage: %d' % self.usage
3135
        self.assertEqual(self.usage, self.initial_usage)
3136

    
3137
        self.client.delete_container('container')
3138
        meta = self.client.retrieve_account_metadata()
3139
        self.usage = int(meta['x-account-bytes-used'])
3140
        print 'Current account usage: %d' % self.usage
3141
        self.assertEqual(self.usage, self.initial_usage)
3142

    
3143
    def test_purge_delete_object(self):
3144
        """
3145
            assume account usage = 0
3146
            scenario:
3147
                upload object (length=50)
3148
                update object (length=100)
3149
                update object (length=150)
3150
                update object (length=70)
3151
                update object (length=80)
3152
                delete object history
3153
                delete object
3154
                delete container
3155
            account usage sequel: 50|100|150|70|80|80|0|0
3156
        """
3157
        if self.stop_execution:
3158
            return
3159

    
3160
        # purge some object history
3161
        self.client.delete_object('container', 'object', until=self.mtime[0])
3162
        meta = self.client.retrieve_account_metadata()
3163
        self.usage = int(meta['x-account-bytes-used'])
3164
        print 'Current account usage: %d' % self.usage
3165
        self.assertEqual(self.usage, self.curr_acc_usage)
3166

    
3167
        self.client.delete_object('container', 'object')
3168
        meta = self.client.retrieve_account_metadata()
3169
        self.usage = int(meta['x-account-bytes-used'])
3170
        print 'Current account usage: %d' % self.usage
3171
        self.assertEqual(self.usage, self.initial_usage)
3172

    
3173
        self.client.delete_container('container')
3174
        meta = self.client.retrieve_account_metadata()
3175
        self.usage = int(meta['x-account-bytes-used'])
3176
        print 'Current account usage: %d' % self.usage
3177
        self.assertEqual(self.usage, self.initial_usage)
3178

    
3179
    def test_delete_object_purge_container_history(self):
3180
        """
3181
            assume account usage = 0
3182
            scenario:
3183
                upload object (length=50)
3184
                update object (length=100)
3185
                update object (length=150)
3186
                update object (length=70)
3187
                update object (length=80)
3188
                delete object
3189
                delete container history
3190
                delete container
3191
            account usage sequel: 50|100|150|70|80|0|0|0
3192
        """
3193
        if self.stop_execution:
3194
            return
3195

    
3196
        self.client.delete_object('container', 'object')
3197
        meta = self.client.retrieve_account_metadata()
3198
        self.usage = int(meta['x-account-bytes-used'])
3199
        print 'Current account usage: %d' % self.usage
3200
        self.assertEqual(self.usage, self.initial_usage)
3201

    
3202
        # purge some container history
3203
        self.client.delete_container('container', until=self.mtime[0])
3204
        meta = self.client.retrieve_account_metadata()
3205
        self.usage = int(meta['x-account-bytes-used'])
3206
        print 'Current account usage: %d' % self.usage
3207
        self.assertEqual(self.usage, self.initial_usage)
3208

    
3209
        self.client.delete_container('container')
3210
        meta = self.client.retrieve_account_metadata()
3211
        self.usage = int(meta['x-account-bytes-used'])
3212
        print 'Current account usage: %d' % self.usage
3213
        self.assertEqual(self.usage, self.initial_usage)
3214

    
3215
    def test_purge_container_delete_object(self):
3216
        """
3217
            assume account usage = 0
3218
            scenario:
3219
                upload object (length=50)
3220
                update object (length=100)
3221
                update object (length=150)
3222
                update object (length=70)
3223
                update object (length=80)
3224
                delete container history
3225
                delete object
3226
                delete container
3227
            account usage sequel: 50|100|150|70|80|80|0|0
3228
        """
3229
        if self.stop_execution:
3230
            return
3231

    
3232
        # purge some container history
3233
        self.client.delete_container('container', until=self.mtime[0])
3234
        meta = self.client.retrieve_account_metadata()
3235
        self.usage = int(meta['x-account-bytes-used'])
3236
        print 'Current account usage: %d' % self.usage
3237
        self.assertEqual(self.usage, self.curr_acc_usage)
3238

    
3239
        self.client.delete_object('container', 'object')
3240
        meta = self.client.retrieve_account_metadata()
3241
        self.usage = int(meta['x-account-bytes-used'])
3242
        print 'Current account usage: %d' % self.usage
3243
        self.assertEqual(self.usage, self.initial_usage)
3244

    
3245
        self.client.delete_container('container')
3246
        meta = self.client.retrieve_account_metadata()
3247
        self.usage = int(meta['x-account-bytes-used'])
3248
        print 'Current account usage: %d' % self.usage
3249
        self.assertEqual(self.usage, self.initial_usage)
3250

    
3251
    def test_successive_purging(self):
3252
        """
3253
            assume account usage = 0
3254
            scenario:
3255
                upload object (length=50)
3256
                update object (length=100)
3257
                update object (length=150)
3258
                update object (length=70)
3259
                update object (length=80)
3260
                delete earlier 2 object history versions
3261
                delete next earlier object history version
3262
                delete rest object history and current version
3263
                delete object (404 status)
3264
                delete container
3265
            account usage sequel: 50|100|150|70|80|80|80|80|0|0|0
3266
        """
3267
        if self.stop_execution:
3268
            return
3269

    
3270
        # purge some object history
3271
        i = random.randrange(len(self.mtime))
3272
        self.client.delete_object('container', 'object', until=self.mtime[i])
3273
        meta = self.client.retrieve_account_metadata()
3274
        self.usage = int(meta['x-account-bytes-used'])
3275
        print 'Current account usage: %d' % self.usage
3276
        self.assertEqual(self.usage, self.curr_acc_usage)
3277

    
3278
        # purge some object history and current
3279
        t = datetime.datetime.utcnow()
3280
        now = int(_time.mktime(t.timetuple()))
3281
        self.client.delete_object('container', 'object', until=now)
3282
        meta = self.client.retrieve_account_metadata()
3283
        self.usage = int(meta['x-account-bytes-used'])
3284
        print 'Current account usage: %d' % self.usage
3285
        self.assertEqual(self.usage, self.initial_usage)
3286

    
3287
        # try to delete object and assert object is not found
3288
        self.assert_raises_fault(
3289
            404, self.client.delete_object, 'container', 'object'
3290
        )
3291

    
3292
        meta = self.client.retrieve_account_metadata()
3293
        self.usage = int(meta['x-account-bytes-used'])
3294
        print 'Current account usage: %d' % self.usage
3295
        self.assertEqual(self.usage, self.initial_usage)
3296

    
3297
        self.client.delete_container('container')
3298
        meta = self.client.retrieve_account_metadata()
3299
        self.usage = int(meta['x-account-bytes-used'])
3300
        print 'Current account usage: %d' % self.usage
3301
        self.assertEqual(self.usage, self.initial_usage)
3302

    
3303
    def test_delete_object_empty_container_content(self):
3304
        """
3305
            assume account usage = 0
3306
            scenario:
3307
                upload object (length=50)
3308
                update object (length=100)
3309
                update object (length=150)
3310
                update object (length=70)
3311
                update object (length=80)
3312
                delete object
3313
                delete container contents
3314
                delete container
3315
            account usage sequel: 50|100|150|70|80|0|0|0
3316
        """
3317
        if self.stop_execution:
3318
            return
3319

    
3320
        self.client.delete_object('container', 'object')
3321
        meta = self.client.retrieve_account_metadata()
3322
        self.usage = int(meta['x-account-bytes-used'])
3323
        print 'Current account usage: %d' % self.usage
3324
        self.assertEqual(self.usage, self.initial_usage)
3325

    
3326
        self.client.delete_container('container', delimiter='/')
3327
        meta = self.client.retrieve_account_metadata()
3328
        self.usage = int(meta['x-account-bytes-used'])
3329
        print 'Current account usage: %d' % self.usage
3330
        self.assertEqual(self.usage, self.initial_usage)
3331

    
3332
        self.client.delete_container('container')
3333
        meta = self.client.retrieve_account_metadata()
3334
        self.usage = int(meta['x-account-bytes-used'])
3335
        print 'Current account usage: %d' % self.usage
3336
        self.assertEqual(self.usage, self.initial_usage)
3337

    
3338
    def test_delete_container_content(self):
3339
        """
3340
            assume account usage = 0
3341
            scenario:
3342
                upload object (length=50)
3343
                update object (length=100)
3344
                update object (length=150)
3345
                update object (length=70)
3346
                update object (length=80)
3347
                delete container contents
3348
                delete container
3349
            account usage sequel: 50|100|150|70|80|0|0
3350
        """
3351
        if self.stop_execution:
3352
            return
3353

    
3354
        self.client.delete_container('container', delimiter='/')
3355
        meta = self.client.retrieve_account_metadata()
3356
        self.usage = int(meta['x-account-bytes-used'])
3357
        print 'Current account usage: %d' % self.usage
3358
        self.assertEqual(self.usage, self.initial_usage)
3359

    
3360
        self.client.delete_container('container')
3361
        meta = self.client.retrieve_account_metadata()
3362
        self.usage = int(meta['x-account-bytes-used'])
3363
        print 'Current account usage: %d' % self.usage
3364
        self.assertEqual(self.usage, self.initial_usage)
3365

    
3366
class AssertUUidInvariant(object):
3367
    def __init__(self, callable, *args, **kwargs):
3368
        self.callable = callable
3369
        self.args = args
3370
        self.kwargs = kwargs
3371

    
3372
    def __enter__(self):
3373
        self.map = self.callable(*self.args, **self.kwargs)
3374
        assert('x-object-uuid' in self.map)
3375
        self.uuid = self.map['x-object-uuid']
3376
        return self.map
3377

    
3378
    def __exit__(self, type, value, tb):
3379
        map = self.callable(*self.args, **self.kwargs)
3380
        assert('x-object-uuid' in self.map)
3381
        uuid = map['x-object-uuid']
3382
        assert(uuid == self.uuid)
3383

    
3384
class AssertMappingInvariant(object):
3385
    def __init__(self, callable, *args, **kwargs):
3386
        self.callable = callable
3387
        self.args = args
3388
        self.kwargs = kwargs
3389

    
3390
    def __enter__(self):
3391
        self.map = self.callable(*self.args, **self.kwargs)
3392
        return self.map
3393

    
3394
    def __exit__(self, type, value, tb):
3395
        map = self.callable(*self.args, **self.kwargs)
3396
        for k, v in self.map.items():
3397
            if is_date(v):
3398
                continue
3399
            assert(k in map)
3400
            assert v == map[k]
3401

    
3402
class AssertContentInvariant(object):
3403
    def __init__(self, callable, *args, **kwargs):
3404
        self.callable = callable
3405
        self.args = args
3406
        self.kwargs = kwargs
3407

    
3408
    def __enter__(self):
3409
        self.content = self.callable(*self.args, **self.kwargs)[2]
3410
        return self.content
3411

    
3412
    def __exit__(self, type, value, tb):
3413
        content = self.callable(*self.args, **self.kwargs)[2]
3414
        assert self.content == content
3415

    
3416
def get_content_splitted(response):
3417
    if response:
3418
        return response.content.split('\n')
3419

    
3420
def compute_md5_hash(data):
3421
    md5 = hashlib.md5()
3422
    offset = 0
3423
    md5.update(data)
3424
    return md5.hexdigest().lower()
3425

    
3426
def compute_block_hash(data, algorithm):
3427
    h = hashlib.new(algorithm)
3428
    h.update(data.rstrip('\x00'))
3429
    return h.hexdigest()
3430

    
3431
def get_random_data(length=500):