Statistics
| Branch: | Tag: | Revision:

root / kamaki / clients / livetest / pithos.py @ 023230b5

History | View | Annotate | Download (43.8 kB)

1
# Copyright 2012-2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
import time
35
import datetime
36
from os import urandom
37
from tempfile import NamedTemporaryFile
38

    
39
from kamaki.clients import livetest, ClientError
40
from kamaki.clients.pithos import PithosClient
41
from kamaki.clients.astakos import AstakosClient
42

    
43

    
44
class Pithos(livetest.Generic):
45

    
46
    files = []
47

    
48
    def setUp(self):
49
        self.client = PithosClient(
50
            self['store', 'url'],
51
            self['store', 'token'],
52
            AstakosClient(
53
                self['astakos', 'url'],
54
                self['store', 'token']
55
            ).term('uuid'))
56

    
57
        self.now = time.mktime(time.gmtime())
58
        self.now_unformated = datetime.datetime.utcnow()
59
        self._init_data()
60

    
61
        """Prepare an object to be shared - also its container"""
62
        self.client.container = self.c1
63
        self.client.object_post(
64
            'test',
65
            update=True,
66
            permissions={'read': [self.client.account]})
67

    
68
        self.create_remote_object(self.c1, 'another.test')
69

    
70
    def _init_data(self):
71
        self.c1 = 'c1_' + unicode(self.now)
72
        self.c2 = 'c2_' + unicode(self.now)
73
        self.c3 = 'c3_' + unicode(self.now)
74
        try:
75
            self.client.create_container(self.c2)
76
        except ClientError:
77
            pass
78
        try:
79
            self.client.create_container(self.c1)
80
        except ClientError:
81
            pass
82
        try:
83
            self.client.create_container(self.c3)
84
        except ClientError:
85
            pass
86

    
87
        self.create_remote_object(self.c1, 'test')
88
        self.create_remote_object(self.c2, 'test')
89
        self.create_remote_object(self.c1, 'test1')
90
        self.create_remote_object(self.c2, 'test1')
91

    
92
    def create_remote_object(self, container, obj):
93
        self.client.container = container
94
        self.client.object_put(
95
            obj,
96
            content_type='application/octet-stream',
97
            data='file %s that lives in %s' % (obj, container),
98
            metadata={'incontainer': container})
99

    
100
    def forceDeleteContainer(self, container):
101
        self.client.container = container
102
        try:
103
            r = self.client.list_objects()
104
        except ClientError:
105
            return
106
        for obj in r:
107
            name = obj['name']
108
            self.client.del_object(name)
109
        r = self.client.container_delete()
110
        self.container = ''
111

    
112
    def tearDown(self):
113
        """Destroy test cases"""
114
        for f in self.files:
115
            f.close()
116
        self.forceDeleteContainer(self.c1)
117
        self.forceDeleteContainer(self.c2)
118
        try:
119
            self.forceDeleteContainer(self.c3)
120
        except ClientError:
121
            pass
122
        self.client.container = ''
123

    
124
    def test_000(self):
125
        """Prepare a full Pithos+ test"""
126
        print('')
127
        super(self.__class__, self).test_000()
128

    
129
    def test_account_head(self):
130
        """Test account_HEAD"""
131
        self._test_0010_account_head()
132

    
133
    def _test_0010_account_head(self):
134
        r = self.client.account_head()
135
        self.assertEqual(r.status_code, 204)
136

    
137
        r = self.client.account_head(until='1000000000')
138
        self.assertEqual(r.status_code, 204)
139

    
140
        r = self.client.get_account_info(until='1000000000')
141
        datestring = unicode(r['x-account-until-timestamp'])
142
        self.assertEqual(u'Sun, 09 Sep 2001 01:46:40 GMT', datestring)
143

    
144
        r = self.client.get_account_quota()
145
        self.assertTrue('x-account-policy-quota' in r)
146

    
147
        r = self.client.get_account_versioning()
148
        self.assertTrue('x-account-policy-versioning' in r)
149

    
150
        """Check if(un)modified_since"""
151
        for format in self.client.DATE_FORMATS:
152
            now_formated = self.now_unformated.strftime(format)
153
            r1 = self.client.account_head(
154
                if_modified_since=now_formated,
155
                success=(204, 304, 412))
156
            sc1 = r1.status_code
157
            r1.release()
158
            r2 = self.client.account_head(
159
                if_unmodified_since=now_formated,
160
                success=(204, 304, 412))
161
            sc2 = r2.status_code
162
            r2.release()
163
            self.assertNotEqual(sc1, sc2)
164

    
165
    def test_account_get(self):
166
        """Test account_GET"""
167
        self._test_0020_account_get()
168

    
169
    def _test_0020_account_get(self):
170
        #r = self.client.account_get()
171
        #self.assertEqual(r.status_code, 200)
172
        r = self.client.list_containers()
173
        fullLen = len(r)
174
        self.assertTrue(fullLen > 2)
175

    
176
        r = self.client.account_get(limit=1)
177
        self.assertEqual(len(r.json), 1)
178

    
179
        r = self.client.account_get(marker='c2_')
180
        temp_c0 = r.json[0]['name']
181
        temp_c2 = r.json[2]['name']
182

    
183
        r = self.client.account_get(limit=2, marker='c2_')
184
        conames = [container['name'] for container in r.json if (
185
            container['name'].lower().startswith('c2_'))]
186
        self.assertTrue(temp_c0 in conames)
187
        self.assertFalse(temp_c2 in conames)
188

    
189
        r = self.client.account_get(show_only_shared=True)
190
        self.assertTrue(self.c1 in [c['name'] for c in r.json])
191

    
192
        r = self.client.account_get(until=1342609206)
193
        self.assertTrue(len(r.json) <= fullLen)
194

    
195
        """Check if(un)modified_since"""
196
        for format in self.client.DATE_FORMATS:
197
            now_formated = self.now_unformated.strftime(format)
198
            r1 = self.client.account_get(
199
                if_modified_since=now_formated,
200
                success=(200, 304, 412))
201
            sc1 = r1.status_code
202
            r1.release()
203
            r2 = self.client.account_get(
204
                if_unmodified_since=now_formated,
205
                success=(200, 304, 412))
206
            sc2 = r2.status_code
207
            r2.release()
208
            self.assertNotEqual(sc1, sc2)
209

    
210
        """Check sharing_accounts"""
211
        r = self.client.get_sharing_accounts()
212
        self.assertTrue(len(r) > 0)
213

    
214
    def test_account_post(self):
215
        """Test account_POST"""
216
        self._test_0030_account_post()
217

    
218
    def _test_0030_account_post(self):
219
        r = self.client.account_post()
220
        self.assertEqual(r.status_code, 202)
221
        grpName = 'grp' + unicode(self.now)
222

    
223
        """Method set/del_account_meta and set_account_groupcall use
224
            account_post internally
225
        """
226
        u1 = self.client.account
227
        u2 = '1nc0r3c7-d15p14y-n4m3'
228
        self.assertRaises(
229
            ClientError,
230
            self.client.set_account_group,
231
            grpName, [u1, u2])
232
        self.client.set_account_group(grpName, [u1])
233
        r = self.client.get_account_group()
234
        self.assertEqual(r['x-account-group-' + grpName], '%s' % u1)
235
        self.client.del_account_group(grpName)
236
        r = self.client.get_account_group()
237
        self.assertTrue('x-account-group-' + grpName not in r)
238

    
239
        mprefix = 'meta' + unicode(self.now)
240
        self.client.set_account_meta({
241
            mprefix + '1': 'v1',
242
            mprefix + '2': 'v2'})
243
        r = self.client.get_account_meta()
244
        self.assertEqual(r['x-account-meta-' + mprefix + '1'], 'v1')
245
        self.assertEqual(r['x-account-meta-' + mprefix + '2'], 'v2')
246

    
247
        self.client.del_account_meta(mprefix + '1')
248
        r = self.client.get_account_meta()
249
        self.assertTrue('x-account-meta-' + mprefix + '1' not in r)
250

    
251
        self.client.del_account_meta(mprefix + '2')
252
        r = self.client.get_account_meta()
253
        self.assertTrue('x-account-meta-' + mprefix + '2' not in r)
254

    
255
        """Missing testing for quota, versioning, because normally
256
        you don't have permissions to modify those at account level
257
        """
258

    
259
        newquota = 1000000
260
        self.client.set_account_quota(newquota)
261
        #r = self.client.get_account_info()
262
        #print(unicode(r))
263
        #r = self.client.get_account_quota()
264
        #self.assertEqual(r['x-account-policy-quota'], newquota)
265
        self.client.set_account_versioning('auto')
266

    
267
    def test_container_head(self):
268
        """Test container_HEAD"""
269
        self._test_0040_container_head()
270

    
271
    def _test_0040_container_head(self):
272
        self.client.container = self.c1
273

    
274
        r = self.client.container_head()
275
        self.assertEqual(r.status_code, 204)
276

    
277
        """Check until"""
278
        r = self.client.container_head(until=1000000, success=(204, 404))
279
        self.assertEqual(r.status_code, 404)
280

    
281
        """Check and if(un)modified_since"""
282
        for format in self.client.DATE_FORMATS:
283
            now_formated = self.now_unformated.strftime(format)
284
            r1 = self.client.container_head(
285
                if_modified_since=now_formated,
286
                success=(204, 304, 412))
287
            sc1 = r1.status_code
288
            r1.release()
289
            r2 = self.client.container_head(
290
                if_unmodified_since=now_formated,
291
                success=(204, 304, 412))
292
            sc2 = r2.status_code
293
            r2.release()
294
            self.assertNotEqual(sc1, sc2)
295

    
296
        """Check container object meta"""
297
        r = self.client.get_container_object_meta()
298
        self.assertEqual(r['x-container-object-meta'], 'Incontainer')
299

    
300
    def test_container_get(self):
301
        """Test container_GET"""
302
        self._test_0050_container_get()
303

    
304
    def _test_0050_container_get(self):
305
        self.client.container = self.c1
306

    
307
        r = self.client.container_get()
308
        self.assertEqual(r.status_code, 200)
309
        fullLen = len(r.json)
310

    
311
        r = self.client.container_get(prefix='test')
312
        lalobjects = [obj for obj in r.json if obj['name'].startswith('test')]
313
        self.assertTrue(len(r.json) > 1)
314
        self.assertEqual(len(r.json), len(lalobjects))
315

    
316
        r = self.client.container_get(limit=1)
317
        self.assertEqual(len(r.json), 1)
318

    
319
        r = self.client.container_get(marker='another')
320
        self.assertTrue(len(r.json) > 1)
321
        neobjects = [obj for obj in r.json if obj['name'] > 'another']
322
        self.assertEqual(len(r.json), len(neobjects))
323

    
324
        r = self.client.container_get(prefix='another.test', delimiter='.')
325
        self.assertTrue(fullLen > len(r.json))
326

    
327
        r = self.client.container_get(path='/')
328
        self.assertEqual(fullLen, len(r.json))
329

    
330
        r = self.client.container_get(format='xml')
331
        self.assertEqual(r.text.split()[4], 'name="' + self.c1 + '">')
332

    
333
        r = self.client.container_get(meta=['incontainer'])
334
        self.assertTrue(len(r.json) > 0)
335

    
336
        r = self.client.container_get(show_only_shared=True)
337
        self.assertTrue(len(r.json) < fullLen)
338

    
339
        try:
340
            r = self.client.container_get(until=1000000000)
341
            datestring = unicode(r.headers['x-account-until-timestamp'])
342
            self.assertEqual(u'Sun, 09 Sep 2001 01:46:40 GMT', datestring)
343

    
344
        except ClientError:
345

    
346
            pass
347

    
348
        """Check and if un/modified_since"""
349
        for format in self.client.DATE_FORMATS:
350
            now_formated = self.now_unformated.strftime(format)
351
            r1 = self.client.container_get(
352
                if_modified_since=now_formated,
353
                success=(200, 304, 412))
354
            sc1 = r1.status_code
355
            r1.release()
356
            r2 = self.client.container_get(
357
                if_unmodified_since=now_formated,
358
                success=(200, 304, 412))
359
            sc2 = r2.status_code
360
            r2.release()
361
            self.assertNotEqual(sc1, sc2)
362

    
363
    def test_container_put(self):
364
        """Test container_PUT"""
365
        self._test_0050_container_put()
366

    
367
    def _test_0050_container_put(self):
368
        self.client.container = self.c2
369

    
370
        r = self.client.container_put()
371
        self.assertEqual(r.status_code, 202)
372

    
373
        r = self.client.get_container_quota(self.client.container)
374
        cquota = r.values()[0]
375
        newquota = 2 * int(cquota)
376

    
377
        r = self.client.container_put(quota=newquota)
378
        self.assertEqual(r.status_code, 202)
379

    
380
        r = self.client.get_container_quota(self.client.container)
381
        xquota = int(r.values()[0])
382
        self.assertEqual(newquota, xquota)
383

    
384
        r = self.client.container_put(versioning='auto')
385
        self.assertEqual(r.status_code, 202)
386

    
387
        r = self.client.get_container_versioning(self.client.container)
388
        nvers = r.values()[0]
389
        self.assertEqual('auto', nvers)
390

    
391
        r = self.client.container_put(versioning='none')
392
        self.assertEqual(r.status_code, 202)
393

    
394
        r = self.client.get_container_versioning(self.client.container)
395
        nvers = r.values()[0]
396
        self.assertEqual('none', nvers)
397

    
398
        r = self.client.container_put(metadata={'m1': 'v1', 'm2': 'v2'})
399
        self.assertEqual(r.status_code, 202)
400

    
401
        r = self.client.get_container_meta(self.client.container)
402
        self.assertTrue('x-container-meta-m1' in r)
403
        self.assertEqual(r['x-container-meta-m1'], 'v1')
404
        self.assertTrue('x-container-meta-m2' in r)
405
        self.assertEqual(r['x-container-meta-m2'], 'v2')
406

    
407
        r = self.client.container_put(metadata={'m1': '', 'm2': 'v2a'})
408
        self.assertEqual(r.status_code, 202)
409

    
410
        r = self.client.get_container_meta(self.client.container)
411
        self.assertTrue('x-container-meta-m1' not in r)
412
        self.assertTrue('x-container-meta-m2' in r)
413
        self.assertEqual(r['x-container-meta-m2'], 'v2a')
414

    
415
        self.client.del_container_meta(self.client.container)
416

    
417
    def test_container_post(self):
418
        """Test container_POST"""
419
        self._test_0060_container_post()
420

    
421
    def _test_0060_container_post(self):
422
        self.client.container = self.c2
423

    
424
        """Simple post"""
425
        r = self.client.container_post()
426
        self.assertEqual(r.status_code, 202)
427

    
428
        """post meta"""
429
        self.client.set_container_meta({'m1': 'v1', 'm2': 'v2'})
430
        r = self.client.get_container_meta(self.client.container)
431
        self.assertTrue('x-container-meta-m1' in r)
432
        self.assertEqual(r['x-container-meta-m1'], 'v1')
433
        self.assertTrue('x-container-meta-m2' in r)
434
        self.assertEqual(r['x-container-meta-m2'], 'v2')
435

    
436
        """post/2del meta"""
437
        r = self.client.del_container_meta('m1')
438
        r = self.client.set_container_meta({'m2': 'v2a'})
439
        r = self.client.get_container_meta(self.client.container)
440
        self.assertTrue('x-container-meta-m1' not in r)
441
        self.assertTrue('x-container-meta-m2' in r)
442
        self.assertEqual(r['x-container-meta-m2'], 'v2a')
443

    
444
        """check quota"""
445
        r = self.client.get_container_quota(self.client.container)
446
        cquota = r.values()[0]
447
        newquota = 2 * int(cquota)
448
        r = self.client.set_container_quota(newquota)
449
        r = self.client.get_container_quota(self.client.container)
450
        xquota = int(r.values()[0])
451
        self.assertEqual(newquota, xquota)
452
        r = self.client.set_container_quota(cquota)
453
        r = self.client.get_container_quota(self.client.container)
454
        xquota = r.values()[0]
455
        self.assertEqual(cquota, xquota)
456

    
457
        """Check versioning"""
458
        self.client.set_container_versioning('auto')
459
        r = self.client.get_container_versioning(self.client.container)
460
        nvers = r.values()[0]
461
        self.assertEqual('auto', nvers)
462
        self.client.set_container_versioning('none')
463
        r = self.client.get_container_versioning(self.client.container)
464
        nvers = r.values()[0]
465
        self.assertEqual('none', nvers)
466

    
467
        """put_block uses content_type and content_length to
468
        post blocks of data 2 container. All that in upload_object"""
469
        """Change a file at fs"""
470
        f = self.create_large_file(1024 * 1024 * 100)
471
        """Upload it at a directory in container"""
472
        self.client.create_directory('dir')
473
        self.client.upload_object('/dir/sample.file', f)
474
        """Check if file has been uploaded"""
475
        r = self.client.get_object_info('/dir/sample.file')
476
        self.assertTrue(int(r['content-length']) > 100000000)
477

    
478
        """What is tranfer_encoding? What should I check about it? """
479
        #TODO
480

    
481
        """Check update=False"""
482
        r = self.client.object_post(
483
            'test',
484
            update=False,
485
            metadata={'newmeta': 'newval'})
486

    
487
        r = self.client.get_object_info('test')
488
        self.assertTrue('x-object-meta-newmeta' in r)
489
        self.assertFalse('x-object-meta-incontainer' in r)
490

    
491
        r = self.client.del_container_meta('m2')
492

    
493
    def test_container_delete(self):
494
        """Test container_DELETE"""
495
        self._test_0070_container_delete()
496

    
497
    def _test_0070_container_delete(self):
498

    
499
        """Fail to delete a non-empty container"""
500
        self.client.container = self.c2
501
        r = self.client.container_delete(success=409)
502
        self.assertEqual(r.status_code, 409)
503

    
504
        """Fail to delete c3 (empty) container"""
505
        self.client.container = self.c3
506
        r = self.client.container_delete(until='1000000000')
507
        self.assertEqual(r.status_code, 204)
508

    
509
        """Delete c3 (empty) container"""
510
        r = self.client.container_delete()
511
        self.assertEqual(r.status_code, 204)
512

    
513
        """Purge container(empty a container), check versionlist"""
514
        self.client.container = self.c1
515
        r = self.client.object_head('test', success=(200, 404))
516
        self.assertEqual(r.status_code, 200)
517
        self.client.del_container(delimiter='/')
518
        r = self.client.object_head('test', success=(200, 404))
519
        self.assertEqual(r.status_code, 404)
520
        r = self.client.get_object_versionlist('test')
521
        self.assertTrue(len(r) > 0)
522
        self.assertTrue(len(r[0]) > 1)
523
        self.client.purge_container()
524
        self.assertRaises(
525
            ClientError,
526
            self.client.get_object_versionlist,
527
            'test')
528

    
529
    def _test_0080_recreate_deleted_data(self):
530
        self._init_data()
531

    
532
    def test_object_head(self):
533
        """Test object_HEAD"""
534
        self._test_0090_object_head()
535

    
536
    def _test_0090_object_head(self):
537
        self.client.container = self.c2
538
        obj = 'test'
539

    
540
        r = self.client.object_head(obj)
541
        self.assertEqual(r.status_code, 200)
542
        etag = r.headers['etag']
543

    
544
        r = self.client.object_head(obj, version=40)
545
        self.assertEqual(r.headers['x-object-version'], '40')
546

    
547
        r = self.client.object_head(obj, if_etag_match=etag)
548
        self.assertEqual(r.status_code, 200)
549

    
550
        r = self.client.object_head(
551
            obj,
552
            if_etag_not_match=etag,
553
            success=(200, 412, 304))
554
        self.assertNotEqual(r.status_code, 200)
555

    
556
        r = self.client.object_head(
557
            obj,
558
            version=40,
559
            if_etag_match=etag,
560
            success=412)
561
        self.assertEqual(r.status_code, 412)
562

    
563
        """Check and if(un)modified_since"""
564
        for format in self.client.DATE_FORMATS:
565
            now_formated = self.now_unformated.strftime(format)
566
            r1 = self.client.object_head(
567
                obj,
568
                if_modified_since=now_formated,
569
                success=(200, 304, 412))
570
            sc1 = r1.status_code
571
            r1.release()
572
            r2 = self.client.object_head(
573
                obj,
574
                if_unmodified_since=now_formated,
575
                success=(200, 304, 412))
576
            sc2 = r2.status_code
577
            r2.release()
578
            self.assertNotEqual(sc1, sc2)
579

    
580
    def test_object_get(self):
581
        """Test object_GET"""
582
        self._test_0100_object_get()
583

    
584
    def _test_0100_object_get(self):
585
        self.client.container = self.c1
586
        obj = 'test'
587

    
588
        r = self.client.object_get(obj)
589
        self.assertEqual(r.status_code, 200)
590

    
591
        osize = int(r.headers['content-length'])
592
        etag = r.headers['etag']
593

    
594
        r = self.client.object_get(obj, hashmap=True)
595
        for term in ('hashes', 'block_hash', 'block_hash', 'bytes'):
596
            self.assertTrue(term in r.json)
597

    
598
        r = self.client.object_get(obj, format='xml', hashmap=True)
599
        self.assertEqual(len(r.text.split('hash>')), 3)
600

    
601
        rangestr = 'bytes=%s-%s' % (osize / 3, osize / 2)
602
        r = self.client.object_get(
603
            obj,
604
            data_range=rangestr,
605
            success=(200, 206))
606
        partsize = int(r.headers['content-length'])
607
        self.assertTrue(0 < partsize and partsize <= 1 + osize / 3)
608

    
609
        rangestr = 'bytes=%s-%s' % (osize / 3, osize / 2)
610
        r = self.client.object_get(
611
            obj,
612
            data_range=rangestr,
613
            if_range=True,
614
            success=(200, 206))
615
        partsize = int(r.headers['content-length'])
616
        self.assertTrue(0 < partsize and partsize <= 1 + osize / 3)
617

    
618
        r = self.client.object_get(obj, if_etag_match=etag)
619
        self.assertEqual(r.status_code, 200)
620

    
621
        r = self.client.object_get(obj, if_etag_not_match=etag + 'LALALA')
622
        self.assertEqual(r.status_code, 200)
623

    
624
        """Check and if(un)modified_since"""
625
        for format in self.client.DATE_FORMATS:
626
            now_formated = self.now_unformated.strftime(format)
627
            r1 = self.client.object_get(
628
                obj,
629
                if_modified_since=now_formated,
630
                success=(200, 304, 412))
631
            sc1 = r1.status_code
632
            r1.release()
633
            r2 = self.client.object_get(
634
                obj,
635
                if_unmodified_since=now_formated,
636
                success=(200, 304, 412))
637
            sc2 = r2.status_code
638
            r2.release()
639
            self.assertNotEqual(sc1, sc2)
640

    
641
        """Upload an object to download"""
642
        trg_fname = 'remotefile_%s' % self.now
643
        f_size = 59247824
644
        src_f = self.create_large_file(f_size)
645
        print('\tUploading...')
646
        self.client.upload_object(trg_fname, src_f)
647
        print('\tDownloading...')
648
        self.files.append(NamedTemporaryFile())
649
        dnl_f = self.files[-1]
650
        self.client.download_object(trg_fname, dnl_f)
651

    
652
        print('\tCheck if files match...')
653
        for pos in (0, f_size / 2, f_size - 20):
654
            src_f.seek(pos)
655
            dnl_f.seek(pos)
656
            self.assertEqual(src_f.read(10), dnl_f.read(10))
657

    
658
    def test_object_put(self):
659
        """Test object_PUT"""
660
        self._test_object_put()
661

    
662
    def _test_object_put(self):
663
        self.client.container = self.c2
664
        obj = 'another.test'
665

    
666
        self.client.create_object(obj + '.FAKE')
667
        r = self.client.get_object_info(obj + '.FAKE')
668
        self.assertEqual(r['content-type'], 'application/octet-stream')
669

    
670
        """create the object"""
671
        r = self.client.object_put(
672
            obj,
673
            data='a',
674
            content_type='application/octer-stream',
675
            permissions=dict(
676
                read=['accX:groupA', 'u1', 'u2'],
677
                write=['u2', 'u3']),
678
            metadata=dict(key1='val1', key2='val2'),
679
            content_encoding='UTF-8',
680
            content_disposition='attachment; filename="fname.ext"')
681
        self.assertEqual(r.status_code, 201)
682
        etag = r.headers['etag']
683

    
684
        """Check content-disposition"""
685
        r = self.client.get_object_info(obj)
686
        self.assertTrue('content-disposition' in r)
687

    
688
        """Check permissions"""
689
        r = self.client.get_object_sharing(obj)
690
        self.assertTrue('accx:groupa' in r['read'])
691
        self.assertTrue('u1' in r['read'])
692
        self.assertTrue('u2' in r['write'])
693
        self.assertTrue('u3' in r['write'])
694

    
695
        """Check metadata"""
696
        r = self.client.get_object_meta(obj)
697
        self.assertEqual(r['x-object-meta-key1'], 'val1')
698
        self.assertEqual(r['x-object-meta-key2'], 'val2')
699

    
700
        """Check public and if_etag_match"""
701
        r = self.client.object_put(
702
            obj,
703
            if_etag_match=etag,
704
            data='b',
705
            content_type='application/octet-stream',
706
            public=True)
707

    
708
        r = self.client.object_get(obj)
709
        self.assertTrue('x-object-public' in r.headers)
710
        vers2 = int(r.headers['x-object-version'])
711
        etag = r.headers['etag']
712
        self.assertEqual(r.text, 'b')
713

    
714
        """Check if_etag_not_match"""
715
        r = self.client.object_put(
716
            obj,
717
            if_etag_not_match=etag,
718
            data='c',
719
            content_type='application/octet-stream',
720
            success=(201, 412))
721
        self.assertEqual(r.status_code, 412)
722

    
723
        """Check content_type and content_length"""
724
        tmpdir = 'dir' + unicode(self.now)
725
        r = self.client.object_put(
726
            tmpdir,
727
            content_type='application/directory',
728
            content_length=0)
729

    
730
        r = self.client.get_object_info(tmpdir)
731
        self.assertEqual(r['content-type'], 'application/directory')
732

    
733
        """Check copy_from, content_encoding"""
734
        r = self.client.object_put(
735
            '%s/%s' % (tmpdir, obj),
736
            format=None,
737
            copy_from='/%s/%s' % (self.client.container, obj),
738
            content_encoding='application/octet-stream',
739
            source_account=self.client.account,
740
            content_length=0,
741
            success=201)
742
        self.assertEqual(r.status_code, 201)
743

    
744
        """Test copy_object for cross-conctainer copy"""
745
        self.client.copy_object(
746
            src_container=self.c2,
747
            src_object='%s/%s' % (tmpdir, obj),
748
            dst_container=self.c1,
749
            dst_object=obj)
750
        self.client.container = self.c1
751
        r1 = self.client.get_object_info(obj)
752
        self.client.container = self.c2
753
        r2 = self.client.get_object_info('%s/%s' % (tmpdir, obj))
754
        self.assertEqual(r1['x-object-hash'], r2['x-object-hash'])
755

    
756
        """Check cross-container copy_from, content_encoding"""
757
        self.client.container = self.c1
758
        fromstr = '/%s/%s/%s' % (self.c2, tmpdir, obj)
759
        r = self.client.object_put(
760
            obj,
761
            format=None,
762
            copy_from=fromstr,
763
            content_encoding='application/octet-stream',
764
            source_account=self.client.account,
765
            content_length=0,
766
            success=201)
767

    
768
        self.assertEqual(r.status_code, 201)
769
        r = self.client.get_object_info(obj)
770
        self.assertEqual(r['etag'], etag)
771

    
772
        """Check source_account"""
773
        self.client.container = self.c2
774
        fromstr = '/%s/%s' % (self.c1, obj)
775
        r = self.client.object_put(
776
            '%sv2' % obj,
777
            format=None,
778
            move_from=fromstr,
779
            content_encoding='application/octet-stream',
780
            source_account='nonExistendAddress@NeverLand.com',
781
            content_length=0,
782
            success=(201, 403))
783
        self.assertEqual(r.status_code, 403)
784

    
785
        """Check cross-container move_from"""
786
        self.client.container = self.c1
787
        r1 = self.client.get_object_info(obj)
788
        self.client.container = self.c2
789
        self.client.move_object(
790
            src_container=self.c1,
791
            src_object=obj,
792
            dst_container=self.c2,
793
            dst_object=obj + 'v0')
794
        r0 = self.client.get_object_info(obj + 'v0')
795
        self.assertEqual(r1['x-object-hash'], r0['x-object-hash'])
796

    
797
        """Check move_from"""
798
        r = self.client.object_put(
799
            '%sv1' % obj,
800
            format=None,
801
            move_from='/%s/%s' % (self.c2, obj),
802
            source_version=vers2,
803
            content_encoding='application/octet-stream',
804
            content_length=0, success=201)
805

    
806
        """Check manifest"""
807
        mobj = 'manifest.test'
808
        txt = ''
809
        for i in range(10):
810
            txt += '%s' % i
811
            r = self.client.object_put(
812
                '%s/%s' % (mobj, i),
813
                data='%s' % i,
814
                content_length=1,
815
                success=201,
816
                content_type='application/octet-stream',
817
                content_encoding='application/octet-stream')
818

    
819
        r = self.client.object_put(
820
            mobj,
821
            content_length=0,
822
            content_type='application/octet-stream',
823
            manifest='%s/%s' % (self.client.container, mobj))
824

    
825
        r = self.client.object_get(mobj)
826
        self.assertEqual(r.text, txt)
827

    
828
        """Upload a local file with one request"""
829
        newf = self.create_large_file(1024 * 10)
830
        self.client.upload_object('sample.file', newf)
831
        """Check if file has been uploaded"""
832
        r = self.client.get_object_info('sample.file')
833
        self.assertEqual(int(r['content-length']), 10240)
834

    
835
        """Some problems with transfer-encoding?"""
836

    
837
    def test_object_copy(self):
838
        """Test object_COPY"""
839
        self._test_0110_object_copy()
840

    
841
    def _test_0110_object_copy(self):
842
        #  TODO: check with source_account option
843
        self.client.container = self.c2
844
        obj = 'test2'
845

    
846
        data = '{"key1":"val1", "key2":"val2"}'
847
        r = self.client.object_put(
848
            '%sorig' % obj,
849
            content_type='application/octet-stream',
850
            data=data,
851
            metadata=dict(mkey1='mval1', mkey2='mval2'),
852
            permissions=dict(
853
                read=['accX:groupA', 'u1', 'u2'],
854
                write=['u2', 'u3']),
855
            content_disposition='attachment; filename="fname.ext"')
856

    
857
        r = self.client.object_copy(
858
            '%sorig' % obj,
859
            destination='/%s/%s' % (self.client.container, obj),
860
            ignore_content_type=False, content_type='application/json',
861
            metadata={'mkey2': 'mval2a', 'mkey3': 'mval3'},
862
            permissions={'write': ['u5', 'accX:groupB']})
863
        self.assertEqual(r.status_code, 201)
864

    
865
        """Check content-disposition"""
866
        r = self.client.get_object_info(obj)
867
        self.assertTrue('content-disposition' in r)
868

    
869
        """Check Metadata"""
870
        r = self.client.get_object_meta(obj)
871
        self.assertEqual(r['x-object-meta-mkey1'], 'mval1')
872
        self.assertEqual(r['x-object-meta-mkey2'], 'mval2a')
873
        self.assertEqual(r['x-object-meta-mkey3'], 'mval3')
874

    
875
        """Check permissions"""
876
        r = self.client.get_object_sharing(obj)
877
        self.assertFalse('read' in r or 'u2' in r['write'])
878
        self.assertTrue('accx:groupb' in r['write'])
879

    
880
        """Check destination account"""
881
        r = self.client.object_copy(
882
            obj,
883
            destination='/%s/%s' % (self.c1, obj),
884
            content_encoding='utf8',
885
            content_type='application/json',
886
            destination_account='nonExistendAddress@NeverLand.com',
887
            success=(201, 403))
888
        self.assertEqual(r.status_code, 403)
889

    
890
        """Check destination being another container
891
        and also content_type and content encoding"""
892
        r = self.client.object_copy(
893
            obj,
894
            destination='/%s/%s' % (self.c1, obj),
895
            content_encoding='utf8',
896
            content_type='application/json')
897
        self.assertEqual(r.status_code, 201)
898
        self.assertEqual(
899
            r.headers['content-type'],
900
            'application/json; charset=UTF-8')
901

    
902
        """Check ignore_content_type and content_type"""
903
        r = self.client.object_get(obj)
904
        etag = r.headers['etag']
905
        ctype = r.headers['content-type']
906
        self.assertEqual(ctype, 'application/json')
907

    
908
        r = self.client.object_copy(
909
            '%sorig' % obj,
910
            destination='/%s/%s0' % (self.client.container, obj),
911
            ignore_content_type=True,
912
            content_type='application/json')
913
        self.assertEqual(r.status_code, 201)
914
        self.assertNotEqual(r.headers['content-type'], 'application/json')
915

    
916
        """Check if_etag_(not_)match"""
917
        r = self.client.object_copy(
918
            obj,
919
            destination='/%s/%s1' % (self.client.container, obj),
920
            if_etag_match=etag)
921
        self.assertEqual(r.status_code, 201)
922

    
923
        r = self.client.object_copy(
924
            obj,
925
            destination='/%s/%s2' % (self.client.container, obj),
926
            if_etag_not_match='lalala')
927
        self.assertEqual(r.status_code, 201)
928
        vers2 = r.headers['x-object-version']
929

    
930
        """Check source_version, public and format """
931
        r = self.client.object_copy(
932
            '%s2' % obj,
933
            destination='/%s/%s3' % (self.client.container, obj),
934
            source_version=vers2,
935
            format='xml',
936
            public=True)
937
        self.assertEqual(r.status_code, 201)
938
        self.assertTrue(r.headers['content-type'].index('xml') > 0)
939

    
940
        r = self.client.get_object_info(obj + '3')
941
        self.assertTrue('x-object-public' in r)
942

    
943
    def test_object_move(self):
944
        """Test object_MOVE"""
945
        self._test_0120_object_move()
946

    
947
    def _test_0120_object_move(self):
948
        self.client.container = self.c2
949
        obj = 'test2'
950

    
951
        data = '{"key1": "val1", "key2": "val2"}'
952
        r = self.client.object_put(
953
            '%sorig' % obj,
954
            content_type='application/octet-stream',
955
            data=data,
956
            metadata=dict(mkey1='mval1', mkey2='mval2'),
957
            permissions=dict(
958
                read=['accX:groupA', 'u1', 'u2'],
959
                write=['u2', 'u3']))
960

    
961
        r = self.client.object_move(
962
            '%sorig' % obj,
963
            destination='/%s/%s' % (self.client.container, obj),
964
            ignore_content_type=False,
965
            content_type='application/json',
966
            metadata=dict(mkey2='mval2a', mkey3='mval3'),
967
            permissions=dict(write=['u5', 'accX:groupB']))
968
        self.assertEqual(r.status_code, 201)
969

    
970
        """Check Metadata"""
971
        r = self.client.get_object_meta(obj)
972
        self.assertEqual(r['x-object-meta-mkey1'], 'mval1')
973
        self.assertEqual(r['x-object-meta-mkey2'], 'mval2a')
974
        self.assertEqual(r['x-object-meta-mkey3'], 'mval3')
975

    
976
        """Check permissions"""
977
        r = self.client.get_object_sharing(obj)
978
        self.assertFalse('read' in r)
979
        self.assertTrue('u5' in r['write'])
980
        self.assertTrue('accx:groupb' in r['write'])
981

    
982
        """Check destination account"""
983
        r = self.client.object_move(
984
            obj,
985
            destination='/%s/%s' % (self.c1, obj),
986
            content_encoding='utf8',
987
            content_type='application/json',
988
            destination_account='nonExistendAddress@NeverLand.com',
989
            success=(201, 403))
990
        self.assertEqual(r.status_code, 403)
991

    
992
        """Check destination being another container and also
993
        content_type, content_disposition and content encoding"""
994
        r = self.client.object_move(
995
            obj,
996
            destination='/%s/%s' % (self.c1, obj),
997
            content_encoding='utf8',
998
            content_type='application/json',
999
            content_disposition='attachment; filename="fname.ext"')
1000
        self.assertEqual(r.status_code, 201)
1001
        self.assertEqual(
1002
            r.headers['content-type'],
1003
            'application/json; charset=UTF-8')
1004
        self.client.container = self.c1
1005
        r = self.client.get_object_info(obj)
1006
        self.assertTrue('content-disposition' in r)
1007
        self.assertTrue('fname.ext' in r['content-disposition'])
1008
        etag = r['etag']
1009
        ctype = r['content-type']
1010
        self.assertEqual(ctype, 'application/json')
1011

    
1012
        """Check ignore_content_type and content_type"""
1013
        r = self.client.object_move(
1014
            obj,
1015
            destination='/%s/%s' % (self.c2, obj),
1016
            ignore_content_type=True,
1017
            content_type='application/json')
1018
        self.assertEqual(r.status_code, 201)
1019
        self.assertNotEqual(r.headers['content-type'], 'application/json')
1020

    
1021
        """Check if_etag_(not_)match"""
1022
        self.client.container = self.c2
1023
        r = self.client.object_move(
1024
            obj,
1025
            destination='/%s/%s0' % (self.client.container, obj),
1026
            if_etag_match=etag)
1027
        self.assertEqual(r.status_code, 201)
1028

    
1029
        r = self.client.object_move(
1030
            '%s0' % obj,
1031
            destination='/%s/%s1' % (self.client.container, obj),
1032
            if_etag_not_match='lalala')
1033
        self.assertEqual(r.status_code, 201)
1034

    
1035
        """Check public and format """
1036
        r = self.client.object_move(
1037
            '%s1' % obj,
1038
            destination='/%s/%s2' % (self.client.container, obj),
1039
            format='xml',
1040
            public=True)
1041
        self.assertEqual(r.status_code, 201)
1042
        self.assertTrue(r.headers['content-type'].index('xml') > 0)
1043

    
1044
        r = self.client.get_object_info(obj + '2')
1045
        self.assertTrue('x-object-public' in r)
1046

    
1047
    def test_object_post(self):
1048
        """Test object_POST"""
1049
        self._test_0130_object_post()
1050

    
1051
    def _test_0130_object_post(self):
1052
        self.client.container = self.c2
1053
        obj = 'test2'
1054
        """create a filesystem file"""
1055
        self.files.append(NamedTemporaryFile())
1056
        newf = self.files[-1]
1057
        newf.writelines([
1058
            'ello!\n',
1059
            'This is a test line\n',
1060
            'inside a test file\n'])
1061
        """create a file on container"""
1062
        r = self.client.object_put(
1063
            obj,
1064
            content_type='application/octet-stream',
1065
            data='H',
1066
            metadata=dict(mkey1='mval1', mkey2='mval2'),
1067
            permissions=dict(
1068
                read=['accX:groupA', 'u1', 'u2'],
1069
                write=['u2', 'u3']))
1070

    
1071
        """Append livetest update, content_[range|type|length]"""
1072
        newf.seek(0)
1073
        self.client.append_object(obj, newf)
1074
        r = self.client.object_get(obj)
1075
        self.assertTrue(r.text.startswith('Hello!'))
1076

    
1077
        """Overwrite livetest update,
1078
            content_type, content_length, content_range
1079
        """
1080
        newf.seek(0)
1081
        r = self.client.overwrite_object(obj, 0, 10, newf)
1082
        r = self.client.object_get(obj)
1083
        self.assertTrue(r.text.startswith('ello!'))
1084

    
1085
        """Truncate livetest update,
1086
            content_range, content_type, object_bytes and source_object"""
1087
        r = self.client.truncate_object(obj, 5)
1088
        r = self.client.object_get(obj)
1089
        self.assertEqual(r.text, 'ello!')
1090

    
1091
        """Check metadata"""
1092
        self.client.set_object_meta(obj, {'mkey2': 'mval2a', 'mkey3': 'mval3'})
1093
        r = self.client.get_object_meta(obj)
1094
        self.assertEqual(r['x-object-meta-mkey1'], 'mval1')
1095
        self.assertEqual(r['x-object-meta-mkey2'], 'mval2a')
1096
        self.assertEqual(r['x-object-meta-mkey3'], 'mval3')
1097
        self.client.del_object_meta(obj, 'mkey1')
1098
        r = self.client.get_object_meta(obj)
1099
        self.assertFalse('x-object-meta-mkey1' in r)
1100

    
1101
        """Check permissions"""
1102
        self.client.set_object_sharing(
1103
            obj,
1104
            read_permition=['u4', 'u5'],
1105
            write_permition=['u4'])
1106
        r = self.client.get_object_sharing(obj)
1107
        self.assertTrue('read' in r)
1108
        self.assertTrue('u5' in r['read'])
1109
        self.assertTrue('write' in r)
1110
        self.assertTrue('u4' in r['write'])
1111
        self.client.del_object_sharing(obj)
1112
        r = self.client.get_object_sharing(obj)
1113
        self.assertTrue(len(r) == 0)
1114

    
1115
        """Check publish"""
1116
        self.client.publish_object(obj)
1117
        r = self.client.get_object_info(obj)
1118
        self.assertTrue('x-object-public' in r)
1119
        self.client.unpublish_object(obj)
1120
        r = self.client.get_object_info(obj)
1121
        self.assertFalse('x-object-public' in r)
1122

    
1123
        """Check if_etag_(not)match"""
1124
        etag = r['etag']
1125
        """
1126
        r = self.client.object_post(
1127
            obj,
1128
            update=True,
1129
            public=True,
1130
            if_etag_not_match=etag,
1131
            success=(412, 202, 204))
1132
        self.assertEqual(r.status_code, 412)
1133
        """
1134

    
1135
        r = self.client.object_post(
1136
            obj,
1137
            update=True,
1138
            public=True,
1139
            if_etag_match=etag,
1140
            content_encoding='application/json')
1141

    
1142
        r = self.client.get_object_info(obj)
1143
        helloVersion = r['x-object-version']
1144
        self.assertTrue('x-object-public' in r)
1145
        self.assertEqual(r['content-encoding'], 'application/json')
1146

    
1147
        """Check source_version and source_account and content_disposition"""
1148
        r = self.client.object_post(
1149
            obj,
1150
            update=True,
1151
            content_type='application/octet-srteam',
1152
            content_length=5,
1153
            content_range='bytes 1-5/*',
1154
            source_object='/%s/%s' % (self.c2, obj),
1155
            source_account='thisAccountWillNeverExist@adminland.com',
1156
            source_version=helloVersion,
1157
            data='12345',
1158
            success=(403, 202, 204))
1159
        self.assertEqual(r.status_code, 403)
1160

    
1161
        r = self.client.object_post(
1162
            obj,
1163
            update=True,
1164
            content_type='application/octet-srteam',
1165
            content_length=5,
1166
            content_range='bytes 1-5/*',
1167
            source_object='/%s/%s' % (self.c2, obj),
1168
            source_account=self.client.account,
1169
            source_version=helloVersion,
1170
            data='12345',
1171
            content_disposition='attachment; filename="fname.ext"')
1172

    
1173
        r = self.client.object_get(obj)
1174
        self.assertEqual(r.text, 'eello!')
1175
        self.assertTrue('content-disposition' in r.headers)
1176
        self.assertTrue('fname.ext' in r.headers['content-disposition'])
1177

    
1178
        """Check manifest"""
1179
        mobj = 'manifest.test'
1180
        txt = ''
1181
        for i in range(10):
1182
            txt += '%s' % i
1183
            r = self.client.object_put(
1184
                '%s/%s' % (mobj, i),
1185
                data='%s' % i,
1186
                content_length=1,
1187
                success=201,
1188
                content_encoding='application/octet-stream',
1189
                content_type='application/octet-stream')
1190

    
1191
        self.client.create_object_by_manifestation(
1192
            mobj,
1193
            content_type='application/octet-stream')
1194

    
1195
        r = self.client.object_post(
1196
            mobj,
1197
            manifest='%s/%s' % (self.client.container, mobj))
1198

    
1199
        r = self.client.object_get(mobj)
1200
        self.assertEqual(r.text, txt)
1201

    
1202
        """We need to check transfer_encoding """
1203

    
1204
    def test_object_delete(self):
1205
        """Test object_DELETE"""
1206
        self._test_0140_object_delete()
1207

    
1208
    def _test_0140_object_delete(self):
1209
        self.client.container = self.c2
1210
        obj = 'test2'
1211
        """create a file on container"""
1212
        r = self.client.object_put(
1213
            obj,
1214
            content_type='application/octet-stream',
1215
            data='H',
1216
            metadata=dict(mkey1='mval1', mkey2='mval2'),
1217
            permissions=dict(
1218
                read=['accX:groupA', 'u1', 'u2'],
1219
                write=['u2', 'u3']))
1220

    
1221
        """Check with false until"""
1222
        r = self.client.object_delete(obj, until=1000000)
1223

    
1224
        r = self.client.object_get(obj, success=(200, 404))
1225
        self.assertEqual(r.status_code, 200)
1226

    
1227
        """Check normal case"""
1228
        r = self.client.object_delete(obj)
1229
        self.assertEqual(r.status_code, 204)
1230

    
1231
        r = self.client.object_get(obj, success=(200, 404))
1232
        self.assertEqual(r.status_code, 404)
1233

    
1234
    def create_large_file(self, size):
1235
        """Create a large file at fs"""
1236
        self.files.append(NamedTemporaryFile())
1237
        f = self.files[-1]
1238
        Ki = size / 8
1239
        bytelist = [b * Ki for b in range(size / Ki)]
1240

    
1241
        def append2file(step):
1242
            f.seek(step)
1243
            f.write(urandom(Ki))
1244
            f.flush()
1245
        self.do_with_progress_bar(
1246
            append2file,
1247
            ' create rand file %s (%sB): ' % (f.name, size),
1248
            bytelist)
1249
        f.seek(0)
1250
        return f