Statistics
| Branch: | Tag: | Revision:

root / kamaki / clients / test / pithos.py @ c1004a00

History | View | Annotate | Download (43.7 kB)

1
# Copyright 2012-2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
import time
35
import datetime
36
from os import urandom
37
from tempfile import NamedTemporaryFile
38

    
39
from kamaki.clients import livetest, ClientError
40
from kamaki.clients.pithos import PithosClient
41
from kamaki.clients.astakos import AstakosClient
42

    
43

    
44
class Pithos(livetest.Generic):
45

    
46
    files = []
47

    
48
    def setUp(self):
49
        self.client = PithosClient(
50
            self['store', 'url'],
51
            self['store', 'token'],
52
            AstakosClient(
53
                self['astakos', 'url'],
54
                self['store', 'token']
55
            ).term('uuid'))
56

    
57
        self.now = time.mktime(time.gmtime())
58
        self.now_unformated = datetime.datetime.utcnow()
59
        self._init_data()
60

    
61
        """Prepare an object to be shared - also its container"""
62
        self.client.container = self.c1
63
        self.client.object_post(
64
            'test',
65
            update=True,
66
            permissions={'read': [self.client.account]})
67

    
68
        self.create_remote_object(self.c1, 'another.test')
69

    
70
    def _init_data(self):
71
        self.c1 = 'c1_' + unicode(self.now)
72
        self.c2 = 'c2_' + unicode(self.now)
73
        self.c3 = 'c3_' + unicode(self.now)
74
        try:
75
            self.client.create_container(self.c2)
76
        except ClientError:
77
            pass
78
        try:
79
            self.client.create_container(self.c1)
80
        except ClientError:
81
            pass
82
        try:
83
            self.client.create_container(self.c3)
84
        except ClientError:
85
            pass
86

    
87
        self.create_remote_object(self.c1, 'test')
88
        self.create_remote_object(self.c2, 'test')
89
        self.create_remote_object(self.c1, 'test1')
90
        self.create_remote_object(self.c2, 'test1')
91

    
92
    def create_remote_object(self, container, obj):
93
        self.client.container = container
94
        self.client.object_put(
95
            obj,
96
            content_type='application/octet-stream',
97
            data='file %s that lives in %s' % (obj, container),
98
            metadata={'incontainer': container})
99

    
100
    def forceDeleteContainer(self, container):
101
        self.client.container = container
102
        try:
103
            r = self.client.list_objects()
104
        except ClientError:
105
            return
106
        for obj in r:
107
            name = obj['name']
108
            self.client.del_object(name)
109
        r = self.client.container_delete()
110
        self.container = ''
111

    
112
    def tearDown(self):
113
        """Destroy test cases"""
114
        for f in self.files:
115
            f.close()
116
        self.forceDeleteContainer(self.c1)
117
        self.forceDeleteContainer(self.c2)
118
        try:
119
            self.forceDeleteContainer(self.c3)
120
        except ClientError:
121
            pass
122
        self.client.container = ''
123

    
124
    def test_000(self):
125
        """Prepare a full Pithos+ test"""
126
        print('')
127
        super(self.__class__, self).test_000()
128

    
129
    def test_account_head(self):
130
        """Test account_HEAD"""
131
        self._test_0010_account_head()
132

    
133
    def _test_0010_account_head(self):
134
        r = self.client.account_head()
135
        self.assertEqual(r.status_code, 204)
136

    
137
        r = self.client.account_head(until='1000000000')
138
        self.assertEqual(r.status_code, 204)
139

    
140
        r = self.client.get_account_info(until='1000000000')
141
        datestring = unicode(r['x-account-until-timestamp'])
142
        self.assertEqual(u'Sun, 09 Sep 2001 01:46:40 GMT', datestring)
143

    
144
        r = self.client.get_account_quota()
145
        self.assertTrue('x-account-policy-quota' in r)
146

    
147
        r = self.client.get_account_versioning()
148
        self.assertTrue('x-account-policy-versioning' in r)
149

    
150
        """Check if(un)modified_since"""
151
        for format in self.client.DATE_FORMATS:
152
            now_formated = self.now_unformated.strftime(format)
153
            r1 = self.client.account_head(
154
                if_modified_since=now_formated,
155
                success=(204, 304, 412))
156
            sc1 = r1.status_code
157
            r1.release()
158
            r2 = self.client.account_head(
159
                if_unmodified_since=now_formated,
160
                success=(204, 304, 412))
161
            sc2 = r2.status_code
162
            r2.release()
163
            self.assertNotEqual(sc1, sc2)
164

    
165
    def test_account_get(self):
166
        """Test account_GET"""
167
        self._test_0020_account_get()
168

    
169
    def _test_0020_account_get(self):
170
        #r = self.client.account_get()
171
        #self.assertEqual(r.status_code, 200)
172
        r = self.client.list_containers()
173
        fullLen = len(r)
174
        self.assertTrue(fullLen > 2)
175

    
176
        r = self.client.account_get(limit=1)
177
        self.assertEqual(len(r.json), 1)
178

    
179
        r = self.client.account_get(marker='c2_')
180
        temp_c0 = r.json[0]['name']
181
        temp_c2 = r.json[2]['name']
182

    
183
        r = self.client.account_get(limit=2, marker='c2_')
184
        conames = [container['name'] for container in r.json if (
185
            container['name'].lower().startswith('c2_'))]
186
        self.assertTrue(temp_c0 in conames)
187
        self.assertFalse(temp_c2 in conames)
188

    
189
        r = self.client.account_get(show_only_shared=True)
190
        self.assertTrue(self.c1 in [c['name'] for c in r.json])
191

    
192
        r = self.client.account_get(until=1342609206)
193
        self.assertTrue(len(r.json) <= fullLen)
194

    
195
        """Check if(un)modified_since"""
196
        for format in self.client.DATE_FORMATS:
197
            now_formated = self.now_unformated.strftime(format)
198
            r1 = self.client.account_get(
199
                if_modified_since=now_formated,
200
                success=(200, 304, 412))
201
            sc1 = r1.status_code
202
            r1.release()
203
            r2 = self.client.account_get(
204
                if_unmodified_since=now_formated,
205
                success=(200, 304, 412))
206
            sc2 = r2.status_code
207
            r2.release()
208
            self.assertNotEqual(sc1, sc2)
209

    
210
        """Check sharing_accounts"""
211
        r = self.client.get_sharing_accounts()
212
        self.assertTrue(len(r) > 0)
213

    
214
    def test_account_post(self):
215
        """Test account_POST"""
216
        self._test_0030_account_post()
217

    
218
    def _test_0030_account_post(self):
219
        r = self.client.account_post()
220
        self.assertEqual(r.status_code, 202)
221
        grpName = 'grp' + unicode(self.now)
222

    
223
        """Method set/del_account_meta and set_account_groupcall use
224
            account_post internally
225
        """
226
        u1 = self.client.account
227
        u2 = self.client.account
228
        self.client.set_account_group(grpName, [u1, u2])
229
        r = self.client.get_account_group()
230
        self.assertEqual(r['x-account-group-' + grpName], '%s,%s' % (u1, u2))
231
        self.client.del_account_group(grpName)
232
        r = self.client.get_account_group()
233
        self.assertTrue('x-account-group-' + grpName not in r)
234

    
235
        mprefix = 'meta' + unicode(self.now)
236
        self.client.set_account_meta({
237
            mprefix + '1': 'v1',
238
            mprefix + '2': 'v2'})
239
        r = self.client.get_account_meta()
240
        self.assertEqual(r['x-account-meta-' + mprefix + '1'], 'v1')
241
        self.assertEqual(r['x-account-meta-' + mprefix + '2'], 'v2')
242

    
243
        self.client.del_account_meta(mprefix + '1')
244
        r = self.client.get_account_meta()
245
        self.assertTrue('x-account-meta-' + mprefix + '1' not in r)
246

    
247
        self.client.del_account_meta(mprefix + '2')
248
        r = self.client.get_account_meta()
249
        self.assertTrue('x-account-meta-' + mprefix + '2' not in r)
250

    
251
        """Missing testing for quota, versioning, because normally
252
        you don't have permissions to modify those at account level
253
        """
254

    
255
        newquota = 1000000
256
        self.client.set_account_quota(newquota)
257
        #r = self.client.get_account_info()
258
        #print(unicode(r))
259
        #r = self.client.get_account_quota()
260
        #self.assertEqual(r['x-account-policy-quota'], newquota)
261
        self.client.set_account_versioning('auto')
262

    
263
    def test_container_head(self):
264
        """Test container_HEAD"""
265
        self._test_0040_container_head()
266

    
267
    def _test_0040_container_head(self):
268
        self.client.container = self.c1
269

    
270
        r = self.client.container_head()
271
        self.assertEqual(r.status_code, 204)
272

    
273
        """Check until"""
274
        r = self.client.container_head(until=1000000, success=(204, 404))
275
        self.assertEqual(r.status_code, 404)
276

    
277
        """Check and if(un)modified_since"""
278
        for format in self.client.DATE_FORMATS:
279
            now_formated = self.now_unformated.strftime(format)
280
            r1 = self.client.container_head(
281
                if_modified_since=now_formated,
282
                success=(204, 304, 412))
283
            sc1 = r1.status_code
284
            r1.release()
285
            r2 = self.client.container_head(
286
                if_unmodified_since=now_formated,
287
                success=(204, 304, 412))
288
            sc2 = r2.status_code
289
            r2.release()
290
            self.assertNotEqual(sc1, sc2)
291

    
292
        """Check container object meta"""
293
        r = self.client.get_container_object_meta()
294
        self.assertEqual(r['x-container-object-meta'], 'Incontainer')
295

    
296
    def test_container_get(self):
297
        """Test container_GET"""
298
        self._test_0050_container_get()
299

    
300
    def _test_0050_container_get(self):
301
        self.client.container = self.c1
302

    
303
        r = self.client.container_get()
304
        self.assertEqual(r.status_code, 200)
305
        fullLen = len(r.json)
306

    
307
        r = self.client.container_get(prefix='test')
308
        lalobjects = [obj for obj in r.json if obj['name'].startswith('test')]
309
        self.assertTrue(len(r.json) > 1)
310
        self.assertEqual(len(r.json), len(lalobjects))
311

    
312
        r = self.client.container_get(limit=1)
313
        self.assertEqual(len(r.json), 1)
314

    
315
        r = self.client.container_get(marker='another')
316
        self.assertTrue(len(r.json) > 1)
317
        neobjects = [obj for obj in r.json if obj['name'] > 'another']
318
        self.assertEqual(len(r.json), len(neobjects))
319

    
320
        r = self.client.container_get(prefix='another.test', delimiter='.')
321
        self.assertTrue(fullLen > len(r.json))
322

    
323
        r = self.client.container_get(path='/')
324
        self.assertEqual(fullLen, len(r.json))
325

    
326
        r = self.client.container_get(format='xml')
327
        self.assertEqual(r.text.split()[4], 'name="' + self.c1 + '">')
328

    
329
        r = self.client.container_get(meta=['incontainer'])
330
        self.assertTrue(len(r.json) > 0)
331

    
332
        r = self.client.container_get(show_only_shared=True)
333
        self.assertTrue(len(r.json) < fullLen)
334

    
335
        try:
336
            r = self.client.container_get(until=1000000000)
337
            datestring = unicode(r.headers['x-account-until-timestamp'])
338
            self.assertEqual(u'Sun, 09 Sep 2001 01:46:40 GMT', datestring)
339

    
340
        except ClientError:
341

    
342
            pass
343

    
344
        """Check and if un/modified_since"""
345
        for format in self.client.DATE_FORMATS:
346
            now_formated = self.now_unformated.strftime(format)
347
            r1 = self.client.container_get(
348
                if_modified_since=now_formated,
349
                success=(200, 304, 412))
350
            sc1 = r1.status_code
351
            r1.release()
352
            r2 = self.client.container_get(
353
                if_unmodified_since=now_formated,
354
                success=(200, 304, 412))
355
            sc2 = r2.status_code
356
            r2.release()
357
            self.assertNotEqual(sc1, sc2)
358

    
359
    def test_container_put(self):
360
        """Test container_PUT"""
361
        self._test_0050_container_put()
362

    
363
    def _test_0050_container_put(self):
364
        self.client.container = self.c2
365

    
366
        r = self.client.container_put()
367
        self.assertEqual(r.status_code, 202)
368

    
369
        r = self.client.get_container_quota(self.client.container)
370
        cquota = r.values()[0]
371
        newquota = 2 * int(cquota)
372

    
373
        r = self.client.container_put(quota=newquota)
374
        self.assertEqual(r.status_code, 202)
375

    
376
        r = self.client.get_container_quota(self.client.container)
377
        xquota = int(r.values()[0])
378
        self.assertEqual(newquota, xquota)
379

    
380
        r = self.client.container_put(versioning='auto')
381
        self.assertEqual(r.status_code, 202)
382

    
383
        r = self.client.get_container_versioning(self.client.container)
384
        nvers = r.values()[0]
385
        self.assertEqual('auto', nvers)
386

    
387
        r = self.client.container_put(versioning='none')
388
        self.assertEqual(r.status_code, 202)
389

    
390
        r = self.client.get_container_versioning(self.client.container)
391
        nvers = r.values()[0]
392
        self.assertEqual('none', nvers)
393

    
394
        r = self.client.container_put(metadata={'m1': 'v1', 'm2': 'v2'})
395
        self.assertEqual(r.status_code, 202)
396

    
397
        r = self.client.get_container_meta(self.client.container)
398
        self.assertTrue('x-container-meta-m1' in r)
399
        self.assertEqual(r['x-container-meta-m1'], 'v1')
400
        self.assertTrue('x-container-meta-m2' in r)
401
        self.assertEqual(r['x-container-meta-m2'], 'v2')
402

    
403
        r = self.client.container_put(metadata={'m1': '', 'm2': 'v2a'})
404
        self.assertEqual(r.status_code, 202)
405

    
406
        r = self.client.get_container_meta(self.client.container)
407
        self.assertTrue('x-container-meta-m1' not in r)
408
        self.assertTrue('x-container-meta-m2' in r)
409
        self.assertEqual(r['x-container-meta-m2'], 'v2a')
410

    
411
        self.client.del_container_meta(self.client.container)
412

    
413
    def test_container_post(self):
414
        """Test container_POST"""
415
        self._test_0060_container_post()
416

    
417
    def _test_0060_container_post(self):
418
        self.client.container = self.c2
419

    
420
        """Simple post"""
421
        r = self.client.container_post()
422
        self.assertEqual(r.status_code, 202)
423

    
424
        """post meta"""
425
        self.client.set_container_meta({'m1': 'v1', 'm2': 'v2'})
426
        r = self.client.get_container_meta(self.client.container)
427
        self.assertTrue('x-container-meta-m1' in r)
428
        self.assertEqual(r['x-container-meta-m1'], 'v1')
429
        self.assertTrue('x-container-meta-m2' in r)
430
        self.assertEqual(r['x-container-meta-m2'], 'v2')
431

    
432
        """post/2del meta"""
433
        r = self.client.del_container_meta('m1')
434
        r = self.client.set_container_meta({'m2': 'v2a'})
435
        r = self.client.get_container_meta(self.client.container)
436
        self.assertTrue('x-container-meta-m1' not in r)
437
        self.assertTrue('x-container-meta-m2' in r)
438
        self.assertEqual(r['x-container-meta-m2'], 'v2a')
439

    
440
        """check quota"""
441
        r = self.client.get_container_quota(self.client.container)
442
        cquota = r.values()[0]
443
        newquota = 2 * int(cquota)
444
        r = self.client.set_container_quota(newquota)
445
        r = self.client.get_container_quota(self.client.container)
446
        xquota = int(r.values()[0])
447
        self.assertEqual(newquota, xquota)
448
        r = self.client.set_container_quota(cquota)
449
        r = self.client.get_container_quota(self.client.container)
450
        xquota = r.values()[0]
451
        self.assertEqual(cquota, xquota)
452

    
453
        """Check versioning"""
454
        self.client.set_container_versioning('auto')
455
        r = self.client.get_container_versioning(self.client.container)
456
        nvers = r.values()[0]
457
        self.assertEqual('auto', nvers)
458
        self.client.set_container_versioning('none')
459
        r = self.client.get_container_versioning(self.client.container)
460
        nvers = r.values()[0]
461
        self.assertEqual('none', nvers)
462

    
463
        """put_block uses content_type and content_length to
464
        post blocks of data 2 container. All that in upload_object"""
465
        """Change a file at fs"""
466
        f = self.create_large_file(1024 * 1024 * 100)
467
        """Upload it at a directory in container"""
468
        self.client.create_directory('dir')
469
        self.client.upload_object('/dir/sample.file', f)
470
        """Check if file has been uploaded"""
471
        r = self.client.get_object_info('/dir/sample.file')
472
        self.assertTrue(int(r['content-length']) > 100000000)
473

    
474
        """What is tranfer_encoding? What should I check about it? """
475
        #TODO
476

    
477
        """Check update=False"""
478
        r = self.client.object_post(
479
            'test',
480
            update=False,
481
            metadata={'newmeta': 'newval'})
482

    
483
        r = self.client.get_object_info('test')
484
        self.assertTrue('x-object-meta-newmeta' in r)
485
        self.assertFalse('x-object-meta-incontainer' in r)
486

    
487
        r = self.client.del_container_meta('m2')
488

    
489
    def test_container_delete(self):
490
        """Test container_DELETE"""
491
        self._test_0070_container_delete()
492

    
493
    def _test_0070_container_delete(self):
494

    
495
        """Fail to delete a non-empty container"""
496
        self.client.container = self.c2
497
        r = self.client.container_delete(success=409)
498
        self.assertEqual(r.status_code, 409)
499

    
500
        """Fail to delete c3 (empty) container"""
501
        self.client.container = self.c3
502
        r = self.client.container_delete(until='1000000000')
503
        self.assertEqual(r.status_code, 204)
504

    
505
        """Delete c3 (empty) container"""
506
        r = self.client.container_delete()
507
        self.assertEqual(r.status_code, 204)
508

    
509
        """Purge container(empty a container), check versionlist"""
510
        self.client.container = self.c1
511
        r = self.client.object_head('test', success=(200, 404))
512
        self.assertEqual(r.status_code, 200)
513
        self.client.del_container(delimiter='/')
514
        r = self.client.object_head('test', success=(200, 404))
515
        self.assertEqual(r.status_code, 404)
516
        r = self.client.get_object_versionlist('test')
517
        self.assertTrue(len(r) > 0)
518
        self.assertTrue(len(r[0]) > 1)
519
        self.client.purge_container()
520
        self.assertRaises(
521
            ClientError,
522
            self.client.get_object_versionlist,
523
            'test')
524

    
525
    def _test_0080_recreate_deleted_data(self):
526
        self._init_data()
527

    
528
    def test_object_head(self):
529
        """Test object_HEAD"""
530
        self._test_0090_object_head()
531

    
532
    def _test_0090_object_head(self):
533
        self.client.container = self.c2
534
        obj = 'test'
535

    
536
        r = self.client.object_head(obj)
537
        self.assertEqual(r.status_code, 200)
538
        etag = r.headers['etag']
539

    
540
        r = self.client.object_head(obj, version=40)
541
        self.assertEqual(r.headers['x-object-version'], '40')
542

    
543
        r = self.client.object_head(obj, if_etag_match=etag)
544
        self.assertEqual(r.status_code, 200)
545

    
546
        r = self.client.object_head(
547
            obj,
548
            if_etag_not_match=etag,
549
            success=(200, 412, 304))
550
        self.assertNotEqual(r.status_code, 200)
551

    
552
        r = self.client.object_head(
553
            obj,
554
            version=40,
555
            if_etag_match=etag,
556
            success=412)
557
        self.assertEqual(r.status_code, 412)
558

    
559
        """Check and if(un)modified_since"""
560
        for format in self.client.DATE_FORMATS:
561
            now_formated = self.now_unformated.strftime(format)
562
            r1 = self.client.object_head(
563
                obj,
564
                if_modified_since=now_formated,
565
                success=(200, 304, 412))
566
            sc1 = r1.status_code
567
            r1.release()
568
            r2 = self.client.object_head(
569
                obj,
570
                if_unmodified_since=now_formated,
571
                success=(200, 304, 412))
572
            sc2 = r2.status_code
573
            r2.release()
574
            self.assertNotEqual(sc1, sc2)
575

    
576
    def test_object_get(self):
577
        """Test object_GET"""
578
        self._test_0100_object_get()
579

    
580
    def _test_0100_object_get(self):
581
        self.client.container = self.c1
582
        obj = 'test'
583

    
584
        r = self.client.object_get(obj)
585
        self.assertEqual(r.status_code, 200)
586

    
587
        osize = int(r.headers['content-length'])
588
        etag = r.headers['etag']
589

    
590
        r = self.client.object_get(obj, hashmap=True)
591
        for term in ('hashes', 'block_hash', 'block_hash', 'bytes'):
592
            self.assertTrue(term in r.json)
593

    
594
        r = self.client.object_get(obj, format='xml', hashmap=True)
595
        self.assertEqual(len(r.text.split('hash>')), 3)
596

    
597
        rangestr = 'bytes=%s-%s' % (osize / 3, osize / 2)
598
        r = self.client.object_get(
599
            obj,
600
            data_range=rangestr,
601
            success=(200, 206))
602
        partsize = int(r.headers['content-length'])
603
        self.assertTrue(0 < partsize and partsize <= 1 + osize / 3)
604

    
605
        rangestr = 'bytes=%s-%s' % (osize / 3, osize / 2)
606
        r = self.client.object_get(
607
            obj,
608
            data_range=rangestr,
609
            if_range=True,
610
            success=(200, 206))
611
        partsize = int(r.headers['content-length'])
612
        self.assertTrue(0 < partsize and partsize <= 1 + osize / 3)
613

    
614
        r = self.client.object_get(obj, if_etag_match=etag)
615
        self.assertEqual(r.status_code, 200)
616

    
617
        r = self.client.object_get(obj, if_etag_not_match=etag + 'LALALA')
618
        self.assertEqual(r.status_code, 200)
619

    
620
        """Check and if(un)modified_since"""
621
        for format in self.client.DATE_FORMATS:
622
            now_formated = self.now_unformated.strftime(format)
623
            r1 = self.client.object_get(
624
                obj,
625
                if_modified_since=now_formated,
626
                success=(200, 304, 412))
627
            sc1 = r1.status_code
628
            r1.release()
629
            r2 = self.client.object_get(
630
                obj,
631
                if_unmodified_since=now_formated,
632
                success=(200, 304, 412))
633
            sc2 = r2.status_code
634
            r2.release()
635
            self.assertNotEqual(sc1, sc2)
636

    
637
        """Upload an object to download"""
638
        trg_fname = 'remotefile_%s' % self.now
639
        f_size = 59247824
640
        src_f = self.create_large_file(f_size)
641
        print('\tUploading...')
642
        self.client.upload_object(trg_fname, src_f)
643
        print('\tDownloading...')
644
        self.files.append(NamedTemporaryFile())
645
        dnl_f = self.files[-1]
646
        self.client.download_object(trg_fname, dnl_f)
647

    
648
        print('\tCheck if files match...')
649
        for pos in (0, f_size / 2, f_size - 20):
650
            src_f.seek(pos)
651
            dnl_f.seek(pos)
652
            self.assertEqual(src_f.read(10), dnl_f.read(10))
653

    
654
    def test_object_put(self):
655
        """Test object_PUT"""
656
        self._test_object_put()
657

    
658
    def _test_object_put(self):
659
        self.client.container = self.c2
660
        obj = 'another.test'
661

    
662
        self.client.create_object(obj + '.FAKE')
663
        r = self.client.get_object_info(obj + '.FAKE')
664
        self.assertEqual(r['content-type'], 'application/octet-stream')
665

    
666
        """create the object"""
667
        r = self.client.object_put(
668
            obj,
669
            data='a',
670
            content_type='application/octer-stream',
671
            permissions=dict(
672
                read=['accX:groupA', 'u1', 'u2'],
673
                write=['u2', 'u3']),
674
            metadata=dict(key1='val1', key2='val2'),
675
            content_encoding='UTF-8',
676
            content_disposition='attachment; filename="fname.ext"')
677
        self.assertEqual(r.status_code, 201)
678
        etag = r.headers['etag']
679

    
680
        """Check content-disposition"""
681
        r = self.client.get_object_info(obj)
682
        self.assertTrue('content-disposition' in r)
683

    
684
        """Check permissions"""
685
        r = self.client.get_object_sharing(obj)
686
        self.assertTrue('accx:groupa' in r['read'])
687
        self.assertTrue('u1' in r['read'])
688
        self.assertTrue('u2' in r['write'])
689
        self.assertTrue('u3' in r['write'])
690

    
691
        """Check metadata"""
692
        r = self.client.get_object_meta(obj)
693
        self.assertEqual(r['x-object-meta-key1'], 'val1')
694
        self.assertEqual(r['x-object-meta-key2'], 'val2')
695

    
696
        """Check public and if_etag_match"""
697
        r = self.client.object_put(
698
            obj,
699
            if_etag_match=etag,
700
            data='b',
701
            content_type='application/octet-stream',
702
            public=True)
703

    
704
        r = self.client.object_get(obj)
705
        self.assertTrue('x-object-public' in r.headers)
706
        vers2 = int(r.headers['x-object-version'])
707
        etag = r.headers['etag']
708
        self.assertEqual(r.text, 'b')
709

    
710
        """Check if_etag_not_match"""
711
        r = self.client.object_put(
712
            obj,
713
            if_etag_not_match=etag,
714
            data='c',
715
            content_type='application/octet-stream',
716
            success=(201, 412))
717
        self.assertEqual(r.status_code, 412)
718

    
719
        """Check content_type and content_length"""
720
        tmpdir = 'dir' + unicode(self.now)
721
        r = self.client.object_put(
722
            tmpdir,
723
            content_type='application/directory',
724
            content_length=0)
725

    
726
        r = self.client.get_object_info(tmpdir)
727
        self.assertEqual(r['content-type'], 'application/directory')
728

    
729
        """Check copy_from, content_encoding"""
730
        r = self.client.object_put(
731
            '%s/%s' % (tmpdir, obj),
732
            format=None,
733
            copy_from='/%s/%s' % (self.client.container, obj),
734
            content_encoding='application/octet-stream',
735
            source_account=self.client.account,
736
            content_length=0,
737
            success=201)
738
        self.assertEqual(r.status_code, 201)
739

    
740
        """Test copy_object for cross-conctainer copy"""
741
        self.client.copy_object(
742
            src_container=self.c2,
743
            src_object='%s/%s' % (tmpdir, obj),
744
            dst_container=self.c1,
745
            dst_object=obj)
746
        self.client.container = self.c1
747
        r1 = self.client.get_object_info(obj)
748
        self.client.container = self.c2
749
        r2 = self.client.get_object_info('%s/%s' % (tmpdir, obj))
750
        self.assertEqual(r1['x-object-hash'], r2['x-object-hash'])
751

    
752
        """Check cross-container copy_from, content_encoding"""
753
        self.client.container = self.c1
754
        fromstr = '/%s/%s/%s' % (self.c2, tmpdir, obj)
755
        r = self.client.object_put(
756
            obj,
757
            format=None,
758
            copy_from=fromstr,
759
            content_encoding='application/octet-stream',
760
            source_account=self.client.account,
761
            content_length=0,
762
            success=201)
763

    
764
        self.assertEqual(r.status_code, 201)
765
        r = self.client.get_object_info(obj)
766
        self.assertEqual(r['etag'], etag)
767

    
768
        """Check source_account"""
769
        self.client.container = self.c2
770
        fromstr = '/%s/%s' % (self.c1, obj)
771
        r = self.client.object_put(
772
            '%sv2' % obj,
773
            format=None,
774
            move_from=fromstr,
775
            content_encoding='application/octet-stream',
776
            source_account='nonExistendAddress@NeverLand.com',
777
            content_length=0,
778
            success=(201, 403))
779
        self.assertEqual(r.status_code, 403)
780

    
781
        """Check cross-container move_from"""
782
        self.client.container = self.c1
783
        r1 = self.client.get_object_info(obj)
784
        self.client.container = self.c2
785
        self.client.move_object(
786
            src_container=self.c1,
787
            src_object=obj,
788
            dst_container=self.c2,
789
            dst_object=obj + 'v0')
790
        r0 = self.client.get_object_info(obj + 'v0')
791
        self.assertEqual(r1['x-object-hash'], r0['x-object-hash'])
792

    
793
        """Check move_from"""
794
        r = self.client.object_put(
795
            '%sv1' % obj,
796
            format=None,
797
            move_from='/%s/%s' % (self.c2, obj),
798
            source_version=vers2,
799
            content_encoding='application/octet-stream',
800
            content_length=0, success=201)
801

    
802
        """Check manifest"""
803
        mobj = 'manifest.test'
804
        txt = ''
805
        for i in range(10):
806
            txt += '%s' % i
807
            r = self.client.object_put(
808
                '%s/%s' % (mobj, i),
809
                data='%s' % i,
810
                content_length=1,
811
                success=201,
812
                content_type='application/octet-stream',
813
                content_encoding='application/octet-stream')
814

    
815
        r = self.client.object_put(
816
            mobj,
817
            content_length=0,
818
            content_type='application/octet-stream',
819
            manifest='%s/%s' % (self.client.container, mobj))
820

    
821
        r = self.client.object_get(mobj)
822
        self.assertEqual(r.text, txt)
823

    
824
        """Upload a local file with one request"""
825
        newf = self.create_large_file(1024 * 10)
826
        self.client.upload_object('sample.file', newf)
827
        """Check if file has been uploaded"""
828
        r = self.client.get_object_info('sample.file')
829
        self.assertEqual(int(r['content-length']), 10240)
830

    
831
        """Some problems with transfer-encoding?"""
832

    
833
    def test_object_copy(self):
834
        """Test object_COPY"""
835
        self._test_0110_object_copy()
836

    
837
    def _test_0110_object_copy(self):
838
        #  TODO: check with source_account option
839
        self.client.container = self.c2
840
        obj = 'test2'
841

    
842
        data = '{"key1":"val1", "key2":"val2"}'
843
        r = self.client.object_put(
844
            '%sorig' % obj,
845
            content_type='application/octet-stream',
846
            data=data,
847
            metadata=dict(mkey1='mval1', mkey2='mval2'),
848
            permissions=dict(
849
                read=['accX:groupA', 'u1', 'u2'],
850
                write=['u2', 'u3']),
851
            content_disposition='attachment; filename="fname.ext"')
852

    
853
        r = self.client.object_copy(
854
            '%sorig' % obj,
855
            destination='/%s/%s' % (self.client.container, obj),
856
            ignore_content_type=False, content_type='application/json',
857
            metadata={'mkey2': 'mval2a', 'mkey3': 'mval3'},
858
            permissions={'write': ['u5', 'accX:groupB']})
859
        self.assertEqual(r.status_code, 201)
860

    
861
        """Check content-disposition"""
862
        r = self.client.get_object_info(obj)
863
        self.assertTrue('content-disposition' in r)
864

    
865
        """Check Metadata"""
866
        r = self.client.get_object_meta(obj)
867
        self.assertEqual(r['x-object-meta-mkey1'], 'mval1')
868
        self.assertEqual(r['x-object-meta-mkey2'], 'mval2a')
869
        self.assertEqual(r['x-object-meta-mkey3'], 'mval3')
870

    
871
        """Check permissions"""
872
        r = self.client.get_object_sharing(obj)
873
        self.assertFalse('read' in r or 'u2' in r['write'])
874
        self.assertTrue('accx:groupb' in r['write'])
875

    
876
        """Check destination account"""
877
        r = self.client.object_copy(
878
            obj,
879
            destination='/%s/%s' % (self.c1, obj),
880
            content_encoding='utf8',
881
            content_type='application/json',
882
            destination_account='nonExistendAddress@NeverLand.com',
883
            success=(201, 403))
884
        self.assertEqual(r.status_code, 403)
885

    
886
        """Check destination being another container
887
        and also content_type and content encoding"""
888
        r = self.client.object_copy(
889
            obj,
890
            destination='/%s/%s' % (self.c1, obj),
891
            content_encoding='utf8',
892
            content_type='application/json')
893
        self.assertEqual(r.status_code, 201)
894
        self.assertEqual(
895
            r.headers['content-type'],
896
            'application/json; charset=UTF-8')
897

    
898
        """Check ignore_content_type and content_type"""
899
        r = self.client.object_get(obj)
900
        etag = r.headers['etag']
901
        ctype = r.headers['content-type']
902
        self.assertEqual(ctype, 'application/json')
903

    
904
        r = self.client.object_copy(
905
            '%sorig' % obj,
906
            destination='/%s/%s0' % (self.client.container, obj),
907
            ignore_content_type=True,
908
            content_type='application/json')
909
        self.assertEqual(r.status_code, 201)
910
        self.assertNotEqual(r.headers['content-type'], 'application/json')
911

    
912
        """Check if_etag_(not_)match"""
913
        r = self.client.object_copy(
914
            obj,
915
            destination='/%s/%s1' % (self.client.container, obj),
916
            if_etag_match=etag)
917
        self.assertEqual(r.status_code, 201)
918

    
919
        r = self.client.object_copy(
920
            obj,
921
            destination='/%s/%s2' % (self.client.container, obj),
922
            if_etag_not_match='lalala')
923
        self.assertEqual(r.status_code, 201)
924
        vers2 = r.headers['x-object-version']
925

    
926
        """Check source_version, public and format """
927
        r = self.client.object_copy(
928
            '%s2' % obj,
929
            destination='/%s/%s3' % (self.client.container, obj),
930
            source_version=vers2,
931
            format='xml',
932
            public=True)
933
        self.assertEqual(r.status_code, 201)
934
        self.assertTrue(r.headers['content-type'].index('xml') > 0)
935

    
936
        r = self.client.get_object_info(obj + '3')
937
        self.assertTrue('x-object-public' in r)
938

    
939
    def test_object_move(self):
940
        """Test object_MOVE"""
941
        self._test_0120_object_move()
942

    
943
    def _test_0120_object_move(self):
944
        self.client.container = self.c2
945
        obj = 'test2'
946

    
947
        data = '{"key1": "val1", "key2": "val2"}'
948
        r = self.client.object_put(
949
            '%sorig' % obj,
950
            content_type='application/octet-stream',
951
            data=data,
952
            metadata=dict(mkey1='mval1', mkey2='mval2'),
953
            permissions=dict(
954
                read=['accX:groupA', 'u1', 'u2'],
955
                write=['u2', 'u3']))
956

    
957
        r = self.client.object_move(
958
            '%sorig' % obj,
959
            destination='/%s/%s' % (self.client.container, obj),
960
            ignore_content_type=False,
961
            content_type='application/json',
962
            metadata=dict(mkey2='mval2a', mkey3='mval3'),
963
            permissions=dict(write=['u5', 'accX:groupB']))
964
        self.assertEqual(r.status_code, 201)
965

    
966
        """Check Metadata"""
967
        r = self.client.get_object_meta(obj)
968
        self.assertEqual(r['x-object-meta-mkey1'], 'mval1')
969
        self.assertEqual(r['x-object-meta-mkey2'], 'mval2a')
970
        self.assertEqual(r['x-object-meta-mkey3'], 'mval3')
971

    
972
        """Check permissions"""
973
        r = self.client.get_object_sharing(obj)
974
        self.assertFalse('read' in r)
975
        self.assertTrue('u5' in r['write'])
976
        self.assertTrue('accx:groupb' in r['write'])
977

    
978
        """Check destination account"""
979
        r = self.client.object_move(
980
            obj,
981
            destination='/%s/%s' % (self.c1, obj),
982
            content_encoding='utf8',
983
            content_type='application/json',
984
            destination_account='nonExistendAddress@NeverLand.com',
985
            success=(201, 403))
986
        self.assertEqual(r.status_code, 403)
987

    
988
        """Check destination being another container and also
989
        content_type, content_disposition and content encoding"""
990
        r = self.client.object_move(
991
            obj,
992
            destination='/%s/%s' % (self.c1, obj),
993
            content_encoding='utf8',
994
            content_type='application/json',
995
            content_disposition='attachment; filename="fname.ext"')
996
        self.assertEqual(r.status_code, 201)
997
        self.assertEqual(
998
            r.headers['content-type'],
999
            'application/json; charset=UTF-8')
1000
        self.client.container = self.c1
1001
        r = self.client.get_object_info(obj)
1002
        self.assertTrue('content-disposition' in r)
1003
        self.assertTrue('fname.ext' in r['content-disposition'])
1004
        etag = r['etag']
1005
        ctype = r['content-type']
1006
        self.assertEqual(ctype, 'application/json')
1007

    
1008
        """Check ignore_content_type and content_type"""
1009
        r = self.client.object_move(
1010
            obj,
1011
            destination='/%s/%s' % (self.c2, obj),
1012
            ignore_content_type=True,
1013
            content_type='application/json')
1014
        self.assertEqual(r.status_code, 201)
1015
        self.assertNotEqual(r.headers['content-type'], 'application/json')
1016

    
1017
        """Check if_etag_(not_)match"""
1018
        self.client.container = self.c2
1019
        r = self.client.object_move(
1020
            obj,
1021
            destination='/%s/%s0' % (self.client.container, obj),
1022
            if_etag_match=etag)
1023
        self.assertEqual(r.status_code, 201)
1024

    
1025
        r = self.client.object_move(
1026
            '%s0' % obj,
1027
            destination='/%s/%s1' % (self.client.container, obj),
1028
            if_etag_not_match='lalala')
1029
        self.assertEqual(r.status_code, 201)
1030

    
1031
        """Check public and format """
1032
        r = self.client.object_move(
1033
            '%s1' % obj,
1034
            destination='/%s/%s2' % (self.client.container, obj),
1035
            format='xml',
1036
            public=True)
1037
        self.assertEqual(r.status_code, 201)
1038
        self.assertTrue(r.headers['content-type'].index('xml') > 0)
1039

    
1040
        r = self.client.get_object_info(obj + '2')
1041
        self.assertTrue('x-object-public' in r)
1042

    
1043
    def test_object_post(self):
1044
        """Test object_POST"""
1045
        self._test_0130_object_post()
1046

    
1047
    def _test_0130_object_post(self):
1048
        self.client.container = self.c2
1049
        obj = 'test2'
1050
        """create a filesystem file"""
1051
        self.files.append(NamedTemporaryFile())
1052
        newf = self.files[-1]
1053
        newf.writelines([
1054
            'ello!\n',
1055
            'This is a test line\n',
1056
            'inside a test file\n'])
1057
        """create a file on container"""
1058
        r = self.client.object_put(
1059
            obj,
1060
            content_type='application/octet-stream',
1061
            data='H',
1062
            metadata=dict(mkey1='mval1', mkey2='mval2'),
1063
            permissions=dict(
1064
                read=['accX:groupA', 'u1', 'u2'],
1065
                write=['u2', 'u3']))
1066

    
1067
        """Append livetest update, content_[range|type|length]"""
1068
        newf.seek(0)
1069
        self.client.append_object(obj, newf)
1070
        r = self.client.object_get(obj)
1071
        self.assertTrue(r.text.startswith('Hello!'))
1072

    
1073
        """Overwrite livetest update,
1074
            content_type, content_length, content_range
1075
        """
1076
        newf.seek(0)
1077
        r = self.client.overwrite_object(obj, 0, 10, newf)
1078
        r = self.client.object_get(obj)
1079
        self.assertTrue(r.text.startswith('ello!'))
1080

    
1081
        """Truncate livetest update,
1082
            content_range, content_type, object_bytes and source_object"""
1083
        r = self.client.truncate_object(obj, 5)
1084
        r = self.client.object_get(obj)
1085
        self.assertEqual(r.text, 'ello!')
1086

    
1087
        """Check metadata"""
1088
        self.client.set_object_meta(obj, {'mkey2': 'mval2a', 'mkey3': 'mval3'})
1089
        r = self.client.get_object_meta(obj)
1090
        self.assertEqual(r['x-object-meta-mkey1'], 'mval1')
1091
        self.assertEqual(r['x-object-meta-mkey2'], 'mval2a')
1092
        self.assertEqual(r['x-object-meta-mkey3'], 'mval3')
1093
        self.client.del_object_meta(obj, 'mkey1')
1094
        r = self.client.get_object_meta(obj)
1095
        self.assertFalse('x-object-meta-mkey1' in r)
1096

    
1097
        """Check permissions"""
1098
        self.client.set_object_sharing(
1099
            obj,
1100
            read_permition=['u4', 'u5'],
1101
            write_permition=['u4'])
1102
        r = self.client.get_object_sharing(obj)
1103
        self.assertTrue('read' in r)
1104
        self.assertTrue('u5' in r['read'])
1105
        self.assertTrue('write' in r)
1106
        self.assertTrue('u4' in r['write'])
1107
        self.client.del_object_sharing(obj)
1108
        r = self.client.get_object_sharing(obj)
1109
        self.assertTrue(len(r) == 0)
1110

    
1111
        """Check publish"""
1112
        self.client.publish_object(obj)
1113
        r = self.client.get_object_info(obj)
1114
        self.assertTrue('x-object-public' in r)
1115
        self.client.unpublish_object(obj)
1116
        r = self.client.get_object_info(obj)
1117
        self.assertFalse('x-object-public' in r)
1118

    
1119
        """Check if_etag_(not)match"""
1120
        etag = r['etag']
1121
        """
1122
        r = self.client.object_post(
1123
            obj,
1124
            update=True,
1125
            public=True,
1126
            if_etag_not_match=etag,
1127
            success=(412, 202, 204))
1128
        self.assertEqual(r.status_code, 412)
1129
        """
1130

    
1131
        r = self.client.object_post(
1132
            obj,
1133
            update=True,
1134
            public=True,
1135
            if_etag_match=etag,
1136
            content_encoding='application/json')
1137

    
1138
        r = self.client.get_object_info(obj)
1139
        helloVersion = r['x-object-version']
1140
        self.assertTrue('x-object-public' in r)
1141
        self.assertEqual(r['content-encoding'], 'application/json')
1142

    
1143
        """Check source_version and source_account and content_disposition"""
1144
        r = self.client.object_post(
1145
            obj,
1146
            update=True,
1147
            content_type='application/octet-srteam',
1148
            content_length=5,
1149
            content_range='bytes 1-5/*',
1150
            source_object='/%s/%s' % (self.c2, obj),
1151
            source_account='thisAccountWillNeverExist@adminland.com',
1152
            source_version=helloVersion,
1153
            data='12345',
1154
            success=(403, 202, 204))
1155
        self.assertEqual(r.status_code, 403)
1156

    
1157
        r = self.client.object_post(
1158
            obj,
1159
            update=True,
1160
            content_type='application/octet-srteam',
1161
            content_length=5,
1162
            content_range='bytes 1-5/*',
1163
            source_object='/%s/%s' % (self.c2, obj),
1164
            source_account=self.client.account,
1165
            source_version=helloVersion,
1166
            data='12345',
1167
            content_disposition='attachment; filename="fname.ext"')
1168

    
1169
        r = self.client.object_get(obj)
1170
        self.assertEqual(r.text, 'eello!')
1171
        self.assertTrue('content-disposition' in r.headers)
1172
        self.assertTrue('fname.ext' in r.headers['content-disposition'])
1173

    
1174
        """Check manifest"""
1175
        mobj = 'manifest.test'
1176
        txt = ''
1177
        for i in range(10):
1178
            txt += '%s' % i
1179
            r = self.client.object_put(
1180
                '%s/%s' % (mobj, i),
1181
                data='%s' % i,
1182
                content_length=1,
1183
                success=201,
1184
                content_encoding='application/octet-stream',
1185
                content_type='application/octet-stream')
1186

    
1187
        self.client.create_object_by_manifestation(
1188
            mobj,
1189
            content_type='application/octet-stream')
1190

    
1191
        r = self.client.object_post(
1192
            mobj,
1193
            manifest='%s/%s' % (self.client.container, mobj))
1194

    
1195
        r = self.client.object_get(mobj)
1196
        self.assertEqual(r.text, txt)
1197

    
1198
        """We need to check transfer_encoding """
1199

    
1200
    def test_object_delete(self):
1201
        """Test object_DELETE"""
1202
        self._test_0140_object_delete()
1203

    
1204
    def _test_0140_object_delete(self):
1205
        self.client.container = self.c2
1206
        obj = 'test2'
1207
        """create a file on container"""
1208
        r = self.client.object_put(
1209
            obj,
1210
            content_type='application/octet-stream',
1211
            data='H',
1212
            metadata=dict(mkey1='mval1', mkey2='mval2'),
1213
            permissions=dict(
1214
                read=['accX:groupA', 'u1', 'u2'],
1215
                write=['u2', 'u3']))
1216

    
1217
        """Check with false until"""
1218
        r = self.client.object_delete(obj, until=1000000)
1219

    
1220
        r = self.client.object_get(obj, success=(200, 404))
1221
        self.assertEqual(r.status_code, 200)
1222

    
1223
        """Check normal case"""
1224
        r = self.client.object_delete(obj)
1225
        self.assertEqual(r.status_code, 204)
1226

    
1227
        r = self.client.object_get(obj, success=(200, 404))
1228
        self.assertEqual(r.status_code, 404)
1229

    
1230
    def create_large_file(self, size):
1231
        """Create a large file at fs"""
1232
        self.files.append(NamedTemporaryFile())
1233
        f = self.files[-1]
1234
        Ki = size / 8
1235
        bytelist = [b * Ki for b in range(size / Ki)]
1236

    
1237
        def append2file(step):
1238
            f.seek(step)
1239
            f.write(urandom(Ki))
1240
            f.flush()
1241
        self.do_with_progress_bar(
1242
            append2file,
1243
            ' create rand file %s (%sB): ' % (f.name, size),
1244
            bytelist)
1245
        f.seek(0)
1246
        return f