Statistics
| Branch: | Tag: | Revision:

root / kamaki / clients / pithos / test.py @ d15e5249

History | View | Annotate | Download (43.1 kB)

1
# Copyright 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from unittest import TestCase
35
from mock import patch, call
36
from tempfile import NamedTemporaryFile
37
from os import urandom
38

    
39
from kamaki.clients import ClientError
40
from kamaki.clients.pithos import PithosClient as PC
41
from kamaki.clients.connection.kamakicon import KamakiHTTPConnection as C
42

    
43
client_pkg = 'kamaki.clients.Client'
44
pithos_pkg = 'kamaki.clients.pithos.PithosClient'
45

    
46
user_id = 'ac0un7-1d-5tr1ng'
47
obj = 'obj3c7N4m3'
48

    
49
account_info = {
50
    'content-language': 'en-us',
51
    'content-type': 'text/html; charset=utf-8',
52
    'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
53
    'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
54
    'server': 'gunicorn/0.14.5',
55
    'vary': 'Accept-Language',
56
    'x-account-bytes-used': '751615526',
57
    'x-account-container-count': 7,
58
    'x-account-policy-quota': 53687091200,
59
    'x-account-policy-versioning': 'auto'}
60
container_info = {
61
    'content-language': 'en-us',
62
    'content-type': 'text/html; charset=utf-8',
63
    'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
64
    'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
65
    'server': 'gunicorn/0.14.5',
66
    'vary': 'Accept-Language',
67
    'x-container-block-hash': 'sha256',
68
    'x-container-block-size': 4194304,
69
    'x-container-bytes-used': 309528938,
70
    'x-container-object-count': 14,
71
    'x-container-object-meta': '',
72
    'x-container-policy-quota': 53687091200,
73
    'x-container-policy-versioning': 'auto'}
74
object_info = {
75
    'content-language': 'en-us',
76
    'content-length': 254965,
77
    'content-type': 'application/octet-stream',
78
    'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
79
    'etag': '',
80
    'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
81
    'server': 'gunicorn/0.14.5',
82
    'vary': 'Accept-Language',
83
    'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
84
    'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
85
    'x-object-version': '525996',
86
    'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
87
    'x-object-meta-k1': 'v1',
88
    'x-object-meta-k2': 'v2'}
89
container_list = [
90
    dict(
91
        count=2,
92
        last_modified="2013-02-27T11:56:09.893033+00:00",
93
        bytes=677076979,
94
        name="pithos",
95
        x_container_policy=dict(quota="21474836480", versioning="auto")),
96
    dict(
97
        count=0,
98
        last_modified="2012-10-23T12:25:17.229187+00:00",
99
        bytes=0,
100
        name="trash",
101
        x_container_policy=dict(quota="21474836480", versioning="auto"))]
102
object_list = [
103
    dict(hash="",
104
        name="The_Secret_Garden.zip",
105
        x_object_public="/public/wdp9p",
106
        bytes=203304947,
107
        x_object_version_timestamp="1360237915.7027509",
108
        x_object_uuid="s0m3uu1df0r0bj0n3",
109
        last_modified="2013-02-07T11:51:55.702751+00:00",
110
        content_type="application/octet-stream",
111
        x_object_hash="0afdf29f71cd53126225c3f54ca",
112
        x_object_version=17737,
113
        x_object_modified_by=user_id),
114
    dict(hash="",
115
        name="The_Revealed_Garden.zip",
116
        x_object_public="/public/wpd7p",
117
        bytes=20330947,
118
        x_object_version_timestamp="13602915.7027509",
119
        x_object_uuid="s0m3uu1df0r0bj70w",
120
        last_modified="2013-02-07T11:51:55.702751+00:00",
121
        content_type="application/octet-stream",
122
        x_object_hash="0afdf29f71cd53126225c3f54ca",
123
        x_object_version=17737,
124
        x_object_modified_by=user_id)]
125
object_hashmap = dict(
126
    block_hash="sha256", block_size=4194304, bytes=33554432,
127
    hashes=[
128
        "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
129
        "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
130
        "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
131
        "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
132
        "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
133
        "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
134
        "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
135
        "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
136
sharers = [
137
    dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
138
    dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
139
    dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
140

    
141

    
142
class FR(object):
143
    """FR stands for Fake Response"""
144
    json = dict()
145
    headers = dict()
146
    content = json
147
    status = None
148
    status_code = 200
149

    
150
    def release(self):
151
        pass
152

    
153

    
154
class Pithos(TestCase):
155

    
156
    files = []
157

    
158
    def _create_temp_file(self, num_of_blocks):
159
        self.files.append(NamedTemporaryFile())
160
        tmpFile = self.files[-1]
161
        file_size = num_of_blocks * 4 * 1024 * 1024
162
        print('\n\tCreate tmp file')
163
        tmpFile.write(urandom(file_size))
164
        tmpFile.flush()
165
        tmpFile.seek(0)
166
        print('\t\tDone')
167
        return tmpFile
168

    
169
    def assert_dicts_are_equal(self, d1, d2):
170
        for k, v in d1.items():
171
            self.assertTrue(k in d2)
172
            if isinstance(v, dict):
173
                self.assert_dicts_are_equal(v, d2[k])
174
            else:
175
                self.assertEqual(unicode(v), unicode(d2[k]))
176

    
177
    def setUp(self):
178
        self.url = 'https://www.example.com/pithos'
179
        self.token = 'p17h0570k3n'
180
        self.client = PC(self.url, self.token)
181
        self.client.account = user_id
182
        self.client.container = 'c0nt@1n3r_i'
183

    
184
    def tearDown(self):
185
        FR.headers = dict()
186
        FR.status_code = 200
187
        FR.json = dict()
188
        FR.content = FR.json
189
        for f in self.files:
190
            f.close()
191

    
192
    #  Pithos+ methods that extend storage API
193

    
194
    @patch('%s.account_head' % pithos_pkg, return_value=FR())
195
    def test_get_account_info(self, AH):
196
        FR.headers = account_info
197
        r = self.client.get_account_info()
198
        self.assert_dicts_are_equal(r, account_info)
199
        self.assertEqual(AH.mock_calls[-1], call(until=None))
200
        unt = 'un71L-d473'
201
        r = self.client.get_account_info(until=unt)
202
        self.assert_dicts_are_equal(r, account_info)
203
        self.assertEqual(AH.mock_calls[-1], call(until=unt))
204
        FR.status_code = 401
205
        self.assertRaises(ClientError, self.client.get_account_info)
206

    
207
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
208
    def test_del_account_meta(self, ap):
209
        keys = ['k1', 'k2', 'k3']
210
        expected = []
211
        for key in keys:
212
            self.client.del_account_meta(key)
213
            expected.append(call(update=True, metadata={key: ''}))
214
        self.assertEqual(ap.mock_calls, expected)
215

    
216
    @patch('%s.container_head' % pithos_pkg, return_value=FR())
217
    def test_get_container_info(self, ch):
218
        FR.headers = container_info
219
        r = self.client.get_container_info()
220
        self.assert_dicts_are_equal(r, container_info)
221
        u = 'some date'
222
        r = self.client.get_container_info(until=u)
223
        self.assertEqual(ch.mock_calls, [call(until=None), call(until=u)])
224

    
225
    @patch('%s.account_get' % pithos_pkg, return_value=FR())
226
    def test_list_containers(self, get):
227
        FR.json = container_list
228
        r = self.client.list_containers()
229
        for i in range(len(r)):
230
            self.assert_dicts_are_equal(r[i], container_list[i])
231

    
232
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
233
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
234
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
235
    def test_upload_object(self, CI, CP, OP):
236
        num_of_blocks = 8
237
        tmpFile = self._create_temp_file(num_of_blocks)
238

    
239
        # Without kwargs
240
        self.client.upload_object(obj, tmpFile)
241
        self.assertEqual(PC.get_container_info.mock_calls, [call()])
242
        [call1, call2] = PC.object_put.mock_calls
243

    
244
        (args1, kwargs1) = call1[1:3]
245
        (args2, kwargs2) = call2[1:3]
246
        self.assertEqual(args1, (obj,))
247
        expected1 = dict(
248
            hashmap=True,
249
            success=(201, 409),
250
            format='json',
251
            json=dict(
252
                hashes=['s0m3h@5h'] * num_of_blocks,
253
                bytes=num_of_blocks * 4 * 1024 * 1024),
254
            etag=None,
255
            content_encoding=None,
256
            content_type='application/octet-stream',
257
            content_disposition=None,
258
            public=None,
259
            permissions=None)
260
        for k, v in expected1.items():
261
            if k == 'json':
262
                self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
263
                self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
264
            else:
265
                self.assertEqual(v, kwargs1[k])
266

    
267
        (args2, kwargs2) = call2[1:3]
268
        self.assertEqual(args2, (obj,))
269
        expected2 = dict(
270
            json=dict(
271
                hashes=['s0m3h@5h'] * num_of_blocks,
272
                bytes=num_of_blocks * 4 * 1024 * 1024),
273
            content_type='application/octet-stream',
274
            hashmap=True,
275
            success=201,
276
            format='json')
277
        for k, v in expected2.items():
278
            if k == 'json':
279
                self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
280
                self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
281
            else:
282
                self.assertEqual(v, kwargs2[k])
283

    
284
        OP = PC.object_put
285
        mock_offset = 2
286

    
287
        #  With progress bars
288
        try:
289
            from progress.bar import ShadyBar
290
            blck_bar = ShadyBar('Mock blck calc.')
291
            upld_bar = ShadyBar('Mock uplds')
292
        except ImportError:
293
            blck_bar = None
294
            upld_bar = None
295

    
296
        if blck_bar and upld_bar:
297

    
298
            def blck_gen(n):
299
                for i in blck_bar.iter(range(n)):
300
                    yield
301
                yield
302

    
303
            def upld_gen(n):
304
                for i in upld_bar.iter(range(n)):
305
                    yield
306
                yield
307

    
308
            tmpFile.seek(0)
309
            self.client.upload_object(
310
                obj, tmpFile,
311
                hash_cb=blck_gen, upload_cb=upld_gen)
312

    
313
            for i, c in enumerate(OP.mock_calls[-mock_offset:]):
314
                self.assertEqual(OP.mock_calls[i], c)
315

    
316
        #  With content-type
317
        tmpFile.seek(0)
318
        ctype = 'video/mpeg'
319
        sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
320
        self.client.upload_object(obj, tmpFile,
321
            content_type=ctype, sharing=sharing)
322
        self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
323
        self.assert_dicts_are_equal(
324
            OP.mock_calls[-2][2]['permissions'],
325
            sharing)
326

    
327
        # With other args
328
        tmpFile.seek(0)
329
        kwargs = dict(
330
            etag='s0m3E74g',
331
            content_type=ctype,
332
            content_disposition=ctype + 'd15p051710n',
333
            public=True,
334
            content_encoding='802.11')
335
        self.client.upload_object(obj, tmpFile, **kwargs)
336
        for arg, val in kwargs.items():
337
            self.assertEqual(OP.mock_calls[-2][2][arg], val)
338

    
339
    @patch('%s.put' % pithos_pkg, return_value=FR())
340
    @patch('%s.set_header' % client_pkg)
341
    def test_create_object(self, SH, put):
342
        cont = self.client.container
343
        ctype = 'c0n73n7/typ3'
344
        exp_shd = [
345
            call('Content-Type', 'application/octet-stream'),
346
            call('Content-length', '0'),
347
            call('Content-Type', ctype), call('Content-length', '42')]
348
        exp_put = [call('/%s/%s/%s' % (user_id, cont, obj), success=201)] * 2
349
        self.client.create_object(obj)
350
        self.client.create_object(obj, content_type=ctype, content_length=42)
351
        self.assertEqual(PC.set_header.mock_calls, exp_shd)
352
        self.assertEqual(put.mock_calls, exp_put)
353

    
354
    @patch('%s.put' % pithos_pkg, return_value=FR())
355
    @patch('%s.set_header' % client_pkg)
356
    def test_create_directory(self, SH, put):
357
        cont = self.client.container
358
        exp_shd = [
359
            call('Content-Type', 'application/directory'),
360
            call('Content-length', '0')]
361
        exp_put = [call('/%s/%s/%s' % (user_id, cont, obj), success=201)]
362
        self.client.create_directory(obj)
363
        self.assertEqual(PC.set_header.mock_calls, exp_shd)
364
        self.assertEqual(put.mock_calls, exp_put)
365

    
366
    def test_get_object_info(self):
367
        FR.headers = object_info
368
        version = 'v3r510n'
369
        with patch.object(PC, 'object_head', return_value=FR()) as head:
370
            r = self.client.get_object_info(obj)
371
            self.assertEqual(r, object_info)
372
            r = self.client.get_object_info(obj, version=version)
373
            self.assertEqual(head.mock_calls, [
374
                call(obj, version=None),
375
                call(obj, version=version)])
376
        with patch.object(
377
                PC,
378
                'object_head',
379
                side_effect=ClientError('Obj not found', 404)):
380
            self.assertRaises(
381
                ClientError,
382
                self.client.get_object_info,
383
                obj, version=version)
384

    
385
    @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
386
    def test_get_object_meta(self, GOI):
387
        expected = dict()
388
        for k, v in object_info.items():
389
            expected[k] = v
390
        r = self.client.get_object_meta(obj)
391
        self.assert_dicts_are_equal(r, expected)
392

    
393
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
394
    def test_del_object_meta(self, post):
395
        metakey = '50m3m3t4k3y'
396
        self.client.del_object_meta(obj, metakey)
397
        expected = call(obj, update=True, metadata={metakey: ''})
398
        self.assertEqual(post.mock_calls[-1], expected)
399

    
400
    @patch('%s.post' % client_pkg, return_value=FR())
401
    @patch('%s.set_header' % client_pkg)
402
    def test_replace_object_meta(self, SH, post):
403
        metas = dict(k1='new1', k2='new2', k3='new3')
404
        cont = self.client.container
405
        self.client.replace_object_meta(metas)
406
        expected = call('/%s/%s' % (user_id, cont), success=202)
407
        self.assertEqual(post.mock_calls[-1], expected)
408
        prfx = 'X-Object-Meta-'
409
        expected = [call('%s%s' % (prfx, k), v) for k, v in metas.items()]
410
        self.assertEqual(PC.set_header.mock_calls, expected)
411

    
412
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
413
    def test_copy_object(self, put):
414
        src_cont = 'src-c0nt41n3r'
415
        src_obj = 'src-0bj'
416
        dst_cont = 'dst-c0nt41n3r'
417
        dst_obj = 'dst-0bj'
418
        expected = call(
419
            src_obj,
420
            content_length=0,
421
            source_account=None,
422
            success=201,
423
            copy_from='/%s/%s' % (src_cont, src_obj),
424
            delimiter=None,
425
            content_type=None,
426
            source_version=None,
427
            public=False)
428
        self.client.copy_object(src_cont, src_obj, dst_cont)
429
        self.assertEqual(put.mock_calls[-1], expected)
430
        self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
431
        self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
432
        kwargs = dict(
433
            source_version='src-v3r510n',
434
            source_account='src-4cc0un7',
435
            public=True,
436
            content_type='c0n73n7Typ3',
437
            delimiter='5')
438
        self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
439
        for k, v in kwargs.items():
440
            self.assertEqual(v, put.mock_calls[-1][2][k])
441

    
442
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
443
    def test_move_object(self, put):
444
        src_cont = 'src-c0nt41n3r'
445
        src_obj = 'src-0bj'
446
        dst_cont = 'dst-c0nt41n3r'
447
        dst_obj = 'dst-0bj'
448
        expected = call(
449
            src_obj,
450
            content_length=0,
451
            source_account=None,
452
            success=201,
453
            move_from='/%s/%s' % (src_cont, src_obj),
454
            delimiter=None,
455
            content_type=None,
456
            source_version=None,
457
            public=False)
458
        self.client.move_object(src_cont, src_obj, dst_cont)
459
        self.assertEqual(put.mock_calls[-1], expected)
460
        self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
461
        self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
462
        kwargs = dict(
463
            source_version='src-v3r510n',
464
            source_account='src-4cc0un7',
465
            public=True,
466
            content_type='c0n73n7Typ3',
467
            delimiter='5')
468
        self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
469
        for k, v in kwargs.items():
470
            self.assertEqual(v, put.mock_calls[-1][2][k])
471

    
472
    @patch('%s.delete' % client_pkg, return_value=FR())
473
    def test_delete_object(self, delete):
474
        cont = self.client.container
475
        self.client.delete_object(obj)
476
        self.assertEqual(
477
            delete.mock_calls[-1],
478
            call('/%s/%s/%s' % (user_id, cont, obj), success=(204, 404)))
479
        FR.status_code = 404
480
        self.assertRaises(ClientError, self.client.delete_object, obj)
481

    
482
    @patch('%s.get' % client_pkg, return_value=FR())
483
    @patch('%s.set_param' % client_pkg)
484
    def test_list_objects(self, SP, get):
485
        FR.json = object_list
486
        acc = self.client.account
487
        cont = self.client.container
488
        SP = PC.set_param
489
        r = self.client.list_objects()
490
        for i in range(len(r)):
491
            self.assert_dicts_are_equal(r[i], object_list[i])
492
        self.assertEqual(get.mock_calls, [
493
            call('/%s/%s' % (acc, cont), success=(200, 204, 304, 404))])
494
        self.assertEqual(SP.mock_calls, [call('format', 'json')])
495
        FR.status_code = 304
496
        self.assertEqual(self.client.list_objects(), [])
497
        FR.status_code = 404
498
        self.assertRaises(ClientError, self.client.list_objects)
499

    
500
    @patch('%s.get' % client_pkg, return_value=FR())
501
    @patch('%s.set_param' % client_pkg)
502
    def test_list_objects_in_path(self, SP, get):
503
        FR.json = object_list
504
        path = '/some/awsome/path'
505
        acc = self.client.account
506
        cont = self.client.container
507
        SP = PC.set_param
508
        self.client.list_objects_in_path(path)
509
        self.assertEqual(get.mock_calls, [
510
            call('/%s/%s' % (acc, cont), success=(200, 204, 404))])
511
        self.assertEqual(SP.mock_calls, [
512
            call('format', 'json'), call('path', path)])
513
        FR.status_code = 404
514
        self.assertRaises(ClientError, self.client.list_objects)
515

    
516
    #  Pithos+ only methods
517

    
518
    @patch('%s.container_delete' % pithos_pkg, return_value=FR())
519
    def test_purge_container(self, cd):
520
        self.client.purge_container()
521
        self.assertTrue('until' in cd.mock_calls[-1][2])
522
        cont = self.client.container
523
        self.client.purge_container('another-container')
524
        self.assertEqual(self.client.container, cont)
525

    
526
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
527
    def test_upload_object_unchunked(self, put):
528
        num_of_blocks = 8
529
        tmpFile = self._create_temp_file(num_of_blocks)
530
        expected = dict(
531
                success=201,
532
                data=num_of_blocks * 4 * 1024 * 1024,
533
                etag='some-etag',
534
                content_encoding='some content_encoding',
535
                content_type='some content-type',
536
                content_disposition='some content_disposition',
537
                public=True,
538
                permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
539
        self.client.upload_object_unchunked(obj, tmpFile)
540
        self.assertEqual(put.mock_calls[-1][1], (obj,))
541
        self.assertEqual(
542
            sorted(put.mock_calls[-1][2].keys()),
543
            sorted(expected.keys()))
544
        kwargs = dict(expected)
545
        kwargs.pop('success')
546
        kwargs['size'] = kwargs.pop('data')
547
        kwargs['sharing'] = kwargs.pop('permissions')
548
        tmpFile.seek(0)
549
        self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
550
        pmc = put.mock_calls[-1][2]
551
        for k, v in expected.items():
552
            if k == 'data':
553
                self.assertEqual(len(pmc[k]), v)
554
            else:
555
                self.assertEqual(pmc[k], v)
556
        self.assertRaises(
557
            ClientError,
558
            self.client.upload_object_unchunked,
559
            obj, tmpFile, withHashFile=True)
560

    
561
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
562
    def test_create_object_by_manifestation(self, put):
563
        manifest = '%s/%s' % (self.client.container, obj)
564
        kwargs = dict(
565
                etag='some-etag',
566
                content_encoding='some content_encoding',
567
                content_type='some content-type',
568
                content_disposition='some content_disposition',
569
                public=True,
570
                sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
571
        self.client.create_object_by_manifestation(obj)
572
        expected = dict(content_length=0, manifest=manifest)
573
        for k in kwargs:
574
            expected['permissions' if k == 'sharing' else k] = None
575
        self.assertEqual(put.mock_calls[-1], call(obj, **expected))
576
        self.client.create_object_by_manifestation(obj, **kwargs)
577
        expected.update(kwargs)
578
        expected['permissions'] = expected.pop('sharing')
579
        self.assertEqual(put.mock_calls[-1], call(obj, **expected))
580

    
581
    @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
582
    @patch('%s.object_get' % pithos_pkg, return_value=FR())
583
    def test_download_object(self, GOH, GET):
584
        num_of_blocks = 8
585
        tmpFile = self._create_temp_file(num_of_blocks)
586
        FR.content = tmpFile.read(4 * 1024 * 1024)
587
        tmpFile = self._create_temp_file(num_of_blocks)
588
        GET = PC.object_get
589
        num_of_blocks = len(object_hashmap['hashes'])
590
        kwargs = dict(
591
            resume=True,
592
            version='version',
593
            range_str='10-20',
594
            if_match='if and only if',
595
            if_none_match='if and only not',
596
            if_modified_since='what if not?',
597
            if_unmodified_since='this happens if not!',
598
            async_headers=dict(Range='bytes=0-88888888'))
599

    
600
        self.client.download_object(obj, tmpFile)
601
        self.assertEqual(len(GET.mock_calls), num_of_blocks)
602
        self.assertEqual(GET.mock_calls[-1][1], (obj,))
603
        for k, v in kwargs.items():
604
            if k == 'async_headers':
605
                self.assertTrue('Range' in GET.mock_calls[-1][2][k])
606
            elif k in ('resume', 'range_str'):
607
                continue
608
            else:
609
                self.assertEqual(GET.mock_calls[-1][2][k], None)
610

    
611
        #  Check ranges are consecutive
612
        starts = []
613
        ends = []
614
        for c in GET.mock_calls:
615
            rng_str = c[2]['async_headers']['Range']
616
            (start, rng_str) = rng_str.split('=')
617
            (start, end) = rng_str.split('-')
618
            starts.append(start)
619
            ends.append(end)
620
        ends = sorted(ends)
621
        for i, start in enumerate(sorted(starts)):
622
            if i:
623
                int(ends[i - 1]) == int(start) - 1
624

    
625
        #  With progress bars
626
        try:
627
            from progress.bar import ShadyBar
628
            dl_bar = ShadyBar('Mock dl')
629
        except ImportError:
630
            dl_bar = None
631

    
632
        if dl_bar:
633

    
634
            def blck_gen(n):
635
                for i in dl_bar.iter(range(n)):
636
                    yield
637
                yield
638

    
639
            tmpFile.seek(0)
640
            self.client.download_object(obj, tmpFile, download_cb=blck_gen)
641
            self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
642

    
643
        tmpFile.seek(0)
644
        kwargs.pop('async_headers')
645
        kwargs.pop('resume')
646
        self.client.download_object(obj, tmpFile, **kwargs)
647
        for k, v in kwargs.items():
648
            if k == 'range_str':
649
                self.assertEqual(
650
                    GET.mock_calls[-1][2]['data_range'],
651
                    'bytes=%s' % v)
652
            else:
653
                self.assertEqual(GET.mock_calls[-1][2][k], v)
654

    
655
        #  ALl options on no tty
656

    
657
        def foo():
658
            return True
659

    
660
        tmpFile.seek(0)
661
        tmpFile.isatty = foo
662
        self.client.download_object(obj, tmpFile, **kwargs)
663
        for k, v in kwargs.items():
664
            if k == 'range_str':
665
                self.assertTrue('data_range' in GET.mock_calls[-1][2])
666
            else:
667
                self.assertEqual(GET.mock_calls[-1][2][k], v)
668

    
669
    def test_get_object_hashmap(self):
670
        FR.json = object_hashmap
671
        for empty in (304, 412):
672
            with patch.object(
673
                    PC, 'object_get',
674
                    side_effect=ClientError('Empty', status=empty)):
675
                r = self.client.get_object_hashmap(obj)
676
                self.assertEqual(r, {})
677
        exp_args = dict(
678
            hashmap=True,
679
            data_range=None,
680
            version=None,
681
            if_etag_match=None,
682
            if_etag_not_match=None,
683
            if_modified_since=None,
684
            if_unmodified_since=None)
685
        kwargs = dict(
686
            version='s0m3v3r51on',
687
            if_match='if match',
688
            if_none_match='if non match',
689
            if_modified_since='some date here',
690
            if_unmodified_since='some date here',
691
            data_range='10-20')
692
        with patch.object(PC, 'object_get', return_value=FR()) as get:
693
            r = self.client.get_object_hashmap(obj)
694
            self.assertEqual(r, object_hashmap)
695
            self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
696
            r = self.client.get_object_hashmap(obj, **kwargs)
697
            exp_args['if_etag_match'] = kwargs.pop('if_match')
698
            exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
699
            exp_args.update(kwargs)
700
            self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
701

    
702
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
703
    def test_set_account_group(self, post):
704
        (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
705
        self.client.set_account_group(group, usernames)
706
        self.assertEqual(
707
            post.mock_calls[-1],
708
            call(update=True, groups={group: usernames}))
709

    
710
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
711
    def test_del_account_group(self, post):
712
        group = 'aU53rGr0up'
713
        self.client.del_account_group(group)
714
        self.assertEqual(
715
            post.mock_calls[-1],
716
            call(update=True, groups={group: []}))
717

    
718
    @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
719
    def test_get_account_quota(self, GAI):
720
        key = 'x-account-policy-quota'
721
        r = self.client.get_account_quota()
722
        self.assertEqual(r[key], account_info[key])
723

    
724
    @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
725
    def test_get_account_versioning(self, GAI):
726
        key = 'x-account-policy-versioning'
727
        r = self.client.get_account_versioning()
728
        self.assertEqual(r[key], account_info[key])
729

    
730
    def test_get_account_meta(self):
731
        key = 'x-account-meta-'
732
        with patch.object(PC, 'get_account_info', return_value=account_info):
733
            r = self.client.get_account_meta()
734
            keys = [k for k in r if k.startswith(key)]
735
            self.assertFalse(keys)
736
        acc_info = dict(account_info)
737
        acc_info['%sk1' % key] = 'v1'
738
        acc_info['%sk2' % key] = 'v2'
739
        acc_info['%sk3' % key] = 'v3'
740
        with patch.object(PC, 'get_account_info', return_value=acc_info):
741
            r = self.client.get_account_meta()
742
            for k in [k for k in acc_info if k.startswith(key)]:
743
                self.assertEqual(r[k], acc_info[k])
744

    
745
    def test_get_account_group(self):
746
        key = 'x-account-group-'
747
        with patch.object(PC, 'get_account_info', return_value=account_info):
748
            r = self.client.get_account_group()
749
            keys = [k for k in r if k.startswith(key)]
750
            self.assertFalse(keys)
751
        acc_info = dict(account_info)
752
        acc_info['%sk1' % key] = 'g1'
753
        acc_info['%sk2' % key] = 'g2'
754
        acc_info['%sk3' % key] = 'g3'
755
        with patch.object(PC, 'get_account_info', return_value=acc_info):
756
            r = self.client.get_account_group()
757
            for k in [k for k in acc_info if k.startswith(key)]:
758
                self.assertEqual(r[k], acc_info[k])
759

    
760
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
761
    def test_set_account_meta(self, post):
762
        metas = dict(k1='v1', k2='v2', k3='v3')
763
        self.client.set_account_meta(metas)
764
        self.assertEqual(
765
            post.mock_calls[-1],
766
            call(update=True, metadata=metas))
767

    
768
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
769
    def test_set_account_quota(self, post):
770
        qu = 1024
771
        self.client.set_account_quota(qu)
772
        self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
773

    
774
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
775
    def test_set_account_versioning(self, post):
776
        vrs = 'n3wV3r51on1ngTyp3'
777
        self.client.set_account_versioning(vrs)
778
        self.assertEqual(
779
            post.mock_calls[-1],
780
            call(update=True, versioning=vrs))
781

    
782
    @patch('%s.container_delete' % pithos_pkg, return_value=FR())
783
    def test_del_container(self, delete):
784
        for kwarg in (
785
                dict(delimiter=None, until=None),
786
                dict(delimiter='X', until='50m3d473')):
787
            self.client.del_container(**kwarg)
788
            expected = dict(kwarg)
789
            expected['success'] = (204, 404, 409)
790
            self.assertEqual(delete.mock_calls[-1], call(**expected))
791
        for status_code in (404, 409):
792
            FR.status_code = status_code
793
            self.assertRaises(ClientError, self.client.del_container)
794

    
795
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
796
    def test_get_container_versioning(self, GCI):
797
        key = 'x-container-policy-versioning'
798
        cont = 'c0n7-417'
799
        bu_cnt = self.client.container
800
        for container in (None, cont):
801
            r = self.client.get_container_versioning(container=container)
802
            self.assertEqual(r[key], container_info[key])
803
            self.assertEqual(GCI.mock_calls[-1], call())
804
            self.assertEqual(bu_cnt, self.client.container)
805

    
806
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
807
    def test_get_container_quota(self, GCI):
808
        key = 'x-container-policy-quota'
809
        cont = 'c0n7-417'
810
        bu_cnt = self.client.container
811
        for container in (None, cont):
812
            r = self.client.get_container_quota(container=container)
813
            self.assertEqual(r[key], container_info[key])
814
            self.assertEqual(GCI.mock_calls[-1], call())
815
            self.assertEqual(bu_cnt, self.client.container)
816

    
817
    def test_get_container_meta(self):
818
        somedate = '50m3d473'
819
        key = 'x-container-meta'
820
        metaval = '50m3m374v41'
821
        container_plus = dict(container_info)
822
        container_plus[key] = metaval
823
        for ret in ((container_info, {}), (container_plus, {key: metaval})):
824
            with patch.object(
825
                    PC,
826
                    'get_container_info',
827
                    return_value=ret[0]) as gci:
828
                for until in (None, somedate):
829
                    r = self.client.get_container_meta(until=until)
830
                    self.assertEqual(r, ret[1])
831
                    self.assertEqual(gci.mock_calls[-1], call(until=until))
832

    
833
    def test_get_container_object_meta(self):
834
        somedate = '50m3d473'
835
        key = 'x-container-object-meta'
836
        metaval = '50m3m374v41'
837
        container_plus = dict(container_info)
838
        container_plus[key] = metaval
839
        for ret in (
840
                (container_info, {key: ''}),
841
                (container_plus, {key: metaval})):
842
            with patch.object(
843
                    PC,
844
                    'get_container_info',
845
                    return_value=ret[0]) as gci:
846
                for until in (None, somedate):
847
                    r = self.client.get_container_object_meta(until=until)
848
                    self.assertEqual(r, ret[1])
849
                    self.assertEqual(gci.mock_calls[-1], call(until=until))
850

    
851
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
852
    def test_set_container_meta(self, post):
853
        metas = dict(k1='v1', k2='v2', k3='v3')
854
        self.client.set_container_meta(metas)
855
        self.assertEqual(
856
            post.mock_calls[-1],
857
            call(update=True, metadata=metas))
858

    
859
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
860
    def test_del_container_meta(self, ap):
861
        self.client.del_container_meta('somekey')
862
        expected = [call(update=True, metadata={'somekey': ''})]
863
        self.assertEqual(ap.mock_calls, expected)
864

    
865
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
866
    def test_set_container_quota(self, post):
867
        qu = 1024
868
        self.client.set_container_quota(qu)
869
        self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
870

    
871
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
872
    def test_set_container_versioning(self, post):
873
        vrs = 'n3wV3r51on1ngTyp3'
874
        self.client.set_container_versioning(vrs)
875
        self.assertEqual(
876
            post.mock_calls[-1],
877
            call(update=True, versioning=vrs))
878

    
879
    @patch('%s.object_delete' % pithos_pkg, return_value=FR())
880
    def test_del_object(self, delete):
881
        for kwarg in (
882
                dict(delimiter=None, until=None),
883
                dict(delimiter='X', until='50m3d473')):
884
            self.client.del_object(obj, **kwarg)
885
            self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
886

    
887
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
888
    def test_set_object_meta(self, post):
889
        metas = dict(k1='v1', k2='v2', k3='v3')
890
        self.assertRaises(
891
            AssertionError,
892
            self.client.set_object_meta,
893
            obj, 'Non dict arg')
894
        self.client.set_object_meta(obj, metas)
895
        self.assertEqual(
896
            post.mock_calls[-1],
897
            call(obj, update=True, metadata=metas))
898

    
899
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
900
    def test_publish_object(self, post):
901
        oinfo = dict(object_info)
902
        val = 'pubL1c'
903
        oinfo['x-object-public'] = val
904
        with patch.object(PC, 'get_object_info', return_value=oinfo) as gof:
905
            r = self.client.publish_object(obj)
906
            self.assertEqual(
907
                post.mock_calls[-1],
908
                call(obj, public=True, update=True))
909
            self.assertEqual(gof.mock_calls[-1], call(obj))
910
            self.assertEqual(r, '%s%s' % (self.url[:-6], val))
911

    
912
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
913
    def test_unpublish_object(self, post):
914
        self.client.unpublish_object(obj)
915
        self.assertEqual(
916
            post.mock_calls[-1],
917
            call(obj, public=False, update=True))
918

    
919
    def test_get_object_sharing(self):
920
        info = dict(object_info)
921
        expected = dict(read='u1,g1,u2', write='u1')
922
        info['x-object-sharing'] = '; '.join(
923
            ['%s=%s' % (k, v) for k, v in expected.items()])
924
        with patch.object(PC, 'get_object_info', return_value=info) as GOF:
925
            r = self.client.get_object_sharing(obj)
926
            self.assertEqual(GOF.mock_calls[-1], call(obj))
927
            self.assert_dicts_are_equal(r, expected)
928
            info['x-object-sharing'] = '//'.join(
929
                ['%s=%s' % (k, v) for k, v in expected.items()])
930
            self.assertRaises(
931
                ValueError,
932
                self.client.get_object_sharing,
933
                obj)
934
            info['x-object-sharing'] = '; '.join(
935
                ['%s:%s' % (k, v) for k, v in expected.items()])
936
            self.assertRaises(
937
                ClientError,
938
                self.client.get_object_sharing,
939
                obj)
940
            info['x-object-sharing'] = 'read=%s' % expected['read']
941
            r = self.client.get_object_sharing(obj)
942
            expected.pop('write')
943
            self.assert_dicts_are_equal(r, expected)
944

    
945
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
946
    def test_set_object_sharing(self, POST):
947
        read_perms = ['u1', 'g1', 'u2', 'g2']
948
        write_perms = ['u1', 'g1']
949
        for kwargs in (
950
                dict(read_permition=read_perms, write_permition=write_perms),
951
                dict(read_permition=read_perms),
952
                dict(write_permition=write_perms),
953
                dict()):
954
            self.client.set_object_sharing(obj, **kwargs)
955
            kwargs['read'] = kwargs.pop('read_permition', '')
956
            kwargs['write'] = kwargs.pop('write_permition', '')
957
            self.assertEqual(
958
                POST.mock_calls[-1],
959
                call(obj, update=True, permissions=kwargs))
960

    
961
    @patch('%s.set_object_sharing' % pithos_pkg)
962
    def test_del_object_sharing(self, SOS):
963
        self.client.del_object_sharing(obj)
964
        self.assertEqual(SOS.mock_calls[-1], call(obj))
965

    
966
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
967
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
968
    def test_append_object(self, post, GCI):
969
        num_of_blocks = 4
970
        tmpFile = self._create_temp_file(num_of_blocks)
971
        tmpFile.seek(0, 2)
972
        file_size = tmpFile.tell()
973
        for turn in range(2):
974
            tmpFile.seek(0, 0)
975

    
976
            try:
977
                from progress.bar import ShadyBar
978
                apn_bar = ShadyBar('Mock append')
979
            except ImportError:
980
                apn_bar = None
981

    
982
            if apn_bar:
983

    
984
                def append_gen(n):
985
                    for i in apn_bar.iter(range(n)):
986
                        yield
987
                    yield
988

    
989
            else:
990
                append_gen = None
991

    
992
            self.client.append_object(
993
                obj, tmpFile,
994
                upload_cb=append_gen if turn else None)
995
            self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
996
            (args, kwargs) = post.mock_calls[-1][1:3]
997
            self.assertEqual(args, (obj,))
998
            self.assertEqual(kwargs['content_length'], len(kwargs['data']))
999
            fsize = num_of_blocks * int(kwargs['content_length'])
1000
            self.assertEqual(fsize, file_size)
1001
            self.assertEqual(kwargs['content_range'], 'bytes */*')
1002
            exp = 'application/octet-stream'
1003
            self.assertEqual(kwargs['content_type'], exp)
1004
            self.assertEqual(kwargs['update'], True)
1005

    
1006
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
1007
    def test_truncate_object(self, post):
1008
        upto_bytes = 377
1009
        self.client.truncate_object(obj, upto_bytes)
1010
        self.assertEqual(post.mock_calls[-1], call(
1011
            obj,
1012
            update=True,
1013
            object_bytes=upto_bytes,
1014
            content_range='bytes 0-%s/*' % upto_bytes,
1015
            content_type='application/octet-stream',
1016
            source_object='/%s/%s' % (self.client.container, obj)))
1017

    
1018
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1019
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
1020
    def test_overwrite_object(self, post, GCI):
1021
        num_of_blocks = 4
1022
        tmpFile = self._create_temp_file(num_of_blocks)
1023
        tmpFile.seek(0, 2)
1024
        file_size = tmpFile.tell()
1025
        info = dict(object_info)
1026
        info['content-length'] = file_size
1027
        block_size = container_info['x-container-block-size']
1028
        with patch.object(PC, 'get_object_info', return_value=info) as GOI:
1029
            for start, end in (
1030
                    (0, file_size + 1),
1031
                    (file_size + 1, file_size + 2)):
1032
                tmpFile.seek(0, 0)
1033
                self.assertRaises(
1034
                    ClientError,
1035
                    self.client.overwrite_object,
1036
                    obj, start, end, tmpFile)
1037
            for start, end in ((0, 144), (144, 233), (233, file_size)):
1038
                tmpFile.seek(0, 0)
1039
                owr_gen = None
1040
                exp_size = end - start + 1
1041
                if not start or exp_size > block_size:
1042
                    try:
1043
                        from progress.bar import ShadyBar
1044
                        owr_bar = ShadyBar('Mock append')
1045
                    except ImportError:
1046
                        owr_bar = None
1047

    
1048
                    if owr_bar:
1049

    
1050
                        def owr_gen(n):
1051
                            for i in owr_bar.iter(range(n)):
1052
                                yield
1053
                            yield
1054

    
1055
                    if exp_size > block_size:
1056
                        exp_size = exp_size % block_size or block_size
1057

    
1058
                self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
1059
                self.assertEqual(GOI.mock_calls[-1], call(obj))
1060
                self.assertEqual(GCI.mock_calls[-1], call())
1061
                (args, kwargs) = post.mock_calls[-1][1:3]
1062
                self.assertEqual(args, (obj,))
1063
                self.assertEqual(len(kwargs['data']), exp_size)
1064
                self.assertEqual(kwargs['content_length'], exp_size)
1065
                self.assertEqual(kwargs['update'], True)
1066
                exp = 'application/octet-stream'
1067
                self.assertEqual(kwargs['content_type'], exp)
1068

    
1069
    @patch('%s.set_param' % client_pkg)
1070
    @patch('%s.get' % pithos_pkg, return_value=FR())
1071
    def test_get_sharing_accounts(self, get, SP):
1072
        FR.json = sharers
1073
        for kws in (
1074
                dict(),
1075
                dict(limit='50m3-11m17'),
1076
                dict(marker='X'),
1077
                dict(limit='50m3-11m17', marker='X')):
1078
            r = self.client.get_sharing_accounts(**kws)
1079
            self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
1080
            limit, marker = kws.get('limit', None), kws.get('marker', None)
1081
            self.assertEqual(SP.mock_calls[-2], call(
1082
                'limit', limit,
1083
                iff=limit is not None))
1084
            self.assertEqual(SP.mock_calls[-1], call(
1085
                'marker', marker,
1086
                iff=marker is not None))
1087
            self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1088
            for i in range(len(r)):
1089
                self.assert_dicts_are_equal(r[i], sharers[i])
1090

    
1091
    @patch('%s.object_get' % pithos_pkg, return_value=FR())
1092
    def test_get_object_versionlist(self, get):
1093
        info = dict(object_info)
1094
        info['versions'] = ['v1', 'v2']
1095
        FR.json = info
1096
        r = self.client.get_object_versionlist(obj)
1097
        self.assertEqual(
1098
            get.mock_calls[-1],
1099
            call(obj, format='json', version='list'))
1100
        self.assertEqual(r, info['versions'])
1101

    
1102
if __name__ == '__main__':
1103
    from sys import argv
1104
    from kamaki.clients.test import runTestCase
1105
    runTestCase(Pithos, 'Pithos+ Client', argv[1:])