Statistics
| Branch: | Tag: | Revision:

root / kamaki / clients / pithos / test.py @ aa9c7402

History | View | Annotate | Download (41.9 kB)

1
# Copyright 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from unittest import TestCase
35
from mock import patch, call
36
from tempfile import NamedTemporaryFile
37
from os import urandom
38

    
39
from kamaki.clients import ClientError
40
from kamaki.clients.pithos import PithosClient as PC
41
from kamaki.clients.connection.kamakicon import KamakiHTTPConnection as C
42

    
43
client_pkg = 'kamaki.clients.Client'
44
pithos_pkg = 'kamaki.clients.pithos.PithosClient'
45

    
46
user_id = 'ac0un7-1d-5tr1ng'
47
obj = 'obj3c7N4m3'
48

    
49
account_info = {
50
    'content-language': 'en-us',
51
    'content-type': 'text/html; charset=utf-8',
52
    'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
53
    'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
54
    'server': 'gunicorn/0.14.5',
55
    'vary': 'Accept-Language',
56
    'x-account-bytes-used': '751615526',
57
    'x-account-container-count': 7,
58
    'x-account-policy-quota': 53687091200,
59
    'x-account-policy-versioning': 'auto'}
60
container_info = {
61
    'content-language': 'en-us',
62
    'content-type': 'text/html; charset=utf-8',
63
    'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
64
    'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
65
    'server': 'gunicorn/0.14.5',
66
    'vary': 'Accept-Language',
67
    'x-container-block-hash': 'sha256',
68
    'x-container-block-size': 4194304,
69
    'x-container-bytes-used': 309528938,
70
    'x-container-object-count': 14,
71
    'x-container-object-meta': '',
72
    'x-container-policy-quota': 53687091200,
73
    'x-container-policy-versioning': 'auto'}
74
object_info = {
75
    'content-language': 'en-us',
76
    'content-length': 254965,
77
    'content-type': 'application/octet-stream',
78
    'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
79
    'etag': '',
80
    'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
81
    'server': 'gunicorn/0.14.5',
82
    'vary': 'Accept-Language',
83
    'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
84
    'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
85
    'x-object-version': '525996',
86
    'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
87
    'x-object-meta-k1': 'v1',
88
    'x-object-meta-k2': 'v2'}
89
container_list = [
90
    dict(
91
        count=2,
92
        last_modified="2013-02-27T11:56:09.893033+00:00",
93
        bytes=677076979,
94
        name="pithos",
95
        x_container_policy=dict(quota="21474836480", versioning="auto")),
96
    dict(
97
        count=0,
98
        last_modified="2012-10-23T12:25:17.229187+00:00",
99
        bytes=0,
100
        name="trash",
101
        x_container_policy=dict(quota="21474836480", versioning="auto"))]
102
object_list = [
103
    dict(hash="",
104
        name="The_Secret_Garden.zip",
105
        x_object_public="/public/wdp9p",
106
        bytes=203304947,
107
        x_object_version_timestamp="1360237915.7027509",
108
        x_object_uuid="s0m3uu1df0r0bj0n3",
109
        last_modified="2013-02-07T11:51:55.702751+00:00",
110
        content_type="application/octet-stream",
111
        x_object_hash="0afdf29f71cd53126225c3f54ca",
112
        x_object_version=17737,
113
        x_object_modified_by=user_id),
114
    dict(hash="",
115
        name="The_Revealed_Garden.zip",
116
        x_object_public="/public/wpd7p",
117
        bytes=20330947,
118
        x_object_version_timestamp="13602915.7027509",
119
        x_object_uuid="s0m3uu1df0r0bj70w",
120
        last_modified="2013-02-07T11:51:55.702751+00:00",
121
        content_type="application/octet-stream",
122
        x_object_hash="0afdf29f71cd53126225c3f54ca",
123
        x_object_version=17737,
124
        x_object_modified_by=user_id)]
125
object_hashmap = dict(
126
    block_hash="sha256", block_size=4194304, bytes=33554432,
127
    hashes=[
128
        "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
129
        "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
130
        "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
131
        "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
132
        "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
133
        "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
134
        "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
135
        "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
136
sharers = [
137
    dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
138
    dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
139
    dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
140

    
141

    
142
class FR(object):
143
    """FR stands for Fake Response"""
144
    json = dict()
145
    headers = dict()
146
    content = json
147
    status = None
148
    status_code = 200
149

    
150
    def release(self):
151
        pass
152

    
153

    
154
class Pithos(TestCase):
155

    
156
    files = []
157

    
158
    def _create_temp_file(self, num_of_blocks):
159
        self.files.append(NamedTemporaryFile())
160
        tmpFile = self.files[-1]
161
        file_size = num_of_blocks * 4 * 1024 * 1024
162
        print('\n\tCreate tmp file')
163
        tmpFile.write(urandom(file_size))
164
        tmpFile.flush()
165
        tmpFile.seek(0)
166
        print('\t\tDone')
167
        return tmpFile
168

    
169
    def assert_dicts_are_equal(self, d1, d2):
170
        for k, v in d1.items():
171
            self.assertTrue(k in d2)
172
            if isinstance(v, dict):
173
                self.assert_dicts_are_equal(v, d2[k])
174
            else:
175
                self.assertEqual(unicode(v), unicode(d2[k]))
176

    
177
    def setUp(self):
178
        self.url = 'https://www.example.com/pithos'
179
        self.token = 'p17h0570k3n'
180
        self.client = PC(self.url, self.token)
181
        self.client.account = user_id
182
        self.client.container = 'c0nt@1n3r_i'
183

    
184
    def tearDown(self):
185
        FR.headers = dict()
186
        FR.status_code = 200
187
        FR.json = dict()
188
        FR.content = FR.json
189
        for f in self.files:
190
            f.close()
191

    
192
    #  Pithos+ methods that extend storage API
193

    
194
    @patch('%s.account_head' % pithos_pkg, return_value=FR())
195
    def test_get_account_info(self, AH):
196
        FR.headers = account_info
197
        r = self.client.get_account_info()
198
        self.assert_dicts_are_equal(r, account_info)
199
        self.assertEqual(AH.mock_calls[-1], call(until=None))
200
        unt = 'un71L-d473'
201
        r = self.client.get_account_info(until=unt)
202
        self.assert_dicts_are_equal(r, account_info)
203
        self.assertEqual(AH.mock_calls[-1], call(until=unt))
204
        FR.status_code = 401
205
        self.assertRaises(ClientError, self.client.get_account_info)
206

    
207
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
208
    def test_del_account_meta(self, ap):
209
        keys = ['k1', 'k2', 'k3']
210
        expected = []
211
        for key in keys:
212
            self.client.del_account_meta(key)
213
            expected.append(call(update=True, metadata={key: ''}))
214
        self.assertEqual(ap.mock_calls, expected)
215

    
216
    @patch('%s.container_head' % pithos_pkg, return_value=FR())
217
    def test_get_container_info(self, ch):
218
        FR.headers = container_info
219
        r = self.client.get_container_info()
220
        self.assert_dicts_are_equal(r, container_info)
221
        u = 'some date'
222
        r = self.client.get_container_info(until=u)
223
        self.assertEqual(ch.mock_calls, [call(until=None), call(until=u)])
224

    
225
    @patch('%s.account_get' % pithos_pkg, return_value=FR())
226
    def test_list_containers(self, get):
227
        FR.json = container_list
228
        r = self.client.list_containers()
229
        for i in range(len(r)):
230
            self.assert_dicts_are_equal(r[i], container_list[i])
231

    
232
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
233
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
234
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
235
    def test_upload_object(self, CI, CP, OP):
236
        num_of_blocks = 8
237
        tmpFile = self._create_temp_file(num_of_blocks)
238

    
239
        # Without kwargs
240
        self.client.upload_object(obj, tmpFile)
241
        self.assertEqual(PC.get_container_info.mock_calls, [call()])
242
        [call1, call2] = PC.object_put.mock_calls
243

    
244
        (args1, kwargs1) = call1[1:3]
245
        (args2, kwargs2) = call2[1:3]
246
        self.assertEqual(args1, (obj,))
247
        expected1 = dict(
248
            hashmap=True,
249
            success=(201, 409),
250
            format='json',
251
            json=dict(
252
                hashes=['s0m3h@5h'] * num_of_blocks,
253
                bytes=num_of_blocks * 4 * 1024 * 1024),
254
            etag=None,
255
            content_encoding=None,
256
            content_type='application/octet-stream',
257
            content_disposition=None,
258
            public=None,
259
            permissions=None)
260
        for k, v in expected1.items():
261
            if k == 'json':
262
                self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
263
                self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
264
            else:
265
                self.assertEqual(v, kwargs1[k])
266

    
267
        (args2, kwargs2) = call2[1:3]
268
        self.assertEqual(args2, (obj,))
269
        expected2 = dict(
270
            json=dict(
271
                hashes=['s0m3h@5h'] * num_of_blocks,
272
                bytes=num_of_blocks * 4 * 1024 * 1024),
273
            content_type='application/octet-stream',
274
            hashmap=True,
275
            success=201,
276
            format='json')
277
        for k, v in expected2.items():
278
            if k == 'json':
279
                self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
280
                self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
281
            else:
282
                self.assertEqual(v, kwargs2[k])
283

    
284
        OP = PC.object_put
285
        mock_offset = 2
286

    
287
        #  With progress bars
288
        try:
289
            from progress.bar import ShadyBar
290
            blck_bar = ShadyBar('Mock blck calc.')
291
            upld_bar = ShadyBar('Mock uplds')
292
        except ImportError:
293
            blck_bar = None
294
            upld_bar = None
295

    
296
        if blck_bar and upld_bar:
297

    
298
            def blck_gen(n):
299
                for i in blck_bar.iter(range(n)):
300
                    yield
301
                yield
302

    
303
            def upld_gen(n):
304
                for i in upld_bar.iter(range(n)):
305
                    yield
306
                yield
307

    
308
            tmpFile.seek(0)
309
            self.client.upload_object(
310
                obj, tmpFile,
311
                hash_cb=blck_gen, upload_cb=upld_gen)
312

    
313
            for i, c in enumerate(OP.mock_calls[-mock_offset:]):
314
                self.assertEqual(OP.mock_calls[i], c)
315

    
316
        #  With content-type
317
        tmpFile.seek(0)
318
        ctype = 'video/mpeg'
319
        sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
320
        self.client.upload_object(obj, tmpFile,
321
            content_type=ctype, sharing=sharing)
322
        self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
323
        self.assert_dicts_are_equal(
324
            OP.mock_calls[-2][2]['permissions'],
325
            sharing)
326

    
327
        # With other args
328
        tmpFile.seek(0)
329
        kwargs = dict(
330
            etag='s0m3E74g',
331
            content_type=ctype,
332
            content_disposition=ctype + 'd15p051710n',
333
            public=True,
334
            content_encoding='802.11')
335
        self.client.upload_object(obj, tmpFile, **kwargs)
336
        for arg, val in kwargs.items():
337
            self.assertEqual(OP.mock_calls[-2][2][arg], val)
338

    
339
    def test_get_object_info(self):
340
        FR.headers = object_info
341
        version = 'v3r510n'
342
        with patch.object(PC, 'object_head', return_value=FR()) as head:
343
            r = self.client.get_object_info(obj)
344
            self.assertEqual(r, object_info)
345
            r = self.client.get_object_info(obj, version=version)
346
            self.assertEqual(head.mock_calls, [
347
                call(obj, version=None),
348
                call(obj, version=version)])
349
        with patch.object(
350
                PC, 'object_head',
351
                side_effect=ClientError('Obj not found', 404)):
352
            self.assertRaises(
353
                ClientError,
354
                self.client.get_object_info,
355
                obj, version=version)
356

    
357
    @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
358
    def test_get_object_meta(self, GOI):
359
        expected = dict()
360
        for k, v in object_info.items():
361
            expected[k] = v
362
        r = self.client.get_object_meta(obj)
363
        self.assert_dicts_are_equal(r, expected)
364

    
365
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
366
    def test_del_object_meta(self, post):
367
        metakey = '50m3m3t4k3y'
368
        self.client.del_object_meta(obj, metakey)
369
        expected = call(obj, update=True, metadata={metakey: ''})
370
        self.assertEqual(post.mock_calls[-1], expected)
371

    
372
    @patch('%s.post' % client_pkg, return_value=FR())
373
    @patch('%s.set_header' % client_pkg)
374
    def test_replace_object_meta(self, SH, post):
375
        metas = dict(k1='new1', k2='new2', k3='new3')
376
        cont = self.client.container
377
        self.client.replace_object_meta(metas)
378
        expected = call('/%s/%s' % (user_id, cont), success=202)
379
        self.assertEqual(post.mock_calls[-1], expected)
380
        prfx = 'X-Object-Meta-'
381
        expected = [call('%s%s' % (prfx, k), v) for k, v in metas.items()]
382
        self.assertEqual(PC.set_header.mock_calls, expected)
383

    
384
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
385
    def test_copy_object(self, put):
386
        src_cont = 'src-c0nt41n3r'
387
        src_obj = 'src-0bj'
388
        dst_cont = 'dst-c0nt41n3r'
389
        dst_obj = 'dst-0bj'
390
        expected = call(
391
            src_obj,
392
            content_length=0,
393
            source_account=None,
394
            success=201,
395
            copy_from='/%s/%s' % (src_cont, src_obj),
396
            delimiter=None,
397
            content_type=None,
398
            source_version=None,
399
            public=False)
400
        self.client.copy_object(src_cont, src_obj, dst_cont)
401
        self.assertEqual(put.mock_calls[-1], expected)
402
        self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
403
        self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
404
        kwargs = dict(
405
            source_version='src-v3r510n',
406
            source_account='src-4cc0un7',
407
            public=True,
408
            content_type='c0n73n7Typ3',
409
            delimiter='5')
410
        self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
411
        for k, v in kwargs.items():
412
            self.assertEqual(v, put.mock_calls[-1][2][k])
413

    
414
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
415
    def test_move_object(self, put):
416
        src_cont = 'src-c0nt41n3r'
417
        src_obj = 'src-0bj'
418
        dst_cont = 'dst-c0nt41n3r'
419
        dst_obj = 'dst-0bj'
420
        expected = call(
421
            src_obj,
422
            content_length=0,
423
            source_account=None,
424
            success=201,
425
            move_from='/%s/%s' % (src_cont, src_obj),
426
            delimiter=None,
427
            content_type=None,
428
            source_version=None,
429
            public=False)
430
        self.client.move_object(src_cont, src_obj, dst_cont)
431
        self.assertEqual(put.mock_calls[-1], expected)
432
        self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
433
        self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
434
        kwargs = dict(
435
            source_version='src-v3r510n',
436
            source_account='src-4cc0un7',
437
            public=True,
438
            content_type='c0n73n7Typ3',
439
            delimiter='5')
440
        self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
441
        for k, v in kwargs.items():
442
            self.assertEqual(v, put.mock_calls[-1][2][k])
443

    
444
    @patch('%s.delete' % client_pkg, return_value=FR())
445
    def test_delete_object(self, delete):
446
        cont = self.client.container
447
        self.client.delete_object(obj)
448
        self.assertEqual(
449
            delete.mock_calls[-1],
450
            call('/%s/%s/%s' % (user_id, cont, obj), success=(204, 404)))
451
        FR.status_code = 404
452
        self.assertRaises(ClientError, self.client.delete_object, obj)
453

    
454
    @patch('%s.get' % client_pkg, return_value=FR())
455
    @patch('%s.set_param' % client_pkg)
456
    def test_list_objects(self, SP, get):
457
        FR.json = object_list
458
        acc = self.client.account
459
        cont = self.client.container
460
        SP = PC.set_param
461
        r = self.client.list_objects()
462
        for i in range(len(r)):
463
            self.assert_dicts_are_equal(r[i], object_list[i])
464
        self.assertEqual(get.mock_calls, [
465
            call('/%s/%s' % (acc, cont), success=(200, 204, 304, 404))])
466
        self.assertEqual(SP.mock_calls, [call('format', 'json')])
467
        FR.status_code = 304
468
        self.assertEqual(self.client.list_objects(), [])
469
        FR.status_code = 404
470
        self.assertRaises(ClientError, self.client.list_objects)
471

    
472
    @patch('%s.get' % client_pkg, return_value=FR())
473
    @patch('%s.set_param' % client_pkg)
474
    def test_list_objects_in_path(self, SP, get):
475
        FR.json = object_list
476
        path = '/some/awsome/path'
477
        acc = self.client.account
478
        cont = self.client.container
479
        SP = PC.set_param
480
        self.client.list_objects_in_path(path)
481
        self.assertEqual(get.mock_calls, [
482
            call('/%s/%s' % (acc, cont), success=(200, 204, 404))])
483
        self.assertEqual(SP.mock_calls, [
484
            call('format', 'json'), call('path', path)])
485
        FR.status_code = 404
486
        self.assertRaises(ClientError, self.client.list_objects)
487

    
488
    #  Pithos+ only methods
489

    
490
    @patch('%s.container_delete' % pithos_pkg, return_value=FR())
491
    def test_purge_container(self, cd):
492
        self.client.purge_container()
493
        self.assertTrue('until' in cd.mock_calls[-1][2])
494
        cont = self.client.container
495
        self.client.purge_container('another-container')
496
        self.assertEqual(self.client.container, cont)
497

    
498
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
499
    def test_upload_object_unchunked(self, put):
500
        num_of_blocks = 8
501
        tmpFile = self._create_temp_file(num_of_blocks)
502
        expected = dict(
503
                success=201,
504
                data=num_of_blocks * 4 * 1024 * 1024,
505
                etag='some-etag',
506
                content_encoding='some content_encoding',
507
                content_type='some content-type',
508
                content_disposition='some content_disposition',
509
                public=True,
510
                permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
511
        self.client.upload_object_unchunked(obj, tmpFile)
512
        self.assertEqual(put.mock_calls[-1][1], (obj,))
513
        self.assertEqual(
514
            sorted(put.mock_calls[-1][2].keys()),
515
            sorted(expected.keys()))
516
        kwargs = dict(expected)
517
        kwargs.pop('success')
518
        kwargs['size'] = kwargs.pop('data')
519
        kwargs['sharing'] = kwargs.pop('permissions')
520
        tmpFile.seek(0)
521
        self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
522
        pmc = put.mock_calls[-1][2]
523
        for k, v in expected.items():
524
            if k == 'data':
525
                self.assertEqual(len(pmc[k]), v)
526
            else:
527
                self.assertEqual(pmc[k], v)
528
        self.assertRaises(
529
            ClientError,
530
            self.client.upload_object_unchunked,
531
            obj, tmpFile, withHashFile=True)
532

    
533
    @patch('%s.object_put' % pithos_pkg, return_value=FR())
534
    def test_create_object_by_manifestation(self, put):
535
        manifest = '%s/%s' % (self.client.container, obj)
536
        kwargs = dict(
537
                etag='some-etag',
538
                content_encoding='some content_encoding',
539
                content_type='some content-type',
540
                content_disposition='some content_disposition',
541
                public=True,
542
                sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
543
        self.client.create_object_by_manifestation(obj)
544
        expected = dict(content_length=0, manifest=manifest)
545
        for k in kwargs:
546
            expected['permissions' if k == 'sharing' else k] = None
547
        self.assertEqual(put.mock_calls[-1], call(obj, **expected))
548
        self.client.create_object_by_manifestation(obj, **kwargs)
549
        expected.update(kwargs)
550
        expected['permissions'] = expected.pop('sharing')
551
        self.assertEqual(put.mock_calls[-1], call(obj, **expected))
552

    
553
    @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
554
    @patch('%s.object_get' % pithos_pkg, return_value=FR())
555
    def test_download_object(self, GOH, GET):
556
        num_of_blocks = 8
557
        tmpFile = self._create_temp_file(num_of_blocks)
558
        FR.content = tmpFile.read(4 * 1024 * 1024)
559
        tmpFile = self._create_temp_file(num_of_blocks)
560
        GET = PC.object_get
561
        num_of_blocks = len(object_hashmap['hashes'])
562
        kwargs = dict(
563
            resume=True,
564
            version='version',
565
            range_str='10-20',
566
            if_match='if and only if',
567
            if_none_match='if and only not',
568
            if_modified_since='what if not?',
569
            if_unmodified_since='this happens if not!',
570
            async_headers=dict(Range='bytes=0-88888888'))
571

    
572
        self.client.download_object(obj, tmpFile)
573
        self.assertEqual(len(GET.mock_calls), num_of_blocks)
574
        self.assertEqual(GET.mock_calls[-1][1], (obj,))
575
        for k, v in kwargs.items():
576
            if k == 'async_headers':
577
                self.assertTrue('Range' in GET.mock_calls[-1][2][k])
578
            elif k in ('resume', 'range_str'):
579
                continue
580
            else:
581
                self.assertEqual(GET.mock_calls[-1][2][k], None)
582

    
583
        #  Check ranges are consecutive
584
        starts = []
585
        ends = []
586
        for c in GET.mock_calls:
587
            rng_str = c[2]['async_headers']['Range']
588
            (start, rng_str) = rng_str.split('=')
589
            (start, end) = rng_str.split('-')
590
            starts.append(start)
591
            ends.append(end)
592
        ends = sorted(ends)
593
        for i, start in enumerate(sorted(starts)):
594
            if i:
595
                int(ends[i - 1]) == int(start) - 1
596

    
597
        #  With progress bars
598
        try:
599
            from progress.bar import ShadyBar
600
            dl_bar = ShadyBar('Mock dl')
601
        except ImportError:
602
            dl_bar = None
603

    
604
        if dl_bar:
605

    
606
            def blck_gen(n):
607
                for i in dl_bar.iter(range(n)):
608
                    yield
609
                yield
610

    
611
            tmpFile.seek(0)
612
            self.client.download_object(obj, tmpFile, download_cb=blck_gen)
613
            self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
614

    
615
        tmpFile.seek(0)
616
        kwargs.pop('async_headers')
617
        kwargs.pop('resume')
618
        self.client.download_object(obj, tmpFile, **kwargs)
619
        for k, v in kwargs.items():
620
            if k == 'range_str':
621
                self.assertEqual(
622
                    GET.mock_calls[-1][2]['data_range'],
623
                    'bytes=%s' % v)
624
            else:
625
                self.assertEqual(GET.mock_calls[-1][2][k], v)
626

    
627
        #  ALl options on no tty
628

    
629
        def foo():
630
            return True
631

    
632
        tmpFile.seek(0)
633
        tmpFile.isatty = foo
634
        self.client.download_object(obj, tmpFile, **kwargs)
635
        for k, v in kwargs.items():
636
            if k == 'range_str':
637
                self.assertTrue('data_range' in GET.mock_calls[-1][2])
638
            else:
639
                self.assertEqual(GET.mock_calls[-1][2][k], v)
640

    
641
    def test_get_object_hashmap(self):
642
        FR.json = object_hashmap
643
        for empty in (304, 412):
644
            with patch.object(
645
                    PC, 'object_get',
646
                    side_effect=ClientError('Empty', status=empty)):
647
                r = self.client.get_object_hashmap(obj)
648
                self.assertEqual(r, {})
649
        exp_args = dict(
650
            hashmap=True,
651
            data_range=None,
652
            version=None,
653
            if_etag_match=None,
654
            if_etag_not_match=None,
655
            if_modified_since=None,
656
            if_unmodified_since=None)
657
        kwargs = dict(
658
            version='s0m3v3r51on',
659
            if_match='if match',
660
            if_none_match='if non match',
661
            if_modified_since='some date here',
662
            if_unmodified_since='some date here',
663
            data_range='10-20')
664
        with patch.object(PC, 'object_get', return_value=FR()) as get:
665
            r = self.client.get_object_hashmap(obj)
666
            self.assertEqual(r, object_hashmap)
667
            self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
668
            r = self.client.get_object_hashmap(obj, **kwargs)
669
            exp_args['if_etag_match'] = kwargs.pop('if_match')
670
            exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
671
            exp_args.update(kwargs)
672
            self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
673

    
674
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
675
    def test_set_account_group(self, post):
676
        (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
677
        self.client.set_account_group(group, usernames)
678
        self.assertEqual(
679
            post.mock_calls[-1],
680
            call(update=True, groups={group: usernames}))
681

    
682
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
683
    def test_del_account_group(self, post):
684
        group = 'aU53rGr0up'
685
        self.client.del_account_group(group)
686
        self.assertEqual(
687
            post.mock_calls[-1],
688
            call(update=True, groups={group: []}))
689

    
690
    @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
691
    def test_get_account_quota(self, GAI):
692
        key = 'x-account-policy-quota'
693
        r = self.client.get_account_quota()
694
        self.assertEqual(r[key], account_info[key])
695

    
696
    @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
697
    def test_get_account_versioning(self, GAI):
698
        key = 'x-account-policy-versioning'
699
        r = self.client.get_account_versioning()
700
        self.assertEqual(r[key], account_info[key])
701

    
702
    def test_get_account_meta(self):
703
        key = 'x-account-meta-'
704
        with patch.object(PC, 'get_account_info', return_value=account_info):
705
            r = self.client.get_account_meta()
706
            keys = [k for k in r if k.startswith(key)]
707
            self.assertFalse(keys)
708
        acc_info = dict(account_info)
709
        acc_info['%sk1' % key] = 'v1'
710
        acc_info['%sk2' % key] = 'v2'
711
        acc_info['%sk3' % key] = 'v3'
712
        with patch.object(PC, 'get_account_info', return_value=acc_info):
713
            r = self.client.get_account_meta()
714
            for k in [k for k in acc_info if k.startswith(key)]:
715
                self.assertEqual(r[k], acc_info[k])
716

    
717
    def test_get_account_group(self):
718
        key = 'x-account-group-'
719
        with patch.object(PC, 'get_account_info', return_value=account_info):
720
            r = self.client.get_account_group()
721
            keys = [k for k in r if k.startswith(key)]
722
            self.assertFalse(keys)
723
        acc_info = dict(account_info)
724
        acc_info['%sk1' % key] = 'g1'
725
        acc_info['%sk2' % key] = 'g2'
726
        acc_info['%sk3' % key] = 'g3'
727
        with patch.object(PC, 'get_account_info', return_value=acc_info):
728
            r = self.client.get_account_group()
729
            for k in [k for k in acc_info if k.startswith(key)]:
730
                self.assertEqual(r[k], acc_info[k])
731

    
732
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
733
    def test_set_account_meta(self, post):
734
        metas = dict(k1='v1', k2='v2', k3='v3')
735
        self.client.set_account_meta(metas)
736
        self.assertEqual(
737
            post.mock_calls[-1],
738
            call(update=True, metadata=metas))
739

    
740
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
741
    def test_set_account_quota(self, post):
742
        qu = 1024
743
        self.client.set_account_quota(qu)
744
        self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
745

    
746
    @patch('%s.account_post' % pithos_pkg, return_value=FR())
747
    def test_set_account_versioning(self, post):
748
        vrs = 'n3wV3r51on1ngTyp3'
749
        self.client.set_account_versioning(vrs)
750
        self.assertEqual(
751
            post.mock_calls[-1],
752
            call(update=True, versioning=vrs))
753

    
754
    @patch('%s.container_delete' % pithos_pkg, return_value=FR())
755
    def test_del_container(self, delete):
756
        for kwarg in (
757
                dict(delimiter=None, until=None),
758
                dict(delimiter='X', until='50m3d473')):
759
            self.client.del_container(**kwarg)
760
            expected = dict(kwarg)
761
            expected['success'] = (204, 404, 409)
762
            self.assertEqual(delete.mock_calls[-1], call(**expected))
763
        for status_code in (404, 409):
764
            FR.status_code = status_code
765
            self.assertRaises(ClientError, self.client.del_container)
766

    
767
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
768
    def test_get_container_versioning(self, GCI):
769
        key = 'x-container-policy-versioning'
770
        cont = 'c0n7-417'
771
        bu_cnt = self.client.container
772
        for container in (None, cont):
773
            r = self.client.get_container_versioning(container=container)
774
            self.assertEqual(r[key], container_info[key])
775
            self.assertEqual(GCI.mock_calls[-1], call())
776
            self.assertEqual(bu_cnt, self.client.container)
777

    
778
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
779
    def test_get_container_quota(self, GCI):
780
        key = 'x-container-policy-quota'
781
        cont = 'c0n7-417'
782
        bu_cnt = self.client.container
783
        for container in (None, cont):
784
            r = self.client.get_container_quota(container=container)
785
            self.assertEqual(r[key], container_info[key])
786
            self.assertEqual(GCI.mock_calls[-1], call())
787
            self.assertEqual(bu_cnt, self.client.container)
788

    
789
    def test_get_container_meta(self):
790
        somedate = '50m3d473'
791
        key = 'x-container-meta'
792
        metaval = '50m3m374v41'
793
        container_plus = dict(container_info)
794
        container_plus[key] = metaval
795
        for ret in ((container_info, {}), (container_plus, {key: metaval})):
796
            with patch.object(
797
                    PC,
798
                    'get_container_info',
799
                    return_value=ret[0]) as gci:
800
                for until in (None, somedate):
801
                    r = self.client.get_container_meta(until=until)
802
                    self.assertEqual(r, ret[1])
803
                    self.assertEqual(gci.mock_calls[-1], call(until=until))
804

    
805
    def test_get_container_object_meta(self):
806
        somedate = '50m3d473'
807
        key = 'x-container-object-meta'
808
        metaval = '50m3m374v41'
809
        container_plus = dict(container_info)
810
        container_plus[key] = metaval
811
        for ret in (
812
                (container_info, {key: ''}),
813
                (container_plus, {key: metaval})):
814
            with patch.object(
815
                    PC,
816
                    'get_container_info',
817
                    return_value=ret[0]) as gci:
818
                for until in (None, somedate):
819
                    r = self.client.get_container_object_meta(until=until)
820
                    self.assertEqual(r, ret[1])
821
                    self.assertEqual(gci.mock_calls[-1], call(until=until))
822

    
823
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
824
    def test_set_container_meta(self, post):
825
        metas = dict(k1='v1', k2='v2', k3='v3')
826
        self.client.set_container_meta(metas)
827
        self.assertEqual(
828
            post.mock_calls[-1],
829
            call(update=True, metadata=metas))
830

    
831
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
832
    def test_del_container_meta(self, ap):
833
        self.client.del_container_meta('somekey')
834
        expected = [call(update=True, metadata={'somekey': ''})]
835
        self.assertEqual(ap.mock_calls, expected)
836

    
837
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
838
    def test_set_container_quota(self, post):
839
        qu = 1024
840
        self.client.set_container_quota(qu)
841
        self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
842

    
843
    @patch('%s.container_post' % pithos_pkg, return_value=FR())
844
    def test_set_container_versioning(self, post):
845
        vrs = 'n3wV3r51on1ngTyp3'
846
        self.client.set_container_versioning(vrs)
847
        self.assertEqual(
848
            post.mock_calls[-1],
849
            call(update=True, versioning=vrs))
850

    
851
    @patch('%s.object_delete' % pithos_pkg, return_value=FR())
852
    def test_del_object(self, delete):
853
        for kwarg in (
854
                dict(delimiter=None, until=None),
855
                dict(delimiter='X', until='50m3d473')):
856
            self.client.del_object(obj, **kwarg)
857
            self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
858

    
859
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
860
    def test_set_object_meta(self, post):
861
        metas = dict(k1='v1', k2='v2', k3='v3')
862
        self.assertRaises(
863
            AssertionError,
864
            self.client.set_object_meta,
865
            obj, 'Non dict arg')
866
        self.client.set_object_meta(obj, metas)
867
        self.assertEqual(
868
            post.mock_calls[-1],
869
            call(obj, update=True, metadata=metas))
870

    
871
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
872
    def test_publish_object(self, post):
873
        oinfo = dict(object_info)
874
        val = 'pubL1c'
875
        oinfo['x-object-public'] = val
876
        with patch.object(PC, 'get_object_info', return_value=oinfo) as gof:
877
            r = self.client.publish_object(obj)
878
            self.assertEqual(
879
                post.mock_calls[-1],
880
                call(obj, public=True, update=True))
881
            self.assertEqual(gof.mock_calls[-1], call(obj))
882
            self.assertEqual(r, '%s%s' % (self.url[:-6], val))
883

    
884
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
885
    def test_unpublish_object(self, post):
886
        self.client.unpublish_object(obj)
887
        self.assertEqual(
888
            post.mock_calls[-1],
889
            call(obj, public=False, update=True))
890

    
891
    def test_get_object_sharing(self):
892
        info = dict(object_info)
893
        expected = dict(read='u1,g1,u2', write='u1')
894
        info['x-object-sharing'] = '; '.join(
895
            ['%s=%s' % (k, v) for k, v in expected.items()])
896
        with patch.object(PC, 'get_object_info', return_value=info) as GOF:
897
            r = self.client.get_object_sharing(obj)
898
            self.assertEqual(GOF.mock_calls[-1], call(obj))
899
            self.assert_dicts_are_equal(r, expected)
900
            info['x-object-sharing'] = '//'.join(
901
                ['%s=%s' % (k, v) for k, v in expected.items()])
902
            self.assertRaises(
903
                ValueError,
904
                self.client.get_object_sharing,
905
                obj)
906
            info['x-object-sharing'] = '; '.join(
907
                ['%s:%s' % (k, v) for k, v in expected.items()])
908
            self.assertRaises(
909
                ClientError,
910
                self.client.get_object_sharing,
911
                obj)
912
            info['x-object-sharing'] = 'read=%s' % expected['read']
913
            r = self.client.get_object_sharing(obj)
914
            expected.pop('write')
915
            self.assert_dicts_are_equal(r, expected)
916

    
917
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
918
    def test_set_object_sharing(self, POST):
919
        read_perms = ['u1', 'g1', 'u2', 'g2']
920
        write_perms = ['u1', 'g1']
921
        for kwargs in (
922
                dict(read_permition=read_perms, write_permition=write_perms),
923
                dict(read_permition=read_perms),
924
                dict(write_permition=write_perms),
925
                dict()):
926
            self.client.set_object_sharing(obj, **kwargs)
927
            kwargs['read'] = kwargs.pop('read_permition', '')
928
            kwargs['write'] = kwargs.pop('write_permition', '')
929
            self.assertEqual(
930
                POST.mock_calls[-1],
931
                call(obj, update=True, permissions=kwargs))
932

    
933
    @patch('%s.set_object_sharing' % pithos_pkg)
934
    def test_del_object_sharing(self, SOS):
935
        self.client.del_object_sharing(obj)
936
        self.assertEqual(SOS.mock_calls[-1], call(obj))
937

    
938
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
939
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
940
    def test_append_object(self, post, GCI):
941
        num_of_blocks = 4
942
        tmpFile = self._create_temp_file(num_of_blocks)
943
        tmpFile.seek(0, 2)
944
        file_size = tmpFile.tell()
945
        for turn in range(2):
946
            tmpFile.seek(0, 0)
947

    
948
            try:
949
                from progress.bar import ShadyBar
950
                apn_bar = ShadyBar('Mock append')
951
            except ImportError:
952
                apn_bar = None
953

    
954
            if apn_bar:
955

    
956
                def append_gen(n):
957
                    for i in apn_bar.iter(range(n)):
958
                        yield
959
                    yield
960

    
961
            else:
962
                append_gen = None
963

    
964
            self.client.append_object(
965
                obj, tmpFile,
966
                upload_cb=append_gen if turn else None)
967
            self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
968
            (args, kwargs) = post.mock_calls[-1][1:3]
969
            self.assertEqual(args, (obj,))
970
            self.assertEqual(kwargs['content_length'], len(kwargs['data']))
971
            fsize = num_of_blocks * int(kwargs['content_length'])
972
            self.assertEqual(fsize, file_size)
973
            self.assertEqual(kwargs['content_range'], 'bytes */*')
974
            exp = 'application/octet-stream'
975
            self.assertEqual(kwargs['content_type'], exp)
976
            self.assertEqual(kwargs['update'], True)
977

    
978
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
979
    def test_truncate_object(self, post):
980
        upto_bytes = 377
981
        self.client.truncate_object(obj, upto_bytes)
982
        self.assertEqual(post.mock_calls[-1], call(
983
            obj,
984
            update=True,
985
            object_bytes=upto_bytes,
986
            content_range='bytes 0-%s/*' % upto_bytes,
987
            content_type='application/octet-stream',
988
            source_object='/%s/%s' % (self.client.container, obj)))
989

    
990
    @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
991
    @patch('%s.object_post' % pithos_pkg, return_value=FR())
992
    def test_overwrite_object(self, post, GCI):
993
        num_of_blocks = 4
994
        tmpFile = self._create_temp_file(num_of_blocks)
995
        tmpFile.seek(0, 2)
996
        file_size = tmpFile.tell()
997
        info = dict(object_info)
998
        info['content-length'] = file_size
999
        block_size = container_info['x-container-block-size']
1000
        with patch.object(PC, 'get_object_info', return_value=info) as GOI:
1001
            for start, end in (
1002
                    (0, file_size + 1),
1003
                    (file_size + 1, file_size + 2)):
1004
                tmpFile.seek(0, 0)
1005
                self.assertRaises(
1006
                    ClientError,
1007
                    self.client.overwrite_object,
1008
                    obj, start, end, tmpFile)
1009
            for start, end in ((0, 144), (144, 233), (233, file_size)):
1010
                tmpFile.seek(0, 0)
1011
                owr_gen = None
1012
                exp_size = end - start + 1
1013
                if not start or exp_size > block_size:
1014
                    try:
1015
                        from progress.bar import ShadyBar
1016
                        owr_bar = ShadyBar('Mock append')
1017
                    except ImportError:
1018
                        owr_bar = None
1019

    
1020
                    if owr_bar:
1021

    
1022
                        def owr_gen(n):
1023
                            for i in owr_bar.iter(range(n)):
1024
                                yield
1025
                            yield
1026

    
1027
                    if exp_size > block_size:
1028
                        exp_size = exp_size % block_size or block_size
1029

    
1030
                self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
1031
                self.assertEqual(GOI.mock_calls[-1], call(obj))
1032
                self.assertEqual(GCI.mock_calls[-1], call())
1033
                (args, kwargs) = post.mock_calls[-1][1:3]
1034
                self.assertEqual(args, (obj,))
1035
                self.assertEqual(len(kwargs['data']), exp_size)
1036
                self.assertEqual(kwargs['content_length'], exp_size)
1037
                self.assertEqual(kwargs['update'], True)
1038
                exp = 'application/octet-stream'
1039
                self.assertEqual(kwargs['content_type'], exp)
1040

    
1041
    @patch('%s.set_param' % client_pkg)
1042
    @patch('%s.get' % pithos_pkg, return_value=FR())
1043
    def test_get_sharing_accounts(self, get, SP):
1044
        FR.json = sharers
1045
        for kws in (
1046
                dict(),
1047
                dict(limit='50m3-11m17'),
1048
                dict(marker='X'),
1049
                dict(limit='50m3-11m17', marker='X')):
1050
            r = self.client.get_sharing_accounts(**kws)
1051
            self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
1052
            limit, marker = kws.get('limit', None), kws.get('marker', None)
1053
            self.assertEqual(SP.mock_calls[-2], call(
1054
                'limit', limit,
1055
                iff=limit is not None))
1056
            self.assertEqual(SP.mock_calls[-1], call(
1057
                'marker', marker,
1058
                iff=marker is not None))
1059
            self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1060
            for i in range(len(r)):
1061
                self.assert_dicts_are_equal(r[i], sharers[i])
1062

    
1063
    @patch('%s.object_get' % pithos_pkg, return_value=FR())
1064
    def test_get_object_versionlist(self, get):
1065
        info = dict(object_info)
1066
        info['versions'] = ['v1', 'v2']
1067
        FR.json = info
1068
        r = self.client.get_object_versionlist(obj)
1069
        self.assertEqual(
1070
            get.mock_calls[-1],
1071
            call(obj, format='json', version='list'))
1072
        self.assertEqual(r, info['versions'])
1073

    
1074
if __name__ == '__main__':
1075
    from sys import argv
1076
    from kamaki.clients.test import runTestCase
1077
    runTestCase(Pithos, 'Pithos+ Client', argv[1:])