code.grnet.gr Git - kamaki/blob - kamaki/clients/pithos/test.py

   1 # Copyright 2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from unittest import TestCase
  35 from mock import patch, call
  36 from tempfile import NamedTemporaryFile
  37 from os import urandom
  38
  39 from kamaki.clients import ClientError
  40 from kamaki.clients.pithos import PithosClient as PC
  41
  42 pithos_pkg = 'kamaki.clients.pithos.PithosClient'
  43
  44 user_id = 'ac0un7-1d-5tr1ng'
  45 obj = 'obj3c7N4m3'
  46
  47 account_info = {
  48     'content-language': 'en-us',
  49     'content-type': 'text/html; charset=utf-8',
  50     'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
  51     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  52     'server': 'gunicorn/0.14.5',
  53     'vary': 'Accept-Language',
  54     'x-account-bytes-used': '751615526',
  55     'x-account-container-count': 7,
  56     'x-account-policy-quota': 53687091200,
  57     'x-account-policy-versioning': 'auto'}
  58 container_info = {
  59     'content-language': 'en-us',
  60     'content-type': 'text/html; charset=utf-8',
  61     'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
  62     'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
  63     'server': 'gunicorn/0.14.5',
  64     'vary': 'Accept-Language',
  65     'x-container-block-hash': 'sha256',
  66     'x-container-block-size': 4194304,
  67     'x-container-bytes-used': 309528938,
  68     'x-container-object-count': 14,
  69     'x-container-object-meta': '',
  70     'x-container-policy-quota': 53687091200,
  71     'x-container-policy-versioning': 'auto'}
  72 object_info = {
  73     'content-language': 'en-us',
  74     'content-length': 254965,
  75     'content-type': 'application/octet-stream',
  76     'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
  77     'etag': '',
  78     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  79     'server': 'gunicorn/0.14.5',
  80     'vary': 'Accept-Language',
  81     'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
  82     'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
  83     'x-object-version': '525996',
  84     'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
  85     'x-object-meta-k1': 'v1',
  86     'x-object-meta-k2': 'v2'}
  87 container_list = [
  88     dict(
  89         count=2,
  90         last_modified="2013-02-27T11:56:09.893033+00:00",
  91         bytes=677076979,
  92         name="pithos",
  93         x_container_policy=dict(quota="21474836480", versioning="auto")),
  94     dict(
  95         count=0,
  96         last_modified="2012-10-23T12:25:17.229187+00:00",
  97         bytes=0,
  98         name="trash",
  99         x_container_policy=dict(quota="21474836480", versioning="auto"))]
 100 object_list = [
 101     dict(hash="",
 102         name="The_Secret_Garden.zip",
 103         x_object_public="/public/wdp9p",
 104         bytes=203304947,
 105         x_object_version_timestamp="1360237915.7027509",
 106         x_object_uuid="s0m3uu1df0r0bj0n3",
 107         last_modified="2013-02-07T11:51:55.702751+00:00",
 108         content_type="application/octet-stream",
 109         x_object_hash="0afdf29f71cd53126225c3f54ca",
 110         x_object_version=17737,
 111         x_object_modified_by=user_id),
 112     dict(hash="",
 113         name="The_Revealed_Garden.zip",
 114         x_object_public="/public/wpd7p",
 115         bytes=20330947,
 116         x_object_version_timestamp="13602915.7027509",
 117         x_object_uuid="s0m3uu1df0r0bj70w",
 118         last_modified="2013-02-07T11:51:55.702751+00:00",
 119         content_type="application/octet-stream",
 120         x_object_hash="0afdf29f71cd53126225c3f54ca",
 121         x_object_version=17737,
 122         x_object_modified_by=user_id)]
 123 object_hashmap = dict(
 124     block_hash="sha256", block_size=4194304, bytes=33554432,
 125     hashes=[
 126         "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
 127         "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
 128         "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
 129         "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
 130         "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
 131         "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
 132         "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
 133         "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
 134 sharers = [
 135     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
 136     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
 137     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
 138
 139
 140 class FR(object):
 141     """FR stands for Fake Response"""
 142     json = dict()
 143     headers = dict()
 144     content = json
 145     status = None
 146     status_code = 200
 147
 148     def release(self):
 149         pass
 150
 151
 152 class Pithos(TestCase):
 153
 154     files = []
 155
 156     def _create_temp_file(self, num_of_blocks):
 157         self.files.append(NamedTemporaryFile())
 158         tmpFile = self.files[-1]
 159         file_size = num_of_blocks * 4 * 1024 * 1024
 160         print('\n\tCreate tmp file')
 161         tmpFile.write(urandom(file_size))
 162         tmpFile.flush()
 163         tmpFile.seek(0)
 164         print('\t\tDone')
 165         return tmpFile
 166
 167     def assert_dicts_are_equal(self, d1, d2):
 168         for k, v in d1.items():
 169             self.assertTrue(k in d2)
 170             if isinstance(v, dict):
 171                 self.assert_dicts_are_equal(v, d2[k])
 172             else:
 173                 self.assertEqual(unicode(v), unicode(d2[k]))
 174
 175     def setUp(self):
 176         self.url = 'https://www.example.com/pithos'
 177         self.token = 'p17h0570k3n'
 178         self.client = PC(self.url, self.token)
 179         self.client.account = user_id
 180         self.client.container = 'c0nt@1n3r_i'
 181
 182     def tearDown(self):
 183         FR.headers = dict()
 184         FR.status_code = 200
 185         FR.json = dict()
 186         FR.content = FR.json
 187         for f in self.files:
 188             f.close()
 189
 190     #  Pithos+ methods that extend storage API
 191
 192     @patch('%s.account_head' % pithos_pkg, return_value=FR())
 193     def test_get_account_info(self, AH):
 194         FR.headers = account_info
 195         r = self.client.get_account_info()
 196         self.assert_dicts_are_equal(r, account_info)
 197         self.assertEqual(AH.mock_calls[-1], call(until=None))
 198         unt = 'un71L-d473'
 199         r = self.client.get_account_info(until=unt)
 200         self.assert_dicts_are_equal(r, account_info)
 201         self.assertEqual(AH.mock_calls[-1], call(until=unt))
 202         FR.status_code = 401
 203         self.assertRaises(ClientError, self.client.get_account_info)
 204
 205     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 206     def test_del_account_meta(self, ap):
 207         keys = ['k1', 'k2', 'k3']
 208         expected = []
 209         for key in keys:
 210             self.client.del_account_meta(key)
 211             expected.append(call(update=True, metadata={key: ''}))
 212         self.assertEqual(ap.mock_calls, expected)
 213
 214     @patch('%s.container_head' % pithos_pkg, return_value=FR())
 215     def test_get_container_info(self, ch):
 216         FR.headers = container_info
 217         r = self.client.get_container_info()
 218         self.assert_dicts_are_equal(r, container_info)
 219         u = 'some date'
 220         r = self.client.get_container_info(until=u)
 221         self.assertEqual(ch.mock_calls, [call(until=None), call(until=u)])
 222
 223     @patch('%s.account_get' % pithos_pkg, return_value=FR())
 224     def test_list_containers(self, get):
 225         FR.json = container_list
 226         r = self.client.list_containers()
 227         for i in range(len(r)):
 228             self.assert_dicts_are_equal(r[i], container_list[i])
 229
 230     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 231     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 232     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 233     def test_upload_object(self, CI, CP, OP):
 234         num_of_blocks = 8
 235         tmpFile = self._create_temp_file(num_of_blocks)
 236
 237         # Without kwargs
 238         self.client.upload_object(obj, tmpFile)
 239         self.assertEqual(PC.get_container_info.mock_calls, [call()])
 240         [call1, call2] = PC.object_put.mock_calls
 241
 242         (args1, kwargs1) = call1[1:3]
 243         (args2, kwargs2) = call2[1:3]
 244         self.assertEqual(args1, (obj,))
 245         expected1 = dict(
 246             hashmap=True,
 247             success=(201, 409),
 248             format='json',
 249             json=dict(
 250                 hashes=['s0m3h@5h'] * num_of_blocks,
 251                 bytes=num_of_blocks * 4 * 1024 * 1024),
 252             etag=None,
 253             content_encoding=None,
 254             content_type='application/octet-stream',
 255             content_disposition=None,
 256             public=None,
 257             permissions=None)
 258         for k, v in expected1.items():
 259             if k == 'json':
 260                 self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
 261                 self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
 262             else:
 263                 self.assertEqual(v, kwargs1[k])
 264
 265         (args2, kwargs2) = call2[1:3]
 266         self.assertEqual(args2, (obj,))
 267         expected2 = dict(
 268             json=dict(
 269                 hashes=['s0m3h@5h'] * num_of_blocks,
 270                 bytes=num_of_blocks * 4 * 1024 * 1024),
 271             content_type='application/octet-stream',
 272             hashmap=True,
 273             success=201,
 274             format='json')
 275         for k, v in expected2.items():
 276             if k == 'json':
 277                 self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
 278                 self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
 279             else:
 280                 self.assertEqual(v, kwargs2[k])
 281
 282         OP = PC.object_put
 283         mock_offset = 2
 284
 285         #  With progress bars
 286         try:
 287             from progress.bar import ShadyBar
 288             blck_bar = ShadyBar('Mock blck calc.')
 289             upld_bar = ShadyBar('Mock uplds')
 290         except ImportError:
 291             blck_bar = None
 292             upld_bar = None
 293
 294         if blck_bar and upld_bar:
 295
 296             def blck_gen(n):
 297                 for i in blck_bar.iter(range(n)):
 298                     yield
 299                 yield
 300
 301             def upld_gen(n):
 302                 for i in upld_bar.iter(range(n)):
 303                     yield
 304                 yield
 305
 306             tmpFile.seek(0)
 307             self.client.upload_object(
 308                 obj, tmpFile,
 309                 hash_cb=blck_gen, upload_cb=upld_gen)
 310
 311             for i, c in enumerate(OP.mock_calls[-mock_offset:]):
 312                 self.assertEqual(OP.mock_calls[i], c)
 313
 314         #  With content-type
 315         tmpFile.seek(0)
 316         ctype = 'video/mpeg'
 317         sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
 318         self.client.upload_object(obj, tmpFile,
 319             content_type=ctype, sharing=sharing)
 320         self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
 321         self.assert_dicts_are_equal(
 322             OP.mock_calls[-2][2]['permissions'],
 323             sharing)
 324
 325         # With other args
 326         tmpFile.seek(0)
 327         kwargs = dict(
 328             etag='s0m3E74g',
 329             content_type=ctype,
 330             content_disposition=ctype + 'd15p051710n',
 331             public=True,
 332             content_encoding='802.11')
 333         self.client.upload_object(obj, tmpFile, **kwargs)
 334         for arg, val in kwargs.items():
 335             self.assertEqual(OP.mock_calls[-2][2][arg], val)
 336
 337     def test_get_object_info(self):
 338         FR.headers = object_info
 339         version = 'v3r510n'
 340         with patch.object(PC, 'object_head', return_value=FR()) as head:
 341             r = self.client.get_object_info(obj)
 342             self.assertEqual(r, object_info)
 343             r = self.client.get_object_info(obj, version=version)
 344             self.assertEqual(head.mock_calls, [
 345                 call(obj, version=None),
 346                 call(obj, version=version)])
 347         with patch.object(
 348                 PC, 'object_head',
 349                 side_effect=ClientError('Obj not found', 404)):
 350             self.assertRaises(
 351                 ClientError,
 352                 self.client.get_object_info,
 353                 obj, version=version)
 354
 355     @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
 356     def test_get_object_meta(self, GOI):
 357         for version in (None, 'v3r510n'):
 358             r = self.client.get_object_meta(obj, version)
 359             for k in [k for k in object_info if k.startswith('x-object-meta')]:
 360                 self.assertEqual(r.pop(k), object_info[k])
 361             self.assertFalse(len(r))
 362             self.assertEqual(GOI.mock_calls[-1], call(obj, version=version))
 363
 364     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 365     def test_del_object_meta(self, post):
 366         metakey = '50m3m3t4k3y'
 367         self.client.del_object_meta(obj, metakey)
 368         expected = call(obj, update=True, metadata={metakey: ''})
 369         self.assertEqual(post.mock_calls[-1], expected)
 370
 371     @patch('%s.post' % pithos_pkg, return_value=FR())
 372     @patch('%s.set_header' % pithos_pkg)
 373     def test_replace_object_meta(self, SH, post):
 374         metas = dict(k1='new1', k2='new2', k3='new3')
 375         cont = self.client.container
 376         self.client.replace_object_meta(metas)
 377         expected = call('/%s/%s' % (user_id, cont), success=202)
 378         self.assertEqual(post.mock_calls[-1], expected)
 379         prfx = 'X-Object-Meta-'
 380         expected = [call('%s%s' % (prfx, k), v) for k, v in metas.items()]
 381         self.assertEqual(PC.set_header.mock_calls, expected)
 382
 383     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 384     def test_copy_object(self, put):
 385         src_cont = 'src-c0nt41n3r'
 386         src_obj = 'src-0bj'
 387         dst_cont = 'dst-c0nt41n3r'
 388         dst_obj = 'dst-0bj'
 389         expected = call(
 390             src_obj,
 391             content_length=0,
 392             source_account=None,
 393             success=201,
 394             copy_from='/%s/%s' % (src_cont, src_obj),
 395             delimiter=None,
 396             content_type=None,
 397             source_version=None,
 398             public=False)
 399         self.client.copy_object(src_cont, src_obj, dst_cont)
 400         self.assertEqual(put.mock_calls[-1], expected)
 401         self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
 402         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 403         kwargs = dict(
 404             source_version='src-v3r510n',
 405             source_account='src-4cc0un7',
 406             public=True,
 407             content_type='c0n73n7Typ3',
 408             delimiter='5')
 409         self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
 410         for k, v in kwargs.items():
 411             self.assertEqual(v, put.mock_calls[-1][2][k])
 412
 413     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 414     def test_move_object(self, put):
 415         src_cont = 'src-c0nt41n3r'
 416         src_obj = 'src-0bj'
 417         dst_cont = 'dst-c0nt41n3r'
 418         dst_obj = 'dst-0bj'
 419         expected = call(
 420             src_obj,
 421             content_length=0,
 422             source_account=None,
 423             success=201,
 424             move_from='/%s/%s' % (src_cont, src_obj),
 425             delimiter=None,
 426             content_type=None,
 427             source_version=None,
 428             public=False)
 429         self.client.move_object(src_cont, src_obj, dst_cont)
 430         self.assertEqual(put.mock_calls[-1], expected)
 431         self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
 432         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 433         kwargs = dict(
 434             source_version='src-v3r510n',
 435             source_account='src-4cc0un7',
 436             public=True,
 437             content_type='c0n73n7Typ3',
 438             delimiter='5')
 439         self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
 440         for k, v in kwargs.items():
 441             self.assertEqual(v, put.mock_calls[-1][2][k])
 442
 443     @patch('%s.delete' % pithos_pkg, return_value=FR())
 444     def test_delete_object(self, delete):
 445         cont = self.client.container
 446         self.client.delete_object(obj)
 447         self.assertEqual(
 448             delete.mock_calls[-1],
 449             call('/%s/%s/%s' % (user_id, cont, obj), success=(204, 404)))
 450         FR.status_code = 404
 451         self.assertRaises(ClientError, self.client.delete_object, obj)
 452
 453     @patch('%s.get' % pithos_pkg, return_value=FR())
 454     @patch('%s.set_param' % pithos_pkg)
 455     def test_list_objects(self, SP, get):
 456         FR.json = object_list
 457         acc = self.client.account
 458         cont = self.client.container
 459         SP = PC.set_param
 460         r = self.client.list_objects()
 461         for i in range(len(r)):
 462             self.assert_dicts_are_equal(r[i], object_list[i])
 463         self.assertEqual(get.mock_calls, [
 464             call('/%s/%s' % (acc, cont), success=(200, 204, 304, 404))])
 465         self.assertEqual(SP.mock_calls, [call('format', 'json')])
 466         FR.status_code = 304
 467         self.assertEqual(self.client.list_objects(), [])
 468         FR.status_code = 404
 469         self.assertRaises(ClientError, self.client.list_objects)
 470
 471     @patch('%s.get' % pithos_pkg, return_value=FR())
 472     @patch('%s.set_param' % pithos_pkg)
 473     def test_list_objects_in_path(self, SP, get):
 474         FR.json = object_list
 475         path = '/some/awsome/path'
 476         acc = self.client.account
 477         cont = self.client.container
 478         SP = PC.set_param
 479         self.client.list_objects_in_path(path)
 480         self.assertEqual(get.mock_calls, [
 481             call('/%s/%s' % (acc, cont), success=(200, 204, 404))])
 482         self.assertEqual(SP.mock_calls, [
 483             call('format', 'json'), call('path', path)])
 484         FR.status_code = 404
 485         self.assertRaises(ClientError, self.client.list_objects)
 486
 487     #  Pithos+ only methods
 488
 489     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 490     def test_purge_container(self, cd):
 491         self.client.purge_container()
 492         self.assertTrue('until' in cd.mock_calls[-1][2])
 493         cont = self.client.container
 494         self.client.purge_container('another-container')
 495         self.assertEqual(self.client.container, cont)
 496
 497     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 498     def test_upload_object_unchunked(self, put):
 499         num_of_blocks = 8
 500         tmpFile = self._create_temp_file(num_of_blocks)
 501         expected = dict(
 502                 success=201,
 503                 data=num_of_blocks * 4 * 1024 * 1024,
 504                 etag='some-etag',
 505                 content_encoding='some content_encoding',
 506                 content_type='some content-type',
 507                 content_disposition='some content_disposition',
 508                 public=True,
 509                 permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 510         self.client.upload_object_unchunked(obj, tmpFile)
 511         self.assertEqual(put.mock_calls[-1][1], (obj,))
 512         self.assertEqual(
 513             sorted(put.mock_calls[-1][2].keys()),
 514             sorted(expected.keys()))
 515         kwargs = dict(expected)
 516         kwargs.pop('success')
 517         kwargs['size'] = kwargs.pop('data')
 518         kwargs['sharing'] = kwargs.pop('permissions')
 519         tmpFile.seek(0)
 520         self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
 521         pmc = put.mock_calls[-1][2]
 522         for k, v in expected.items():
 523             if k == 'data':
 524                 self.assertEqual(len(pmc[k]), v)
 525             else:
 526                 self.assertEqual(pmc[k], v)
 527         self.assertRaises(
 528             ClientError,
 529             self.client.upload_object_unchunked,
 530             obj, tmpFile, withHashFile=True)
 531
 532     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 533     def test_create_object_by_manifestation(self, put):
 534         manifest = '%s/%s' % (self.client.container, obj)
 535         kwargs = dict(
 536                 etag='some-etag',
 537                 content_encoding='some content_encoding',
 538                 content_type='some content-type',
 539                 content_disposition='some content_disposition',
 540                 public=True,
 541                 sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 542         self.client.create_object_by_manifestation(obj)
 543         expected = dict(content_length=0, manifest=manifest)
 544         for k in kwargs:
 545             expected['permissions' if k == 'sharing' else k] = None
 546         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 547         self.client.create_object_by_manifestation(obj, **kwargs)
 548         expected.update(kwargs)
 549         expected['permissions'] = expected.pop('sharing')
 550         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 551
 552     @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
 553     @patch('%s.object_get' % pithos_pkg, return_value=FR())
 554     def test_download_object(self, GOH, GET):
 555         num_of_blocks = 8
 556         tmpFile = self._create_temp_file(num_of_blocks)
 557         FR.content = tmpFile.read(4 * 1024 * 1024)
 558         tmpFile = self._create_temp_file(num_of_blocks)
 559         GET = PC.object_get
 560         num_of_blocks = len(object_hashmap['hashes'])
 561         kwargs = dict(
 562             resume=True,
 563             version='version',
 564             range_str='10-20',
 565             if_match='if and only if',
 566             if_none_match='if and only not',
 567             if_modified_since='what if not?',
 568             if_unmodified_since='this happens if not!',
 569             async_headers=dict(Range='bytes=0-88888888'))
 570
 571         self.client.download_object(obj, tmpFile)
 572         self.assertEqual(len(GET.mock_calls), num_of_blocks)
 573         self.assertEqual(GET.mock_calls[-1][1], (obj,))
 574         for k, v in kwargs.items():
 575             if k == 'async_headers':
 576                 self.assertTrue('Range' in GET.mock_calls[-1][2][k])
 577             elif k in ('resume', 'range_str'):
 578                 continue
 579             else:
 580                 self.assertEqual(GET.mock_calls[-1][2][k], None)
 581
 582         #  Check ranges are consecutive
 583         starts = []
 584         ends = []
 585         for c in GET.mock_calls:
 586             rng_str = c[2]['async_headers']['Range']
 587             (start, rng_str) = rng_str.split('=')
 588             (start, end) = rng_str.split('-')
 589             starts.append(start)
 590             ends.append(end)
 591         ends = sorted(ends)
 592         for i, start in enumerate(sorted(starts)):
 593             if i:
 594                 int(ends[i - 1]) == int(start) - 1
 595
 596         #  With progress bars
 597         try:
 598             from progress.bar import ShadyBar
 599             dl_bar = ShadyBar('Mock dl')
 600         except ImportError:
 601             dl_bar = None
 602
 603         if dl_bar:
 604
 605             def blck_gen(n):
 606                 for i in dl_bar.iter(range(n)):
 607                     yield
 608                 yield
 609
 610             tmpFile.seek(0)
 611             self.client.download_object(obj, tmpFile, download_cb=blck_gen)
 612             self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
 613
 614         tmpFile.seek(0)
 615         kwargs.pop('async_headers')
 616         kwargs.pop('resume')
 617         self.client.download_object(obj, tmpFile, **kwargs)
 618         for k, v in kwargs.items():
 619             if k == 'range_str':
 620                 self.assertEqual(
 621                     GET.mock_calls[-1][2]['data_range'],
 622                     'bytes=%s' % v)
 623             else:
 624                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 625
 626         #  ALl options on no tty
 627
 628         def foo():
 629             return True
 630
 631         tmpFile.seek(0)
 632         tmpFile.isatty = foo
 633         self.client.download_object(obj, tmpFile, **kwargs)
 634         for k, v in kwargs.items():
 635             if k == 'range_str':
 636                 self.assertTrue('data_range' in GET.mock_calls[-1][2])
 637             else:
 638                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 639
 640     def test_get_object_hashmap(self):
 641         FR.json = object_hashmap
 642         for empty in (304, 412):
 643             with patch.object(
 644                     PC, 'object_get',
 645                     side_effect=ClientError('Empty', status=empty)):
 646                 r = self.client.get_object_hashmap(obj)
 647                 self.assertEqual(r, {})
 648         exp_args = dict(
 649             hashmap=True,
 650             data_range=None,
 651             version=None,
 652             if_etag_match=None,
 653             if_etag_not_match=None,
 654             if_modified_since=None,
 655             if_unmodified_since=None)
 656         kwargs = dict(
 657             version='s0m3v3r51on',
 658             if_match='if match',
 659             if_none_match='if non match',
 660             if_modified_since='some date here',
 661             if_unmodified_since='some date here',
 662             data_range='10-20')
 663         with patch.object(PC, 'object_get', return_value=FR()) as get:
 664             r = self.client.get_object_hashmap(obj)
 665             self.assertEqual(r, object_hashmap)
 666             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 667             r = self.client.get_object_hashmap(obj, **kwargs)
 668             exp_args['if_etag_match'] = kwargs.pop('if_match')
 669             exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
 670             exp_args.update(kwargs)
 671             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 672
 673     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 674     def test_set_account_group(self, post):
 675         (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
 676         self.client.set_account_group(group, usernames)
 677         self.assertEqual(
 678             post.mock_calls[-1],
 679             call(update=True, groups={group: usernames}))
 680
 681     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 682     def test_del_account_group(self, post):
 683         group = 'aU53rGr0up'
 684         self.client.del_account_group(group)
 685         self.assertEqual(
 686             post.mock_calls[-1],
 687             call(update=True, groups={group: []}))
 688
 689     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 690     def test_get_account_quota(self, GAI):
 691         key = 'x-account-policy-quota'
 692         r = self.client.get_account_quota()
 693         self.assertEqual(r[key], account_info[key])
 694
 695     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 696     def test_get_account_versioning(self, GAI):
 697         key = 'x-account-policy-versioning'
 698         r = self.client.get_account_versioning()
 699         self.assertEqual(r[key], account_info[key])
 700
 701     def test_get_account_meta(self):
 702         key = 'x-account-meta-'
 703         with patch.object(PC, 'get_account_info', return_value=account_info):
 704             r = self.client.get_account_meta()
 705             keys = [k for k in r if k.startswith(key)]
 706             self.assertFalse(keys)
 707         acc_info = dict(account_info)
 708         acc_info['%sk1' % key] = 'v1'
 709         acc_info['%sk2' % key] = 'v2'
 710         acc_info['%sk3' % key] = 'v3'
 711         with patch.object(PC, 'get_account_info', return_value=acc_info):
 712             r = self.client.get_account_meta()
 713             for k in [k for k in acc_info if k.startswith(key)]:
 714                 self.assertEqual(r[k], acc_info[k])
 715
 716     def test_get_account_group(self):
 717         key = 'x-account-group-'
 718         with patch.object(PC, 'get_account_info', return_value=account_info):
 719             r = self.client.get_account_group()
 720             keys = [k for k in r if k.startswith(key)]
 721             self.assertFalse(keys)
 722         acc_info = dict(account_info)
 723         acc_info['%sk1' % key] = 'g1'
 724         acc_info['%sk2' % key] = 'g2'
 725         acc_info['%sk3' % key] = 'g3'
 726         with patch.object(PC, 'get_account_info', return_value=acc_info):
 727             r = self.client.get_account_group()
 728             for k in [k for k in acc_info if k.startswith(key)]:
 729                 self.assertEqual(r[k], acc_info[k])
 730
 731     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 732     def test_set_account_meta(self, post):
 733         metas = dict(k1='v1', k2='v2', k3='v3')
 734         self.client.set_account_meta(metas)
 735         self.assertEqual(
 736             post.mock_calls[-1],
 737             call(update=True, metadata=metas))
 738
 739     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 740     def test_set_account_quota(self, post):
 741         qu = 1024
 742         self.client.set_account_quota(qu)
 743         self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
 744
 745     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 746     def test_set_account_versioning(self, post):
 747         vrs = 'n3wV3r51on1ngTyp3'
 748         self.client.set_account_versioning(vrs)
 749         self.assertEqual(
 750             post.mock_calls[-1],
 751             call(update=True, versioning=vrs))
 752
 753     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 754     def test_del_container(self, delete):
 755         for kwarg in (
 756                 dict(delimiter=None, until=None),
 757                 dict(delimiter='X', until='50m3d473')):
 758             self.client.del_container(**kwarg)
 759             expected = dict(kwarg)
 760             expected['success'] = (204, 404, 409)
 761             self.assertEqual(delete.mock_calls[-1], call(**expected))
 762         for status_code in (404, 409):
 763             FR.status_code = status_code
 764             self.assertRaises(ClientError, self.client.del_container)
 765
 766     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 767     def test_get_container_versioning(self, GCI):
 768         key = 'x-container-policy-versioning'
 769         cont = 'c0n7-417'
 770         bu_cnt = self.client.container
 771         for container in (None, cont):
 772             r = self.client.get_container_versioning(container=container)
 773             self.assertEqual(r[key], container_info[key])
 774             self.assertEqual(GCI.mock_calls[-1], call())
 775             self.assertEqual(bu_cnt, self.client.container)
 776
 777     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 778     def test_get_container_quota(self, GCI):
 779         key = 'x-container-policy-quota'
 780         cont = 'c0n7-417'
 781         bu_cnt = self.client.container
 782         for container in (None, cont):
 783             r = self.client.get_container_quota(container=container)
 784             self.assertEqual(r[key], container_info[key])
 785             self.assertEqual(GCI.mock_calls[-1], call())
 786             self.assertEqual(bu_cnt, self.client.container)
 787
 788     def test_get_container_meta(self):
 789         somedate = '50m3d473'
 790         key = 'x-container-meta'
 791         metaval = '50m3m374v41'
 792         container_plus = dict(container_info)
 793         container_plus[key] = metaval
 794         for ret in ((container_info, {}), (container_plus, {key: metaval})):
 795             with patch.object(
 796                     PC,
 797                     'get_container_info',
 798                     return_value=ret[0]) as gci:
 799                 for until in (None, somedate):
 800                     r = self.client.get_container_meta(until=until)
 801                     self.assertEqual(r, ret[1])
 802                     self.assertEqual(gci.mock_calls[-1], call(until=until))
 803
 804     def test_get_container_object_meta(self):
 805         somedate = '50m3d473'
 806         key = 'x-container-object-meta'
 807         metaval = '50m3m374v41'
 808         container_plus = dict(container_info)
 809         container_plus[key] = metaval
 810         for ret in (
 811                 (container_info, {key: ''}),
 812                 (container_plus, {key: metaval})):
 813             with patch.object(
 814                     PC,
 815                     'get_container_info',
 816                     return_value=ret[0]) as gci:
 817                 for until in (None, somedate):
 818                     r = self.client.get_container_object_meta(until=until)
 819                     self.assertEqual(r, ret[1])
 820                     self.assertEqual(gci.mock_calls[-1], call(until=until))
 821
 822     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 823     def test_set_container_meta(self, post):
 824         metas = dict(k1='v1', k2='v2', k3='v3')
 825         self.client.set_container_meta(metas)
 826         self.assertEqual(
 827             post.mock_calls[-1],
 828             call(update=True, metadata=metas))
 829
 830     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 831     def test_del_container_meta(self, ap):
 832         self.client.del_container_meta('somekey')
 833         expected = [call(update=True, metadata={'somekey': ''})]
 834         self.assertEqual(ap.mock_calls, expected)
 835
 836     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 837     def test_set_container_quota(self, post):
 838         qu = 1024
 839         self.client.set_container_quota(qu)
 840         self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
 841
 842     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 843     def test_set_container_versioning(self, post):
 844         vrs = 'n3wV3r51on1ngTyp3'
 845         self.client.set_container_versioning(vrs)
 846         self.assertEqual(
 847             post.mock_calls[-1],
 848             call(update=True, versioning=vrs))
 849
 850     @patch('%s.object_delete' % pithos_pkg, return_value=FR())
 851     def test_del_object(self, delete):
 852         for kwarg in (
 853                 dict(delimiter=None, until=None),
 854                 dict(delimiter='X', until='50m3d473')):
 855             self.client.del_object(obj, **kwarg)
 856             self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
 857
 858     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 859     def test_set_object_meta(self, post):
 860         metas = dict(k1='v1', k2='v2', k3='v3')
 861         self.assertRaises(
 862             AssertionError,
 863             self.client.set_object_meta,
 864             obj, 'Non dict arg')
 865         self.client.set_object_meta(obj, metas)
 866         self.assertEqual(
 867             post.mock_calls[-1],
 868             call(obj, update=True, metadata=metas))
 869
 870     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 871     def test_publish_object(self, post):
 872         oinfo = dict(object_info)
 873         val = 'pubL1c'
 874         oinfo['x-object-public'] = val
 875         with patch.object(PC, 'get_object_info', return_value=oinfo) as gof:
 876             r = self.client.publish_object(obj)
 877             self.assertEqual(
 878                 post.mock_calls[-1],
 879                 call(obj, public=True, update=True))
 880             self.assertEqual(gof.mock_calls[-1], call(obj))
 881             self.assertEqual(r, '%s%s' % (self.url[:-6], val))
 882
 883     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 884     def test_unpublish_object(self, post):
 885         self.client.unpublish_object(obj)
 886         self.assertEqual(
 887             post.mock_calls[-1],
 888             call(obj, public=False, update=True))
 889
 890     def test_get_object_sharing(self):
 891         info = dict(object_info)
 892         expected = dict(read='u1,g1,u2', write='u1')
 893         info['x-object-sharing'] = '; '.join(
 894             ['%s=%s' % (k, v) for k, v in expected.items()])
 895         with patch.object(PC, 'get_object_info', return_value=info) as GOF:
 896             r = self.client.get_object_sharing(obj)
 897             self.assertEqual(GOF.mock_calls[-1], call(obj))
 898             self.assert_dicts_are_equal(r, expected)
 899             info['x-object-sharing'] = '//'.join(
 900                 ['%s=%s' % (k, v) for k, v in expected.items()])
 901             self.assertRaises(
 902                 ValueError,
 903                 self.client.get_object_sharing,
 904                 obj)
 905             info['x-object-sharing'] = '; '.join(
 906                 ['%s:%s' % (k, v) for k, v in expected.items()])
 907             self.assertRaises(
 908                 ClientError,
 909                 self.client.get_object_sharing,
 910                 obj)
 911             info['x-object-sharing'] = 'read=%s' % expected['read']
 912             r = self.client.get_object_sharing(obj)
 913             expected.pop('write')
 914             self.assert_dicts_are_equal(r, expected)
 915
 916     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 917     def test_set_object_sharing(self, POST):
 918         read_perms = ['u1', 'g1', 'u2', 'g2']
 919         write_perms = ['u1', 'g1']
 920         for kwargs in (
 921                 dict(read_permition=read_perms, write_permition=write_perms),
 922                 dict(read_permition=read_perms),
 923                 dict(write_permition=write_perms),
 924                 dict()):
 925             self.client.set_object_sharing(obj, **kwargs)
 926             kwargs['read'] = kwargs.pop('read_permition', '')
 927             kwargs['write'] = kwargs.pop('write_permition', '')
 928             self.assertEqual(
 929                 POST.mock_calls[-1],
 930                 call(obj, update=True, permissions=kwargs))
 931
 932     @patch('%s.set_object_sharing' % pithos_pkg)
 933     def test_del_object_sharing(self, SOS):
 934         self.client.del_object_sharing(obj)
 935         self.assertEqual(SOS.mock_calls[-1], call(obj))
 936
 937     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 938     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 939     def test_append_object(self, post, GCI):
 940         num_of_blocks = 4
 941         tmpFile = self._create_temp_file(num_of_blocks)
 942         tmpFile.seek(0, 2)
 943         file_size = tmpFile.tell()
 944         for turn in range(2):
 945             tmpFile.seek(0, 0)
 946
 947             try:
 948                 from progress.bar import ShadyBar
 949                 apn_bar = ShadyBar('Mock append')
 950             except ImportError:
 951                 apn_bar = None
 952
 953             if apn_bar:
 954
 955                 def append_gen(n):
 956                     for i in apn_bar.iter(range(n)):
 957                         yield
 958                     yield
 959
 960             else:
 961                 append_gen = None
 962
 963             self.client.append_object(
 964                 obj, tmpFile,
 965                 upload_cb=append_gen if turn else None)
 966             self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
 967             (args, kwargs) = post.mock_calls[-1][1:3]
 968             self.assertEqual(args, (obj,))
 969             self.assertEqual(kwargs['content_length'], len(kwargs['data']))
 970             fsize = num_of_blocks * int(kwargs['content_length'])
 971             self.assertEqual(fsize, file_size)
 972             self.assertEqual(kwargs['content_range'], 'bytes */*')
 973             exp = 'application/octet-stream'
 974             self.assertEqual(kwargs['content_type'], exp)
 975             self.assertEqual(kwargs['update'], True)
 976
 977     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 978     def test_truncate_object(self, post):
 979         upto_bytes = 377
 980         self.client.truncate_object(obj, upto_bytes)
 981         self.assertEqual(post.mock_calls[-1], call(
 982             obj,
 983             update=True,
 984             object_bytes=upto_bytes,
 985             content_range='bytes 0-%s/*' % upto_bytes,
 986             content_type='application/octet-stream',
 987             source_object='/%s/%s' % (self.client.container, obj)))
 988
 989     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 990     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 991     def test_overwrite_object(self, post, GCI):
 992         num_of_blocks = 4
 993         tmpFile = self._create_temp_file(num_of_blocks)
 994         tmpFile.seek(0, 2)
 995         file_size = tmpFile.tell()
 996         info = dict(object_info)
 997         info['content-length'] = file_size
 998         block_size = container_info['x-container-block-size']
 999         with patch.object(PC, 'get_object_info', return_value=info) as GOI:
1000             for start, end in (
1001                     (0, file_size + 1),
1002                     (file_size + 1, file_size + 2)):
1003                 tmpFile.seek(0, 0)
1004                 self.assertRaises(
1005                     ClientError,
1006                     self.client.overwrite_object,
1007                     obj, start, end, tmpFile)
1008             for start, end in ((0, 144), (144, 233), (233, file_size)):
1009                 tmpFile.seek(0, 0)
1010                 owr_gen = None
1011                 exp_size = end - start + 1
1012                 if not start or exp_size > block_size:
1013                     try:
1014                         from progress.bar import ShadyBar
1015                         owr_bar = ShadyBar('Mock append')
1016                     except ImportError:
1017                         owr_bar = None
1018
1019                     if owr_bar:
1020
1021                         def owr_gen(n):
1022                             for i in owr_bar.iter(range(n)):
1023                                 yield
1024                             yield
1025
1026                     if exp_size > block_size:
1027                         exp_size = exp_size % block_size or block_size
1028
1029                 self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
1030                 self.assertEqual(GOI.mock_calls[-1], call(obj))
1031                 self.assertEqual(GCI.mock_calls[-1], call())
1032                 (args, kwargs) = post.mock_calls[-1][1:3]
1033                 self.assertEqual(args, (obj,))
1034                 self.assertEqual(len(kwargs['data']), exp_size)
1035                 self.assertEqual(kwargs['content_length'], exp_size)
1036                 self.assertEqual(kwargs['update'], True)
1037                 exp = 'application/octet-stream'
1038                 self.assertEqual(kwargs['content_type'], exp)
1039
1040     @patch('%s.set_param' % pithos_pkg)
1041     @patch('%s.get' % pithos_pkg, return_value=FR())
1042     def test_get_sharing_accounts(self, get, SP):
1043         FR.json = sharers
1044         for kws in (
1045                 dict(),
1046                 dict(limit='50m3-11m17'),
1047                 dict(marker='X'),
1048                 dict(limit='50m3-11m17', marker='X')):
1049             r = self.client.get_sharing_accounts(**kws)
1050             self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
1051             limit, marker = kws.get('limit', None), kws.get('marker', None)
1052             self.assertEqual(SP.mock_calls[-2], call(
1053                 'limit', limit,
1054                 iff=limit is not None))
1055             self.assertEqual(SP.mock_calls[-1], call(
1056                 'marker', marker,
1057                 iff=marker is not None))
1058             self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1059             for i in range(len(r)):
1060                 self.assert_dicts_are_equal(r[i], sharers[i])
1061
1062     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1063     def test_get_object_versionlist(self, get):
1064         info = dict(object_info)
1065         info['versions'] = ['v1', 'v2']
1066         FR.json = info
1067         r = self.client.get_object_versionlist(obj)
1068         self.assertEqual(
1069             get.mock_calls[-1],
1070             call(obj, format='json', version='list'))
1071         self.assertEqual(r, info['versions'])
1072
1073 if __name__ == '__main__':
1074     from sys import argv
1075     from kamaki.clients.test import runTestCase
1076     runTestCase(Pithos, 'Pithos+ Client', argv[1:])