code.grnet.gr Git - kamaki/blob - kamaki/clients/pithos/test.py

   1 # Copyright 2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from unittest import TestCase
  35 from mock import patch, call
  36 from tempfile import NamedTemporaryFile
  37 from os import urandom
  38
  39 from kamaki.clients import ClientError
  40 from kamaki.clients.pithos import PithosClient as PC
  41
  42 pithos_pkg = 'kamaki.clients.pithos.PithosClient'
  43
  44 user_id = 'ac0un7-1d-5tr1ng'
  45 obj = 'obj3c7N4m3'
  46
  47 account_info = {
  48     'content-language': 'en-us',
  49     'content-type': 'text/html; charset=utf-8',
  50     'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
  51     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  52     'server': 'gunicorn/0.14.5',
  53     'vary': 'Accept-Language',
  54     'x-account-bytes-used': '751615526',
  55     'x-account-container-count': 7,
  56     'x-account-policy-quota': 53687091200,
  57     'x-account-policy-versioning': 'auto'}
  58 container_info = {
  59     'content-language': 'en-us',
  60     'content-type': 'text/html; charset=utf-8',
  61     'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
  62     'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
  63     'server': 'gunicorn/0.14.5',
  64     'vary': 'Accept-Language',
  65     'x-container-block-hash': 'sha256',
  66     'x-container-block-size': 4194304,
  67     'x-container-bytes-used': 309528938,
  68     'x-container-object-count': 14,
  69     'x-container-object-meta': '',
  70     'x-container-policy-quota': 53687091200,
  71     'x-container-policy-versioning': 'auto'}
  72 object_info = {
  73     'content-language': 'en-us',
  74     'content-length': 254965,
  75     'content-type': 'application/octet-stream',
  76     'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
  77     'etag': '',
  78     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  79     'server': 'gunicorn/0.14.5',
  80     'vary': 'Accept-Language',
  81     'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
  82     'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
  83     'x-object-version': '525996',
  84     'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
  85     'x-object-meta-k1': 'v1',
  86     'x-object-meta-k2': 'v2'}
  87 container_list = [
  88     dict(
  89         count=2,
  90         last_modified="2013-02-27T11:56:09.893033+00:00",
  91         bytes=677076979,
  92         name="pithos",
  93         x_container_policy=dict(quota="21474836480", versioning="auto")),
  94     dict(
  95         count=0,
  96         last_modified="2012-10-23T12:25:17.229187+00:00",
  97         bytes=0,
  98         name="trash",
  99         x_container_policy=dict(quota="21474836480", versioning="auto"))]
 100 object_list = [
 101     dict(hash="",
 102         name="The_Secret_Garden.zip",
 103         x_object_public="/public/wdp9p",
 104         bytes=203304947,
 105         x_object_version_timestamp="1360237915.7027509",
 106         x_object_uuid="s0m3uu1df0r0bj0n3",
 107         last_modified="2013-02-07T11:51:55.702751+00:00",
 108         content_type="application/octet-stream",
 109         x_object_hash="0afdf29f71cd53126225c3f54ca",
 110         x_object_version=17737,
 111         x_object_modified_by=user_id),
 112     dict(hash="",
 113         name="The_Revealed_Garden.zip",
 114         x_object_public="/public/wpd7p",
 115         bytes=20330947,
 116         x_object_version_timestamp="13602915.7027509",
 117         x_object_uuid="s0m3uu1df0r0bj70w",
 118         last_modified="2013-02-07T11:51:55.702751+00:00",
 119         content_type="application/octet-stream",
 120         x_object_hash="0afdf29f71cd53126225c3f54ca",
 121         x_object_version=17737,
 122         x_object_modified_by=user_id)]
 123 object_hashmap = dict(
 124     block_hash="sha256", block_size=4194304, bytes=33554432,
 125     hashes=[
 126         "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
 127         "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
 128         "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
 129         "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
 130         "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
 131         "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
 132         "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
 133         "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
 134 sharers = [
 135     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
 136     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
 137     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
 138
 139
 140 class FR(object):
 141     """FR stands for Fake Response"""
 142     json = dict()
 143     headers = dict()
 144     content = json
 145     status = None
 146     status_code = 200
 147
 148     def release(self):
 149         pass
 150
 151
 152 class Pithos(TestCase):
 153
 154     files = []
 155
 156     def _create_temp_file(self, num_of_blocks):
 157         self.files.append(NamedTemporaryFile())
 158         tmpFile = self.files[-1]
 159         file_size = num_of_blocks * 4 * 1024 * 1024
 160         print('\n\tCreate tmp file')
 161         tmpFile.write(urandom(file_size))
 162         tmpFile.flush()
 163         tmpFile.seek(0)
 164         print('\t\tDone')
 165         return tmpFile
 166
 167     def assert_dicts_are_equal(self, d1, d2):
 168         for k, v in d1.items():
 169             self.assertTrue(k in d2)
 170             if isinstance(v, dict):
 171                 self.assert_dicts_are_equal(v, d2[k])
 172             else:
 173                 self.assertEqual(unicode(v), unicode(d2[k]))
 174
 175     def setUp(self):
 176         self.url = 'https://www.example.com/pithos'
 177         self.token = 'p17h0570k3n'
 178         self.client = PC(self.url, self.token)
 179         self.client.account = user_id
 180         self.client.container = 'c0nt@1n3r_i'
 181
 182     def tearDown(self):
 183         FR.headers = dict()
 184         FR.status_code = 200
 185         FR.json = dict()
 186         FR.content = FR.json
 187         for f in self.files:
 188             f.close()
 189
 190     #  Pithos+ methods that extend storage API
 191
 192     @patch('%s.account_head' % pithos_pkg, return_value=FR())
 193     def test_get_account_info(self, AH):
 194         FR.headers = account_info
 195         r = self.client.get_account_info()
 196         self.assert_dicts_are_equal(r, account_info)
 197         self.assertEqual(AH.mock_calls[-1], call(until=None))
 198         unt = 'un71L-d473'
 199         r = self.client.get_account_info(until=unt)
 200         self.assert_dicts_are_equal(r, account_info)
 201         self.assertEqual(AH.mock_calls[-1], call(until=unt))
 202         FR.status_code = 401
 203         self.assertRaises(ClientError, self.client.get_account_info)
 204
 205     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 206     def test_del_account_meta(self, ap):
 207         keys = ['k1', 'k2', 'k3']
 208         expected = []
 209         for key in keys:
 210             self.client.del_account_meta(key)
 211             expected.append(call(update=True, metadata={key: ''}))
 212         self.assertEqual(ap.mock_calls, expected)
 213
 214     @patch('%s.container_head' % pithos_pkg, return_value=FR())
 215     def test_get_container_info(self, ch):
 216         FR.headers = container_info
 217         r = self.client.get_container_info()
 218         self.assert_dicts_are_equal(r, container_info)
 219         u = 'some date'
 220         r = self.client.get_container_info(until=u)
 221         self.assertEqual(ch.mock_calls, [call(until=None), call(until=u)])
 222
 223     @patch('%s.account_get' % pithos_pkg, return_value=FR())
 224     def test_list_containers(self, get):
 225         FR.json = container_list
 226         r = self.client.list_containers()
 227         for i in range(len(r)):
 228             self.assert_dicts_are_equal(r[i], container_list[i])
 229
 230     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 231     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 232     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 233     def test_upload_object(self, CI, CP, OP):
 234         num_of_blocks = 8
 235         tmpFile = self._create_temp_file(num_of_blocks)
 236
 237         # Without kwargs
 238         self.client.upload_object(obj, tmpFile)
 239         self.assertEqual(PC.get_container_info.mock_calls, [call()])
 240         [call1, call2] = PC.object_put.mock_calls
 241
 242         (args1, kwargs1) = call1[1:3]
 243         (args2, kwargs2) = call2[1:3]
 244         self.assertEqual(args1, (obj,))
 245         expected1 = dict(
 246             hashmap=True,
 247             success=(201, 409),
 248             format='json',
 249             json=dict(
 250                 hashes=['s0m3h@5h'] * num_of_blocks,
 251                 bytes=num_of_blocks * 4 * 1024 * 1024),
 252             etag=None,
 253             content_encoding=None,
 254             content_type='application/octet-stream',
 255             content_disposition=None,
 256             public=None,
 257             permissions=None)
 258         for k, v in expected1.items():
 259             if k == 'json':
 260                 self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
 261                 self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
 262             else:
 263                 self.assertEqual(v, kwargs1[k])
 264
 265         (args2, kwargs2) = call2[1:3]
 266         self.assertEqual(args2, (obj,))
 267         expected2 = dict(
 268             json=dict(
 269                 hashes=['s0m3h@5h'] * num_of_blocks,
 270                 bytes=num_of_blocks * 4 * 1024 * 1024),
 271             content_type='application/octet-stream',
 272             hashmap=True,
 273             success=201,
 274             format='json')
 275         for k, v in expected2.items():
 276             if k == 'json':
 277                 self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
 278                 self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
 279             else:
 280                 self.assertEqual(v, kwargs2[k])
 281
 282         OP = PC.object_put
 283         mock_offset = 2
 284
 285         #  With progress bars
 286         try:
 287             from progress.bar import ShadyBar
 288             blck_bar = ShadyBar('Mock blck calc.')
 289             upld_bar = ShadyBar('Mock uplds')
 290         except ImportError:
 291             blck_bar = None
 292             upld_bar = None
 293
 294         if blck_bar and upld_bar:
 295
 296             def blck_gen(n):
 297                 for i in blck_bar.iter(range(n)):
 298                     yield
 299                 yield
 300
 301             def upld_gen(n):
 302                 for i in upld_bar.iter(range(n)):
 303                     yield
 304                 yield
 305
 306             tmpFile.seek(0)
 307             self.client.upload_object(
 308                 obj, tmpFile,
 309                 hash_cb=blck_gen, upload_cb=upld_gen)
 310
 311             for i, c in enumerate(OP.mock_calls[-mock_offset:]):
 312                 self.assertEqual(OP.mock_calls[i], c)
 313
 314         #  With content-type
 315         tmpFile.seek(0)
 316         ctype = 'video/mpeg'
 317         sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
 318         self.client.upload_object(obj, tmpFile,
 319             content_type=ctype, sharing=sharing)
 320         self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
 321         self.assert_dicts_are_equal(
 322             OP.mock_calls[-2][2]['permissions'],
 323             sharing)
 324
 325         # With other args
 326         tmpFile.seek(0)
 327         kwargs = dict(
 328             etag='s0m3E74g',
 329             content_type=ctype,
 330             content_disposition=ctype + 'd15p051710n',
 331             public=True,
 332             content_encoding='802.11')
 333         self.client.upload_object(obj, tmpFile, **kwargs)
 334         for arg, val in kwargs.items():
 335             self.assertEqual(OP.mock_calls[-2][2][arg], val)
 336
 337     def test_get_object_info(self):
 338         FR.headers = object_info
 339         version = 'v3r510n'
 340         with patch.object(PC, 'object_head', return_value=FR()) as head:
 341             r = self.client.get_object_info(obj)
 342             self.assertEqual(r, object_info)
 343             r = self.client.get_object_info(obj, version=version)
 344             self.assertEqual(head.mock_calls, [
 345                 call(obj, version=None),
 346                 call(obj, version=version)])
 347         with patch.object(
 348                 PC, 'object_head',
 349                 side_effect=ClientError('Obj not found', 404)):
 350             self.assertRaises(
 351                 ClientError,
 352                 self.client.get_object_info,
 353                 obj, version=version)
 354
 355     @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
 356     def test_get_object_meta(self, GOI):
 357         for version in (None, 'v3r510n'):
 358             r = self.client.get_object_meta(obj, version)
 359             for k in [k for k in object_info if k.startswith('x-object-meta')]:
 360                 self.assertEqual(r.pop(k), object_info[k])
 361             self.assertFalse(len(r))
 362             self.assertEqual(GOI.mock_calls[-1], call(obj, version=version))
 363
 364     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 365     def test_del_object_meta(self, post):
 366         metakey = '50m3m3t4k3y'
 367         self.client.del_object_meta(obj, metakey)
 368         expected = call(obj, update=True, metadata={metakey: ''})
 369         self.assertEqual(post.mock_calls[-1], expected)
 370
 371     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 372     def test_copy_object(self, put):
 373         src_cont = 'src-c0nt41n3r'
 374         src_obj = 'src-0bj'
 375         dst_cont = 'dst-c0nt41n3r'
 376         dst_obj = 'dst-0bj'
 377         expected = call(
 378             src_obj,
 379             content_length=0,
 380             source_account=None,
 381             success=201,
 382             copy_from='/%s/%s' % (src_cont, src_obj),
 383             delimiter=None,
 384             content_type=None,
 385             source_version=None,
 386             public=False)
 387         self.client.copy_object(src_cont, src_obj, dst_cont)
 388         self.assertEqual(put.mock_calls[-1], expected)
 389         self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
 390         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 391         kwargs = dict(
 392             source_version='src-v3r510n',
 393             source_account='src-4cc0un7',
 394             public=True,
 395             content_type='c0n73n7Typ3',
 396             delimiter='5')
 397         self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
 398         for k, v in kwargs.items():
 399             self.assertEqual(v, put.mock_calls[-1][2][k])
 400
 401     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 402     def test_move_object(self, put):
 403         src_cont = 'src-c0nt41n3r'
 404         src_obj = 'src-0bj'
 405         dst_cont = 'dst-c0nt41n3r'
 406         dst_obj = 'dst-0bj'
 407         expected = call(
 408             src_obj,
 409             content_length=0,
 410             source_account=None,
 411             success=201,
 412             move_from='/%s/%s' % (src_cont, src_obj),
 413             delimiter=None,
 414             content_type=None,
 415             source_version=None,
 416             public=False)
 417         self.client.move_object(src_cont, src_obj, dst_cont)
 418         self.assertEqual(put.mock_calls[-1], expected)
 419         self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
 420         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 421         kwargs = dict(
 422             source_version='src-v3r510n',
 423             source_account='src-4cc0un7',
 424             public=True,
 425             content_type='c0n73n7Typ3',
 426             delimiter='5')
 427         self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
 428         for k, v in kwargs.items():
 429             self.assertEqual(v, put.mock_calls[-1][2][k])
 430
 431     #  Pithos+ only methods
 432
 433     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 434     def test_purge_container(self, cd):
 435         self.client.purge_container()
 436         self.assertTrue('until' in cd.mock_calls[-1][2])
 437         cont = self.client.container
 438         self.client.purge_container('another-container')
 439         self.assertEqual(self.client.container, cont)
 440
 441     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 442     def test_upload_object_unchunked(self, put):
 443         num_of_blocks = 8
 444         tmpFile = self._create_temp_file(num_of_blocks)
 445         expected = dict(
 446                 success=201,
 447                 data=num_of_blocks * 4 * 1024 * 1024,
 448                 etag='some-etag',
 449                 content_encoding='some content_encoding',
 450                 content_type='some content-type',
 451                 content_disposition='some content_disposition',
 452                 public=True,
 453                 permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 454         self.client.upload_object_unchunked(obj, tmpFile)
 455         self.assertEqual(put.mock_calls[-1][1], (obj,))
 456         self.assertEqual(
 457             sorted(put.mock_calls[-1][2].keys()),
 458             sorted(expected.keys()))
 459         kwargs = dict(expected)
 460         kwargs.pop('success')
 461         kwargs['size'] = kwargs.pop('data')
 462         kwargs['sharing'] = kwargs.pop('permissions')
 463         tmpFile.seek(0)
 464         self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
 465         pmc = put.mock_calls[-1][2]
 466         for k, v in expected.items():
 467             if k == 'data':
 468                 self.assertEqual(len(pmc[k]), v)
 469             else:
 470                 self.assertEqual(pmc[k], v)
 471         self.assertRaises(
 472             ClientError,
 473             self.client.upload_object_unchunked,
 474             obj, tmpFile, withHashFile=True)
 475
 476     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 477     def test_create_object_by_manifestation(self, put):
 478         manifest = '%s/%s' % (self.client.container, obj)
 479         kwargs = dict(
 480                 etag='some-etag',
 481                 content_encoding='some content_encoding',
 482                 content_type='some content-type',
 483                 content_disposition='some content_disposition',
 484                 public=True,
 485                 sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 486         self.client.create_object_by_manifestation(obj)
 487         expected = dict(content_length=0, manifest=manifest)
 488         for k in kwargs:
 489             expected['permissions' if k == 'sharing' else k] = None
 490         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 491         self.client.create_object_by_manifestation(obj, **kwargs)
 492         expected.update(kwargs)
 493         expected['permissions'] = expected.pop('sharing')
 494         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 495
 496     @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
 497     @patch('%s.object_get' % pithos_pkg, return_value=FR())
 498     def test_download_object(self, GOH, GET):
 499         num_of_blocks = 8
 500         tmpFile = self._create_temp_file(num_of_blocks)
 501         FR.content = tmpFile.read(4 * 1024 * 1024)
 502         tmpFile = self._create_temp_file(num_of_blocks)
 503         GET = PC.object_get
 504         num_of_blocks = len(object_hashmap['hashes'])
 505         kwargs = dict(
 506             resume=True,
 507             version='version',
 508             range_str='10-20',
 509             if_match='if and only if',
 510             if_none_match='if and only not',
 511             if_modified_since='what if not?',
 512             if_unmodified_since='this happens if not!',
 513             async_headers=dict(Range='bytes=0-88888888'))
 514
 515         self.client.download_object(obj, tmpFile)
 516         self.assertEqual(len(GET.mock_calls), num_of_blocks)
 517         self.assertEqual(GET.mock_calls[-1][1], (obj,))
 518         for k, v in kwargs.items():
 519             if k == 'async_headers':
 520                 self.assertTrue('Range' in GET.mock_calls[-1][2][k])
 521             elif k in ('resume', 'range_str'):
 522                 continue
 523             else:
 524                 self.assertEqual(GET.mock_calls[-1][2][k], None)
 525
 526         #  Check ranges are consecutive
 527         starts = []
 528         ends = []
 529         for c in GET.mock_calls:
 530             rng_str = c[2]['async_headers']['Range']
 531             (start, rng_str) = rng_str.split('=')
 532             (start, end) = rng_str.split('-')
 533             starts.append(start)
 534             ends.append(end)
 535         ends = sorted(ends)
 536         for i, start in enumerate(sorted(starts)):
 537             if i:
 538                 int(ends[i - 1]) == int(start) - 1
 539
 540         #  With progress bars
 541         try:
 542             from progress.bar import ShadyBar
 543             dl_bar = ShadyBar('Mock dl')
 544         except ImportError:
 545             dl_bar = None
 546
 547         if dl_bar:
 548
 549             def blck_gen(n):
 550                 for i in dl_bar.iter(range(n)):
 551                     yield
 552                 yield
 553
 554             tmpFile.seek(0)
 555             self.client.download_object(obj, tmpFile, download_cb=blck_gen)
 556             self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
 557
 558         tmpFile.seek(0)
 559         kwargs.pop('async_headers')
 560         kwargs.pop('resume')
 561         self.client.download_object(obj, tmpFile, **kwargs)
 562         for k, v in kwargs.items():
 563             if k == 'range_str':
 564                 self.assertEqual(
 565                     GET.mock_calls[-1][2]['data_range'],
 566                     'bytes=%s' % v)
 567             else:
 568                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 569
 570         #  ALl options on no tty
 571
 572         def foo():
 573             return True
 574
 575         tmpFile.seek(0)
 576         tmpFile.isatty = foo
 577         self.client.download_object(obj, tmpFile, **kwargs)
 578         for k, v in kwargs.items():
 579             if k == 'range_str':
 580                 self.assertTrue('data_range' in GET.mock_calls[-1][2])
 581             else:
 582                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 583
 584     def test_get_object_hashmap(self):
 585         FR.json = object_hashmap
 586         for empty in (304, 412):
 587             with patch.object(
 588                     PC, 'object_get',
 589                     side_effect=ClientError('Empty', status=empty)):
 590                 r = self.client.get_object_hashmap(obj)
 591                 self.assertEqual(r, {})
 592         exp_args = dict(
 593             hashmap=True,
 594             data_range=None,
 595             version=None,
 596             if_etag_match=None,
 597             if_etag_not_match=None,
 598             if_modified_since=None,
 599             if_unmodified_since=None)
 600         kwargs = dict(
 601             version='s0m3v3r51on',
 602             if_match='if match',
 603             if_none_match='if non match',
 604             if_modified_since='some date here',
 605             if_unmodified_since='some date here',
 606             data_range='10-20')
 607         with patch.object(PC, 'object_get', return_value=FR()) as get:
 608             r = self.client.get_object_hashmap(obj)
 609             self.assertEqual(r, object_hashmap)
 610             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 611             r = self.client.get_object_hashmap(obj, **kwargs)
 612             exp_args['if_etag_match'] = kwargs.pop('if_match')
 613             exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
 614             exp_args.update(kwargs)
 615             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 616
 617     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 618     def test_set_account_group(self, post):
 619         (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
 620         self.client.set_account_group(group, usernames)
 621         self.assertEqual(
 622             post.mock_calls[-1],
 623             call(update=True, groups={group: usernames}))
 624
 625     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 626     def test_del_account_group(self, post):
 627         group = 'aU53rGr0up'
 628         self.client.del_account_group(group)
 629         self.assertEqual(
 630             post.mock_calls[-1],
 631             call(update=True, groups={group: []}))
 632
 633     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 634     def test_get_account_quota(self, GAI):
 635         key = 'x-account-policy-quota'
 636         r = self.client.get_account_quota()
 637         self.assertEqual(r[key], account_info[key])
 638
 639     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 640     def test_get_account_versioning(self, GAI):
 641         key = 'x-account-policy-versioning'
 642         r = self.client.get_account_versioning()
 643         self.assertEqual(r[key], account_info[key])
 644
 645     def test_get_account_meta(self):
 646         key = 'x-account-meta-'
 647         with patch.object(PC, 'get_account_info', return_value=account_info):
 648             r = self.client.get_account_meta()
 649             keys = [k for k in r if k.startswith(key)]
 650             self.assertFalse(keys)
 651         acc_info = dict(account_info)
 652         acc_info['%sk1' % key] = 'v1'
 653         acc_info['%sk2' % key] = 'v2'
 654         acc_info['%sk3' % key] = 'v3'
 655         with patch.object(PC, 'get_account_info', return_value=acc_info):
 656             r = self.client.get_account_meta()
 657             for k in [k for k in acc_info if k.startswith(key)]:
 658                 self.assertEqual(r[k], acc_info[k])
 659
 660     def test_get_account_group(self):
 661         key = 'x-account-group-'
 662         with patch.object(PC, 'get_account_info', return_value=account_info):
 663             r = self.client.get_account_group()
 664             keys = [k for k in r if k.startswith(key)]
 665             self.assertFalse(keys)
 666         acc_info = dict(account_info)
 667         acc_info['%sk1' % key] = 'g1'
 668         acc_info['%sk2' % key] = 'g2'
 669         acc_info['%sk3' % key] = 'g3'
 670         with patch.object(PC, 'get_account_info', return_value=acc_info):
 671             r = self.client.get_account_group()
 672             for k in [k for k in acc_info if k.startswith(key)]:
 673                 self.assertEqual(r[k], acc_info[k])
 674
 675     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 676     def test_set_account_meta(self, post):
 677         metas = dict(k1='v1', k2='v2', k3='v3')
 678         self.client.set_account_meta(metas)
 679         self.assertEqual(
 680             post.mock_calls[-1],
 681             call(update=True, metadata=metas))
 682
 683     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 684     def test_set_account_quota(self, post):
 685         qu = 1024
 686         self.client.set_account_quota(qu)
 687         self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
 688
 689     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 690     def test_set_account_versioning(self, post):
 691         vrs = 'n3wV3r51on1ngTyp3'
 692         self.client.set_account_versioning(vrs)
 693         self.assertEqual(
 694             post.mock_calls[-1],
 695             call(update=True, versioning=vrs))
 696
 697     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 698     def test_del_container(self, delete):
 699         for kwarg in (
 700                 dict(delimiter=None, until=None),
 701                 dict(delimiter='X', until='50m3d473')):
 702             self.client.del_container(**kwarg)
 703             expected = dict(kwarg)
 704             expected['success'] = (204, 404, 409)
 705             self.assertEqual(delete.mock_calls[-1], call(**expected))
 706         for status_code in (404, 409):
 707             FR.status_code = status_code
 708             self.assertRaises(ClientError, self.client.del_container)
 709
 710     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 711     def test_get_container_versioning(self, GCI):
 712         key = 'x-container-policy-versioning'
 713         cont = 'c0n7-417'
 714         bu_cnt = self.client.container
 715         for container in (None, cont):
 716             r = self.client.get_container_versioning(container=container)
 717             self.assertEqual(r[key], container_info[key])
 718             self.assertEqual(GCI.mock_calls[-1], call())
 719             self.assertEqual(bu_cnt, self.client.container)
 720
 721     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 722     def test_get_container_quota(self, GCI):
 723         key = 'x-container-policy-quota'
 724         cont = 'c0n7-417'
 725         bu_cnt = self.client.container
 726         for container in (None, cont):
 727             r = self.client.get_container_quota(container=container)
 728             self.assertEqual(r[key], container_info[key])
 729             self.assertEqual(GCI.mock_calls[-1], call())
 730             self.assertEqual(bu_cnt, self.client.container)
 731
 732     def test_get_container_meta(self):
 733         somedate = '50m3d473'
 734         key = 'x-container-meta'
 735         metaval = '50m3m374v41'
 736         container_plus = dict(container_info)
 737         container_plus[key] = metaval
 738         for ret in ((container_info, {}), (container_plus, {key: metaval})):
 739             with patch.object(
 740                     PC,
 741                     'get_container_info',
 742                     return_value=ret[0]) as gci:
 743                 for until in (None, somedate):
 744                     r = self.client.get_container_meta(until=until)
 745                     self.assertEqual(r, ret[1])
 746                     self.assertEqual(gci.mock_calls[-1], call(until=until))
 747
 748     def test_get_container_object_meta(self):
 749         somedate = '50m3d473'
 750         key = 'x-container-object-meta'
 751         metaval = '50m3m374v41'
 752         container_plus = dict(container_info)
 753         container_plus[key] = metaval
 754         for ret in (
 755                 (container_info, {key: ''}),
 756                 (container_plus, {key: metaval})):
 757             with patch.object(
 758                     PC,
 759                     'get_container_info',
 760                     return_value=ret[0]) as gci:
 761                 for until in (None, somedate):
 762                     r = self.client.get_container_object_meta(until=until)
 763                     self.assertEqual(r, ret[1])
 764                     self.assertEqual(gci.mock_calls[-1], call(until=until))
 765
 766     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 767     def test_set_container_meta(self, post):
 768         metas = dict(k1='v1', k2='v2', k3='v3')
 769         self.client.set_container_meta(metas)
 770         self.assertEqual(
 771             post.mock_calls[-1],
 772             call(update=True, metadata=metas))
 773
 774     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 775     def test_del_container_meta(self, ap):
 776         self.client.del_container_meta('somekey')
 777         expected = [call(update=True, metadata={'somekey': ''})]
 778         self.assertEqual(ap.mock_calls, expected)
 779
 780     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 781     def test_set_container_quota(self, post):
 782         qu = 1024
 783         self.client.set_container_quota(qu)
 784         self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
 785
 786     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 787     def test_set_container_versioning(self, post):
 788         vrs = 'n3wV3r51on1ngTyp3'
 789         self.client.set_container_versioning(vrs)
 790         self.assertEqual(
 791             post.mock_calls[-1],
 792             call(update=True, versioning=vrs))
 793
 794     @patch('%s.object_delete' % pithos_pkg, return_value=FR())
 795     def test_del_object(self, delete):
 796         for kwarg in (
 797                 dict(delimiter=None, until=None),
 798                 dict(delimiter='X', until='50m3d473')):
 799             self.client.del_object(obj, **kwarg)
 800             self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
 801
 802     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 803     def test_set_object_meta(self, post):
 804         metas = dict(k1='v1', k2='v2', k3='v3')
 805         self.assertRaises(
 806             AssertionError,
 807             self.client.set_object_meta,
 808             obj, 'Non dict arg')
 809         self.client.set_object_meta(obj, metas)
 810         self.assertEqual(
 811             post.mock_calls[-1],
 812             call(obj, update=True, metadata=metas))
 813
 814     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 815     def test_publish_object(self, post):
 816         oinfo = dict(object_info)
 817         val = 'pubL1c'
 818         oinfo['x-object-public'] = val
 819         with patch.object(PC, 'get_object_info', return_value=oinfo) as gof:
 820             r = self.client.publish_object(obj)
 821             self.assertEqual(
 822                 post.mock_calls[-1],
 823                 call(obj, public=True, update=True))
 824             self.assertEqual(gof.mock_calls[-1], call(obj))
 825             self.assertEqual(r, '%s%s' % (self.url[:-6], val))
 826
 827     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 828     def test_unpublish_object(self, post):
 829         self.client.unpublish_object(obj)
 830         self.assertEqual(
 831             post.mock_calls[-1],
 832             call(obj, public=False, update=True))
 833
 834     def test_get_object_sharing(self):
 835         info = dict(object_info)
 836         expected = dict(read='u1,g1,u2', write='u1')
 837         info['x-object-sharing'] = '; '.join(
 838             ['%s=%s' % (k, v) for k, v in expected.items()])
 839         with patch.object(PC, 'get_object_info', return_value=info) as GOF:
 840             r = self.client.get_object_sharing(obj)
 841             self.assertEqual(GOF.mock_calls[-1], call(obj))
 842             self.assert_dicts_are_equal(r, expected)
 843             info['x-object-sharing'] = '//'.join(
 844                 ['%s=%s' % (k, v) for k, v in expected.items()])
 845             self.assertRaises(
 846                 ValueError,
 847                 self.client.get_object_sharing,
 848                 obj)
 849             info['x-object-sharing'] = '; '.join(
 850                 ['%s:%s' % (k, v) for k, v in expected.items()])
 851             self.assertRaises(
 852                 ClientError,
 853                 self.client.get_object_sharing,
 854                 obj)
 855             info['x-object-sharing'] = 'read=%s' % expected['read']
 856             r = self.client.get_object_sharing(obj)
 857             expected.pop('write')
 858             self.assert_dicts_are_equal(r, expected)
 859
 860     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 861     def test_set_object_sharing(self, POST):
 862         read_perms = ['u1', 'g1', 'u2', 'g2']
 863         write_perms = ['u1', 'g1']
 864         for kwargs in (
 865                 dict(read_permition=read_perms, write_permition=write_perms),
 866                 dict(read_permition=read_perms),
 867                 dict(write_permition=write_perms),
 868                 dict()):
 869             self.client.set_object_sharing(obj, **kwargs)
 870             kwargs['read'] = kwargs.pop('read_permition', '')
 871             kwargs['write'] = kwargs.pop('write_permition', '')
 872             self.assertEqual(
 873                 POST.mock_calls[-1],
 874                 call(obj, update=True, permissions=kwargs))
 875
 876     @patch('%s.set_object_sharing' % pithos_pkg)
 877     def test_del_object_sharing(self, SOS):
 878         self.client.del_object_sharing(obj)
 879         self.assertEqual(SOS.mock_calls[-1], call(obj))
 880
 881     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 882     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 883     def test_append_object(self, post, GCI):
 884         num_of_blocks = 4
 885         tmpFile = self._create_temp_file(num_of_blocks)
 886         tmpFile.seek(0, 2)
 887         file_size = tmpFile.tell()
 888         for turn in range(2):
 889             tmpFile.seek(0, 0)
 890
 891             try:
 892                 from progress.bar import ShadyBar
 893                 apn_bar = ShadyBar('Mock append')
 894             except ImportError:
 895                 apn_bar = None
 896
 897             if apn_bar:
 898
 899                 def append_gen(n):
 900                     for i in apn_bar.iter(range(n)):
 901                         yield
 902                     yield
 903
 904             else:
 905                 append_gen = None
 906
 907             self.client.append_object(
 908                 obj, tmpFile,
 909                 upload_cb=append_gen if turn else None)
 910             self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
 911             (args, kwargs) = post.mock_calls[-1][1:3]
 912             self.assertEqual(args, (obj,))
 913             self.assertEqual(kwargs['content_length'], len(kwargs['data']))
 914             fsize = num_of_blocks * int(kwargs['content_length'])
 915             self.assertEqual(fsize, file_size)
 916             self.assertEqual(kwargs['content_range'], 'bytes */*')
 917             exp = 'application/octet-stream'
 918             self.assertEqual(kwargs['content_type'], exp)
 919             self.assertEqual(kwargs['update'], True)
 920
 921     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 922     def test_truncate_object(self, post):
 923         upto_bytes = 377
 924         self.client.truncate_object(obj, upto_bytes)
 925         self.assertEqual(post.mock_calls[-1], call(
 926             obj,
 927             update=True,
 928             object_bytes=upto_bytes,
 929             content_range='bytes 0-%s/*' % upto_bytes,
 930             content_type='application/octet-stream',
 931             source_object='/%s/%s' % (self.client.container, obj)))
 932
 933     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 934     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 935     def test_overwrite_object(self, post, GCI):
 936         num_of_blocks = 4
 937         tmpFile = self._create_temp_file(num_of_blocks)
 938         tmpFile.seek(0, 2)
 939         file_size = tmpFile.tell()
 940         info = dict(object_info)
 941         info['content-length'] = file_size
 942         block_size = container_info['x-container-block-size']
 943         with patch.object(PC, 'get_object_info', return_value=info) as GOI:
 944             for start, end in (
 945                     (0, file_size + 1),
 946                     (file_size + 1, file_size + 2)):
 947                 tmpFile.seek(0, 0)
 948                 self.assertRaises(
 949                     ClientError,
 950                     self.client.overwrite_object,
 951                     obj, start, end, tmpFile)
 952             for start, end in ((0, 144), (144, 233), (233, file_size)):
 953                 tmpFile.seek(0, 0)
 954                 owr_gen = None
 955                 exp_size = end - start + 1
 956                 if not start or exp_size > block_size:
 957                     try:
 958                         from progress.bar import ShadyBar
 959                         owr_bar = ShadyBar('Mock append')
 960                     except ImportError:
 961                         owr_bar = None
 962
 963                     if owr_bar:
 964
 965                         def owr_gen(n):
 966                             for i in owr_bar.iter(range(n)):
 967                                 yield
 968                             yield
 969
 970                     if exp_size > block_size:
 971                         exp_size = exp_size % block_size or block_size
 972
 973                 self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
 974                 self.assertEqual(GOI.mock_calls[-1], call(obj))
 975                 self.assertEqual(GCI.mock_calls[-1], call())
 976                 (args, kwargs) = post.mock_calls[-1][1:3]
 977                 self.assertEqual(args, (obj,))
 978                 self.assertEqual(len(kwargs['data']), exp_size)
 979                 self.assertEqual(kwargs['content_length'], exp_size)
 980                 self.assertEqual(kwargs['update'], True)
 981                 exp = 'application/octet-stream'
 982                 self.assertEqual(kwargs['content_type'], exp)
 983
 984     @patch('%s.set_param' % pithos_pkg)
 985     @patch('%s.get' % pithos_pkg, return_value=FR())
 986     def test_get_sharing_accounts(self, get, SP):
 987         FR.json = sharers
 988         for kws in (
 989                 dict(),
 990                 dict(limit='50m3-11m17'),
 991                 dict(marker='X'),
 992                 dict(limit='50m3-11m17', marker='X')):
 993             r = self.client.get_sharing_accounts(**kws)
 994             self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
 995             limit, marker = kws.get('limit', None), kws.get('marker', None)
 996             self.assertEqual(SP.mock_calls[-2], call(
 997                 'limit', limit,
 998                 iff=limit is not None))
 999             self.assertEqual(SP.mock_calls[-1], call(
1000                 'marker', marker,
1001                 iff=marker is not None))
1002             self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1003             for i in range(len(r)):
1004                 self.assert_dicts_are_equal(r[i], sharers[i])
1005
1006     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1007     def test_get_object_versionlist(self, get):
1008         info = dict(object_info)
1009         info['versions'] = ['v1', 'v2']
1010         FR.json = info
1011         r = self.client.get_object_versionlist(obj)
1012         self.assertEqual(
1013             get.mock_calls[-1],
1014             call(obj, format='json', version='list'))
1015         self.assertEqual(r, info['versions'])
1016
1017 if __name__ == '__main__':
1018     from sys import argv
1019     from kamaki.clients.test import runTestCase
1020     runTestCase(Pithos, 'Pithos+ Client', argv[1:])