code.grnet.gr Git - kamaki/blob - kamaki/clients/pithos/test.py

   1 # Copyright 2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from unittest import TestCase
  35 from mock import patch, call
  36 from tempfile import NamedTemporaryFile
  37 from os import urandom
  38
  39 from kamaki.clients import ClientError
  40 from kamaki.clients.pithos import PithosClient, PithosRestClient
  41
  42 pithos_pkg = 'kamaki.clients.pithos.PithosClient'
  43
  44 user_id = 'ac0un7-1d-5tr1ng'
  45 obj = 'obj3c7N4m3'
  46
  47 account_info = {
  48     'content-language': 'en-us',
  49     'content-type': 'text/html; charset=utf-8',
  50     'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
  51     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  52     'server': 'gunicorn/0.14.5',
  53     'vary': 'Accept-Language',
  54     'x-account-bytes-used': '751615526',
  55     'x-account-container-count': 7,
  56     'x-account-policy-quota': 53687091200,
  57     'x-account-policy-versioning': 'auto'}
  58 container_info = {
  59     'content-language': 'en-us',
  60     'content-type': 'text/html; charset=utf-8',
  61     'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
  62     'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
  63     'server': 'gunicorn/0.14.5',
  64     'vary': 'Accept-Language',
  65     'x-container-block-hash': 'sha256',
  66     'x-container-block-size': 4194304,
  67     'x-container-bytes-used': 309528938,
  68     'x-container-object-count': 14,
  69     'x-container-object-meta': '',
  70     'x-container-policy-quota': 53687091200,
  71     'x-container-policy-versioning': 'auto'}
  72 object_info = {
  73     'content-language': 'en-us',
  74     'content-length': 254965,
  75     'content-type': 'application/octet-stream',
  76     'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
  77     'etag': '',
  78     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  79     'server': 'gunicorn/0.14.5',
  80     'vary': 'Accept-Language',
  81     'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
  82     'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
  83     'x-object-version': '525996',
  84     'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
  85     'x-object-meta-k1': 'v1',
  86     'x-object-meta-k2': 'v2'}
  87 container_list = [
  88     dict(
  89         count=2,
  90         last_modified="2013-02-27T11:56:09.893033+00:00",
  91         bytes=677076979,
  92         name="pithos",
  93         x_container_policy=dict(quota="21474836480", versioning="auto")),
  94     dict(
  95         count=0,
  96         last_modified="2012-10-23T12:25:17.229187+00:00",
  97         bytes=0,
  98         name="trash",
  99         x_container_policy=dict(quota="21474836480", versioning="auto"))]
 100 object_list = [
 101     dict(hash="",
 102         name="The_Secret_Garden.zip",
 103         x_object_public="/public/wdp9p",
 104         bytes=203304947,
 105         x_object_version_timestamp="1360237915.7027509",
 106         x_object_uuid="s0m3uu1df0r0bj0n3",
 107         last_modified="2013-02-07T11:51:55.702751+00:00",
 108         content_type="application/octet-stream",
 109         x_object_hash="0afdf29f71cd53126225c3f54ca",
 110         x_object_version=17737,
 111         x_object_modified_by=user_id),
 112     dict(hash="",
 113         name="The_Revealed_Garden.zip",
 114         x_object_public="/public/wpd7p",
 115         bytes=20330947,
 116         x_object_version_timestamp="13602915.7027509",
 117         x_object_uuid="s0m3uu1df0r0bj70w",
 118         last_modified="2013-02-07T11:51:55.702751+00:00",
 119         content_type="application/octet-stream",
 120         x_object_hash="0afdf29f71cd53126225c3f54ca",
 121         x_object_version=17737,
 122         x_object_modified_by=user_id)]
 123 object_hashmap = dict(
 124     block_hash="sha256", block_size=4194304, bytes=33554432,
 125     hashes=[
 126         "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
 127         "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
 128         "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
 129         "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
 130         "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
 131         "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
 132         "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
 133         "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
 134 sharers = [
 135     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
 136     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
 137     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
 138
 139
 140 class FR(object):
 141     """FR stands for Fake Response"""
 142     json = dict()
 143     headers = dict()
 144     content = json
 145     status = None
 146     status_code = 200
 147
 148     def release(self):
 149         pass
 150
 151
 152 class PithosRest(TestCase):
 153
 154     def setUp(self):
 155         self.url = 'https://www.example.com/pithos'
 156         self.token = 'p17h0570k3n'
 157         self.client = PithosClient(self.url, self.token)
 158         self.client.account = user_id
 159         self.client.container = 'c0nt@1n3r_i'
 160
 161     def tearDown(self):
 162         FR.headers = dict()
 163         FR.status_code = 200
 164         FR.json = dict()
 165         FR.content = FR.json
 166         for f in self.files:
 167             f.close()
 168
 169
 170 class Pithos(TestCase):
 171
 172     files = []
 173
 174     def _create_temp_file(self, num_of_blocks):
 175         self.files.append(NamedTemporaryFile())
 176         tmpFile = self.files[-1]
 177         file_size = num_of_blocks * 4 * 1024 * 1024
 178         print('\n\tCreate tmp file')
 179         tmpFile.write(urandom(file_size))
 180         tmpFile.flush()
 181         tmpFile.seek(0)
 182         print('\t\tDone')
 183         return tmpFile
 184
 185     def assert_dicts_are_equal(self, d1, d2):
 186         for k, v in d1.items():
 187             self.assertTrue(k in d2)
 188             if isinstance(v, dict):
 189                 self.assert_dicts_are_equal(v, d2[k])
 190             else:
 191                 self.assertEqual(unicode(v), unicode(d2[k]))
 192
 193     def setUp(self):
 194         self.url = 'https://www.example.com/pithos'
 195         self.token = 'p17h0570k3n'
 196         self.client = PithosClient(self.url, self.token)
 197         self.client.account = user_id
 198         self.client.container = 'c0nt@1n3r_i'
 199
 200     def tearDown(self):
 201         FR.headers = dict()
 202         FR.status_code = 200
 203         FR.json = dict()
 204         FR.content = FR.json
 205         for f in self.files:
 206             f.close()
 207
 208     #  Pithos+ methods that extend storage API
 209
 210     @patch('%s.account_head' % pithos_pkg, return_value=FR())
 211     def test_get_account_info(self, AH):
 212         FR.headers = account_info
 213         for until in (None, 'un71L-d473'):
 214             r = self.client.get_account_info(until=until)
 215             self.assert_dicts_are_equal(r, account_info)
 216             self.assertEqual(AH.mock_calls[-1], call(until=until))
 217         FR.status_code = 401
 218         self.assertRaises(ClientError, self.client.get_account_info)
 219
 220     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 221     def test_del_account_meta(self, AP):
 222         keys = ['k1', 'k2', 'k3']
 223         for key in keys:
 224             self.client.del_account_meta(key)
 225             self.assertEqual(
 226                 AP.mock_calls[-1],
 227                 call(update=True, metadata={key: ''}))
 228
 229     @patch('%s.container_head' % pithos_pkg, return_value=FR())
 230     def test_get_container_info(self, CH):
 231         FR.headers = container_info
 232         r = self.client.get_container_info()
 233         self.assert_dicts_are_equal(r, container_info)
 234         u = 'some date'
 235         r = self.client.get_container_info(until=u)
 236         self.assertEqual(CH.mock_calls, [call(until=None), call(until=u)])
 237
 238     @patch('%s.account_get' % pithos_pkg, return_value=FR())
 239     def test_list_containers(self, get):
 240         FR.json = container_list
 241         r = self.client.list_containers()
 242         get.assert_called_once_with()
 243         for i in range(len(r)):
 244             self.assert_dicts_are_equal(r[i], container_list[i])
 245
 246     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 247     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 248     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 249     def test_upload_object(self, OP, CP, GCI):
 250         num_of_blocks = 8
 251         tmpFile = self._create_temp_file(num_of_blocks)
 252
 253         # Without kwargs
 254         self.client.upload_object(obj, tmpFile)
 255         self.assertEqual(GCI.mock_calls[-1], call())
 256         [call1, call2] = OP.mock_calls
 257
 258         (args1, kwargs1) = call1[1:3]
 259         (args2, kwargs2) = call2[1:3]
 260         self.assertEqual(args1, (obj,))
 261         expected1 = dict(
 262             hashmap=True,
 263             success=(201, 409),
 264             format='json',
 265             json=dict(
 266                 hashes=['s0m3h@5h'] * num_of_blocks,
 267                 bytes=num_of_blocks * 4 * 1024 * 1024),
 268             etag=None,
 269             content_encoding=None,
 270             content_type='application/octet-stream',
 271             content_disposition=None,
 272             public=None,
 273             permissions=None)
 274         for k, v in expected1.items():
 275             if k == 'json':
 276                 self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
 277                 self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
 278             else:
 279                 self.assertEqual(v, kwargs1[k])
 280
 281         (args2, kwargs2) = call2[1:3]
 282         self.assertEqual(args2, (obj,))
 283         expected2 = dict(
 284             json=dict(
 285                 hashes=['s0m3h@5h'] * num_of_blocks,
 286                 bytes=num_of_blocks * 4 * 1024 * 1024),
 287             content_type='application/octet-stream',
 288             hashmap=True,
 289             success=201,
 290             format='json')
 291         for k, v in expected2.items():
 292             if k == 'json':
 293                 self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
 294                 self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
 295             else:
 296                 self.assertEqual(v, kwargs2[k])
 297
 298         mock_offset = 2
 299
 300         #  With progress bars
 301         try:
 302             from progress.bar import ShadyBar
 303             blck_bar = ShadyBar('Mock blck calc.')
 304             upld_bar = ShadyBar('Mock uplds')
 305         except ImportError:
 306             blck_bar = None
 307             upld_bar = None
 308
 309         if blck_bar and upld_bar:
 310
 311             def blck_gen(n):
 312                 for i in blck_bar.iter(range(n)):
 313                     yield
 314                 yield
 315
 316             def upld_gen(n):
 317                 for i in upld_bar.iter(range(n)):
 318                     yield
 319                 yield
 320
 321             tmpFile.seek(0)
 322             self.client.upload_object(
 323                 obj, tmpFile,
 324                 hash_cb=blck_gen, upload_cb=upld_gen)
 325
 326             for i, c in enumerate(OP.mock_calls[-mock_offset:]):
 327                 self.assertEqual(OP.mock_calls[i], c)
 328
 329         #  With content-type
 330         tmpFile.seek(0)
 331         ctype = 'video/mpeg'
 332         sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
 333         self.client.upload_object(obj, tmpFile,
 334             content_type=ctype, sharing=sharing)
 335         self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
 336         self.assert_dicts_are_equal(
 337             OP.mock_calls[-2][2]['permissions'],
 338             sharing)
 339
 340         # With other args
 341         tmpFile.seek(0)
 342         kwargs = dict(
 343             etag='s0m3E74g',
 344             content_type=ctype,
 345             content_disposition=ctype + 'd15p051710n',
 346             public=True,
 347             content_encoding='802.11')
 348         self.client.upload_object(obj, tmpFile, **kwargs)
 349         for arg, val in kwargs.items():
 350             self.assertEqual(OP.mock_calls[-2][2][arg], val)
 351
 352     def test_get_object_info(self):
 353         FR.headers = object_info
 354         version = 'v3r510n'
 355         with patch.object(
 356                 PithosClient, 'object_head',
 357                 return_value=FR()) as head:
 358             r = self.client.get_object_info(obj)
 359             self.assertEqual(r, object_info)
 360             r = self.client.get_object_info(obj, version=version)
 361             self.assertEqual(head.mock_calls, [
 362                 call(obj, version=None),
 363                 call(obj, version=version)])
 364         with patch.object(
 365                 PithosClient, 'object_head',
 366                 side_effect=ClientError('Obj not found', 404)):
 367             self.assertRaises(
 368                 ClientError,
 369                 self.client.get_object_info,
 370                 obj, version=version)
 371
 372     @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
 373     def test_get_object_meta(self, GOI):
 374         for version in (None, 'v3r510n'):
 375             r = self.client.get_object_meta(obj, version)
 376             for k in [k for k in object_info if k.startswith('x-object-meta')]:
 377                 self.assertEqual(r.pop(k), object_info[k])
 378             self.assertFalse(len(r))
 379             self.assertEqual(GOI.mock_calls[-1], call(obj, version=version))
 380
 381     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 382     def test_del_object_meta(self, post):
 383         metakey = '50m3m3t4k3y'
 384         self.client.del_object_meta(obj, metakey)
 385         post.assert_called_once_with(obj, update=True, metadata={metakey: ''})
 386
 387     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 388     def test_copy_object(self, put):
 389         src_cont = 'src-c0nt41n3r'
 390         src_obj = 'src-0bj'
 391         dst_cont = 'dst-c0nt41n3r'
 392         dst_obj = 'dst-0bj'
 393         expected = call(
 394             src_obj,
 395             content_length=0,
 396             source_account=None,
 397             success=201,
 398             copy_from='/%s/%s' % (src_cont, src_obj),
 399             delimiter=None,
 400             content_type=None,
 401             source_version=None,
 402             public=False)
 403         self.client.copy_object(src_cont, src_obj, dst_cont)
 404         self.assertEqual(put.mock_calls[-1], expected)
 405         self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
 406         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 407         kwargs = dict(
 408             source_version='src-v3r510n',
 409             source_account='src-4cc0un7',
 410             public=True,
 411             content_type='c0n73n7Typ3',
 412             delimiter='5')
 413         self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
 414         for k, v in kwargs.items():
 415             self.assertEqual(v, put.mock_calls[-1][2][k])
 416
 417     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 418     def test_move_object(self, put):
 419         src_cont = 'src-c0nt41n3r'
 420         src_obj = 'src-0bj'
 421         dst_cont = 'dst-c0nt41n3r'
 422         dst_obj = 'dst-0bj'
 423         expected = call(
 424             src_obj,
 425             content_length=0,
 426             source_account=None,
 427             success=201,
 428             move_from='/%s/%s' % (src_cont, src_obj),
 429             delimiter=None,
 430             content_type=None,
 431             source_version=None,
 432             public=False)
 433         self.client.move_object(src_cont, src_obj, dst_cont)
 434         self.assertEqual(put.mock_calls[-1], expected)
 435         self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
 436         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 437         kwargs = dict(
 438             source_version='src-v3r510n',
 439             source_account='src-4cc0un7',
 440             public=True,
 441             content_type='c0n73n7Typ3',
 442             delimiter='5')
 443         self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
 444         for k, v in kwargs.items():
 445             self.assertEqual(v, put.mock_calls[-1][2][k])
 446
 447     #  Pithos+ only methods
 448
 449     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 450     def test_purge_container(self, CD):
 451         self.client.purge_container()
 452         self.assertTrue('until' in CD.mock_calls[-1][2])
 453         cont = self.client.container
 454         self.client.purge_container('another-container')
 455         self.assertEqual(self.client.container, cont)
 456
 457     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 458     def test_upload_object_unchunked(self, put):
 459         num_of_blocks = 8
 460         tmpFile = self._create_temp_file(num_of_blocks)
 461         expected = dict(
 462                 success=201,
 463                 data=num_of_blocks * 4 * 1024 * 1024,
 464                 etag='some-etag',
 465                 content_encoding='some content_encoding',
 466                 content_type='some content-type',
 467                 content_disposition='some content_disposition',
 468                 public=True,
 469                 permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 470         self.client.upload_object_unchunked(obj, tmpFile)
 471         self.assertEqual(put.mock_calls[-1][1], (obj,))
 472         self.assertEqual(
 473             sorted(put.mock_calls[-1][2].keys()),
 474             sorted(expected.keys()))
 475         kwargs = dict(expected)
 476         kwargs.pop('success')
 477         kwargs['size'] = kwargs.pop('data')
 478         kwargs['sharing'] = kwargs.pop('permissions')
 479         tmpFile.seek(0)
 480         self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
 481         pmc = put.mock_calls[-1][2]
 482         for k, v in expected.items():
 483             if k == 'data':
 484                 self.assertEqual(len(pmc[k]), v)
 485             else:
 486                 self.assertEqual(pmc[k], v)
 487         self.assertRaises(
 488             ClientError,
 489             self.client.upload_object_unchunked,
 490             obj, tmpFile, withHashFile=True)
 491
 492     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 493     def test_create_object_by_manifestation(self, put):
 494         manifest = '%s/%s' % (self.client.container, obj)
 495         kwargs = dict(
 496                 etag='some-etag',
 497                 content_encoding='some content_encoding',
 498                 content_type='some content-type',
 499                 content_disposition='some content_disposition',
 500                 public=True,
 501                 sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 502         self.client.create_object_by_manifestation(obj)
 503         expected = dict(content_length=0, manifest=manifest)
 504         for k in kwargs:
 505             expected['permissions' if k == 'sharing' else k] = None
 506         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 507         self.client.create_object_by_manifestation(obj, **kwargs)
 508         expected.update(kwargs)
 509         expected['permissions'] = expected.pop('sharing')
 510         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 511
 512     @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
 513     @patch('%s.object_get' % pithos_pkg, return_value=FR())
 514     def test_download_object(self, GET, GOH):
 515         num_of_blocks = 8
 516         tmpFile = self._create_temp_file(num_of_blocks)
 517         FR.content = tmpFile.read(4 * 1024 * 1024)
 518         tmpFile = self._create_temp_file(num_of_blocks)
 519         num_of_blocks = len(object_hashmap['hashes'])
 520         kwargs = dict(
 521             resume=True,
 522             version='version',
 523             range_str='10-20',
 524             if_match='if and only if',
 525             if_none_match='if and only not',
 526             if_modified_since='what if not?',
 527             if_unmodified_since='this happens if not!',
 528             async_headers=dict(Range='bytes=0-88888888'))
 529
 530         self.client.download_object(obj, tmpFile)
 531         self.assertEqual(len(GET.mock_calls), num_of_blocks)
 532         self.assertEqual(GET.mock_calls[-1][1], (obj,))
 533         for k, v in kwargs.items():
 534             if k == 'async_headers':
 535                 self.assertTrue('Range' in GET.mock_calls[-1][2][k])
 536             elif k in ('resume', 'range_str'):
 537                 continue
 538             else:
 539                 self.assertEqual(GET.mock_calls[-1][2][k], None)
 540
 541         #  Check ranges are consecutive
 542         starts = []
 543         ends = []
 544         for c in GET.mock_calls:
 545             rng_str = c[2]['async_headers']['Range']
 546             (start, rng_str) = rng_str.split('=')
 547             (start, end) = rng_str.split('-')
 548             starts.append(start)
 549             ends.append(end)
 550         ends = sorted(ends)
 551         for i, start in enumerate(sorted(starts)):
 552             if i:
 553                 int(ends[i - 1]) == int(start) - 1
 554
 555         #  With progress bars
 556         try:
 557             from progress.bar import ShadyBar
 558             dl_bar = ShadyBar('Mock dl')
 559         except ImportError:
 560             dl_bar = None
 561
 562         if dl_bar:
 563
 564             def blck_gen(n):
 565                 for i in dl_bar.iter(range(n)):
 566                     yield
 567                 yield
 568
 569             tmpFile.seek(0)
 570             self.client.download_object(obj, tmpFile, download_cb=blck_gen)
 571             self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
 572
 573         tmpFile.seek(0)
 574         kwargs.pop('async_headers')
 575         kwargs.pop('resume')
 576         self.client.download_object(obj, tmpFile, **kwargs)
 577         for k, v in kwargs.items():
 578             if k == 'range_str':
 579                 self.assertEqual(
 580                     GET.mock_calls[-1][2]['data_range'],
 581                     'bytes=%s' % v)
 582             else:
 583                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 584
 585         #  ALl options on no tty
 586
 587         def foo():
 588             return True
 589
 590         tmpFile.seek(0)
 591         tmpFile.isatty = foo
 592         self.client.download_object(obj, tmpFile, **kwargs)
 593         for k, v in kwargs.items():
 594             if k == 'range_str':
 595                 self.assertTrue('data_range' in GET.mock_calls[-1][2])
 596             else:
 597                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 598
 599     def test_get_object_hashmap(self):
 600         FR.json = object_hashmap
 601         for empty in (304, 412):
 602             with patch.object(
 603                     PithosClient, 'object_get',
 604                     side_effect=ClientError('Empty', status=empty)):
 605                 r = self.client.get_object_hashmap(obj)
 606                 self.assertEqual(r, {})
 607         exp_args = dict(
 608             hashmap=True,
 609             data_range=None,
 610             version=None,
 611             if_etag_match=None,
 612             if_etag_not_match=None,
 613             if_modified_since=None,
 614             if_unmodified_since=None)
 615         kwargs = dict(
 616             version='s0m3v3r51on',
 617             if_match='if match',
 618             if_none_match='if non match',
 619             if_modified_since='some date here',
 620             if_unmodified_since='some date here',
 621             data_range='10-20')
 622         with patch.object(
 623                 PithosClient, 'object_get',
 624                 return_value=FR()) as get:
 625             r = self.client.get_object_hashmap(obj)
 626             self.assertEqual(r, object_hashmap)
 627             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 628             r = self.client.get_object_hashmap(obj, **kwargs)
 629             exp_args['if_etag_match'] = kwargs.pop('if_match')
 630             exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
 631             exp_args.update(kwargs)
 632             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 633
 634     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 635     def test_set_account_group(self, post):
 636         (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
 637         self.client.set_account_group(group, usernames)
 638         post.assert_called_once_with(update=True, groups={group: usernames})
 639
 640     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 641     def test_del_account_group(self, post):
 642         group = 'aU53rGr0up'
 643         self.client.del_account_group(group)
 644         post.assert_called_once_with(update=True, groups={group: []})
 645
 646     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 647     def test_get_account_quota(self, GAI):
 648         key = 'x-account-policy-quota'
 649         r = self.client.get_account_quota()
 650         GAI.assert_called_once_with()
 651         self.assertEqual(r[key], account_info[key])
 652
 653     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 654     def test_get_account_versioning(self, GAI):
 655         key = 'x-account-policy-versioning'
 656         r = self.client.get_account_versioning()
 657         GAI.assert_called_once_with()
 658         self.assertEqual(r[key], account_info[key])
 659
 660     def test_get_account_meta(self):
 661         key = 'x-account-meta-'
 662         with patch.object(
 663                 PithosClient, 'get_account_info',
 664                 return_value=account_info):
 665             r = self.client.get_account_meta()
 666             keys = [k for k in r if k.startswith(key)]
 667             self.assertFalse(keys)
 668         acc_info = dict(account_info)
 669         acc_info['%sk1' % key] = 'v1'
 670         acc_info['%sk2' % key] = 'v2'
 671         acc_info['%sk3' % key] = 'v3'
 672         with patch.object(
 673                 PithosClient, 'get_account_info',
 674                 return_value=acc_info):
 675             r = self.client.get_account_meta()
 676             for k in [k for k in acc_info if k.startswith(key)]:
 677                 self.assertEqual(r[k], acc_info[k])
 678
 679     def test_get_account_group(self):
 680         key = 'x-account-group-'
 681         with patch.object(
 682                 PithosClient, 'get_account_info',
 683                 return_value=account_info):
 684             r = self.client.get_account_group()
 685             keys = [k for k in r if k.startswith(key)]
 686             self.assertFalse(keys)
 687         acc_info = dict(account_info)
 688         acc_info['%sk1' % key] = 'g1'
 689         acc_info['%sk2' % key] = 'g2'
 690         acc_info['%sk3' % key] = 'g3'
 691         with patch.object(
 692                 PithosClient, 'get_account_info',
 693                 return_value=acc_info):
 694             r = self.client.get_account_group()
 695             for k in [k for k in acc_info if k.startswith(key)]:
 696                 self.assertEqual(r[k], acc_info[k])
 697
 698     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 699     def test_set_account_meta(self, post):
 700         metas = dict(k1='v1', k2='v2', k3='v3')
 701         self.client.set_account_meta(metas)
 702         post.assert_called_once_with(update=True, metadata=metas)
 703
 704     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 705     def test_set_account_quota(self, post):
 706         qu = 1024
 707         self.client.set_account_quota(qu)
 708         post.assert_called_once_with(update=True, quota=qu)
 709
 710     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 711     def test_set_account_versioning(self, post):
 712         vrs = 'n3wV3r51on1ngTyp3'
 713         self.client.set_account_versioning(vrs)
 714         post.assert_called_once_with(update=True, versioning=vrs)
 715
 716     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 717     def test_del_container(self, delete):
 718         for kwarg in (
 719                 dict(delimiter=None, until=None),
 720                 dict(delimiter='X', until='50m3d473')):
 721             self.client.del_container(**kwarg)
 722             expected = dict(kwarg)
 723             expected['success'] = (204, 404, 409)
 724             self.assertEqual(delete.mock_calls[-1], call(**expected))
 725         for status_code in (404, 409):
 726             FR.status_code = status_code
 727             self.assertRaises(ClientError, self.client.del_container)
 728
 729     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 730     def test_get_container_versioning(self, GCI):
 731         key = 'x-container-policy-versioning'
 732         cont = 'c0n7-417'
 733         bu_cnt = self.client.container
 734         for container in (None, cont):
 735             r = self.client.get_container_versioning(container=container)
 736             self.assertEqual(r[key], container_info[key])
 737             self.assertEqual(GCI.mock_calls[-1], call())
 738             self.assertEqual(bu_cnt, self.client.container)
 739
 740     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 741     def test_get_container_quota(self, GCI):
 742         key = 'x-container-policy-quota'
 743         cont = 'c0n7-417'
 744         bu_cnt = self.client.container
 745         for container in (None, cont):
 746             r = self.client.get_container_quota(container=container)
 747             self.assertEqual(r[key], container_info[key])
 748             self.assertEqual(GCI.mock_calls[-1], call())
 749             self.assertEqual(bu_cnt, self.client.container)
 750
 751     def test_get_container_meta(self):
 752         somedate = '50m3d473'
 753         key = 'x-container-meta'
 754         metaval = '50m3m374v41'
 755         container_plus = dict(container_info)
 756         container_plus[key] = metaval
 757         for ret in ((container_info, {}), (container_plus, {key: metaval})):
 758             with patch.object(
 759                     PithosClient,
 760                     'get_container_info',
 761                     return_value=ret[0]) as GCI:
 762                 for until in (None, somedate):
 763                     r = self.client.get_container_meta(until=until)
 764                     self.assertEqual(r, ret[1])
 765                     self.assertEqual(GCI.mock_calls[-1], call(until=until))
 766
 767     def test_get_container_object_meta(self):
 768         somedate = '50m3d473'
 769         key = 'x-container-object-meta'
 770         metaval = '50m3m374v41'
 771         container_plus = dict(container_info)
 772         container_plus[key] = metaval
 773         for ret in (
 774                 (container_info, {key: ''}),
 775                 (container_plus, {key: metaval})):
 776             with patch.object(
 777                     PithosClient,
 778                     'get_container_info',
 779                     return_value=ret[0]) as GCI:
 780                 for until in (None, somedate):
 781                     r = self.client.get_container_object_meta(until=until)
 782                     self.assertEqual(r, ret[1])
 783                     self.assertEqual(GCI.mock_calls[-1], call(until=until))
 784
 785     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 786     def test_set_container_meta(self, post):
 787         metas = dict(k1='v1', k2='v2', k3='v3')
 788         self.client.set_container_meta(metas)
 789         post.assert_called_once_with(update=True, metadata=metas)
 790
 791     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 792     def test_del_container_meta(self, AP):
 793         self.client.del_container_meta('somekey')
 794         AP.assert_called_once_with(update=True, metadata={'somekey': ''})
 795
 796     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 797     def test_set_container_quota(self, post):
 798         qu = 1024
 799         self.client.set_container_quota(qu)
 800         post.assert_called_once_with(update=True, quota=qu)
 801
 802     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 803     def test_set_container_versioning(self, post):
 804         vrs = 'n3wV3r51on1ngTyp3'
 805         self.client.set_container_versioning(vrs)
 806         post.assert_called_once_with(update=True, versioning=vrs)
 807
 808     @patch('%s.object_delete' % pithos_pkg, return_value=FR())
 809     def test_del_object(self, delete):
 810         for kwarg in (
 811                 dict(delimiter=None, until=None),
 812                 dict(delimiter='X', until='50m3d473')):
 813             self.client.del_object(obj, **kwarg)
 814             self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
 815
 816     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 817     def test_set_object_meta(self, post):
 818         metas = dict(k1='v1', k2='v2', k3='v3')
 819         self.assertRaises(
 820             AssertionError,
 821             self.client.set_object_meta,
 822             obj, 'Non dict arg')
 823         self.client.set_object_meta(obj, metas)
 824         post.assert_called_once_with(obj, update=True, metadata=metas)
 825
 826     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 827     def test_publish_object(self, post):
 828         oinfo = dict(object_info)
 829         val = 'pubL1c'
 830         oinfo['x-object-public'] = val
 831         with patch.object(
 832                 PithosClient, 'get_object_info',
 833                 return_value=oinfo) as GOF:
 834             r = self.client.publish_object(obj)
 835             self.assertEqual(
 836                 post.mock_calls[-1],
 837                 call(obj, public=True, update=True))
 838             self.assertEqual(GOF.mock_calls[-1], call(obj))
 839             self.assertEqual(r, '%s%s' % (self.url[:-6], val))
 840
 841     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 842     def test_unpublish_object(self, post):
 843         self.client.unpublish_object(obj)
 844         post.assert_called_once_with(obj, public=False, update=True)
 845
 846     def test_get_object_sharing(self):
 847         info = dict(object_info)
 848         expected = dict(read='u1,g1,u2', write='u1')
 849         info['x-object-sharing'] = '; '.join(
 850             ['%s=%s' % (k, v) for k, v in expected.items()])
 851         with patch.object(
 852                 PithosClient, 'get_object_info',
 853                 return_value=info) as GOF:
 854             r = self.client.get_object_sharing(obj)
 855             self.assertEqual(GOF.mock_calls[-1], call(obj))
 856             self.assert_dicts_are_equal(r, expected)
 857             info['x-object-sharing'] = '//'.join(
 858                 ['%s=%s' % (k, v) for k, v in expected.items()])
 859             self.assertRaises(
 860                 ValueError,
 861                 self.client.get_object_sharing,
 862                 obj)
 863             info['x-object-sharing'] = '; '.join(
 864                 ['%s:%s' % (k, v) for k, v in expected.items()])
 865             self.assertRaises(
 866                 ClientError,
 867                 self.client.get_object_sharing,
 868                 obj)
 869             info['x-object-sharing'] = 'read=%s' % expected['read']
 870             r = self.client.get_object_sharing(obj)
 871             expected.pop('write')
 872             self.assert_dicts_are_equal(r, expected)
 873
 874     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 875     def test_set_object_sharing(self, OP):
 876         read_perms = ['u1', 'g1', 'u2', 'g2']
 877         write_perms = ['u1', 'g1']
 878         for kwargs in (
 879                 dict(read_permition=read_perms, write_permition=write_perms),
 880                 dict(read_permition=read_perms),
 881                 dict(write_permition=write_perms),
 882                 dict()):
 883             self.client.set_object_sharing(obj, **kwargs)
 884             kwargs['read'] = kwargs.pop('read_permition', '')
 885             kwargs['write'] = kwargs.pop('write_permition', '')
 886             self.assertEqual(
 887                 OP.mock_calls[-1],
 888                 call(obj, update=True, permissions=kwargs))
 889
 890     @patch('%s.set_object_sharing' % pithos_pkg)
 891     def test_del_object_sharing(self, SOS):
 892         self.client.del_object_sharing(obj)
 893         SOS.assert_called_once_with(obj)
 894
 895     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 896     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 897     def test_append_object(self, post, GCI):
 898         num_of_blocks = 4
 899         tmpFile = self._create_temp_file(num_of_blocks)
 900         tmpFile.seek(0, 2)
 901         file_size = tmpFile.tell()
 902         for turn in range(2):
 903             tmpFile.seek(0, 0)
 904
 905             try:
 906                 from progress.bar import ShadyBar
 907                 apn_bar = ShadyBar('Mock append')
 908             except ImportError:
 909                 apn_bar = None
 910
 911             if apn_bar:
 912
 913                 def append_gen(n):
 914                     for i in apn_bar.iter(range(n)):
 915                         yield
 916                     yield
 917
 918             else:
 919                 append_gen = None
 920
 921             self.client.append_object(
 922                 obj, tmpFile,
 923                 upload_cb=append_gen if turn else None)
 924             self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
 925             (args, kwargs) = post.mock_calls[-1][1:3]
 926             self.assertEqual(args, (obj,))
 927             self.assertEqual(kwargs['content_length'], len(kwargs['data']))
 928             fsize = num_of_blocks * int(kwargs['content_length'])
 929             self.assertEqual(fsize, file_size)
 930             self.assertEqual(kwargs['content_range'], 'bytes */*')
 931             exp = 'application/octet-stream'
 932             self.assertEqual(kwargs['content_type'], exp)
 933             self.assertEqual(kwargs['update'], True)
 934
 935     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 936     def test_truncate_object(self, post):
 937         upto_bytes = 377
 938         self.client.truncate_object(obj, upto_bytes)
 939         post.assert_called_once_with(
 940             obj,
 941             update=True,
 942             object_bytes=upto_bytes,
 943             content_range='bytes 0-%s/*' % upto_bytes,
 944             content_type='application/octet-stream',
 945             source_object='/%s/%s' % (self.client.container, obj))
 946
 947     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 948     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 949     def test_overwrite_object(self, post, GCI):
 950         num_of_blocks = 4
 951         tmpFile = self._create_temp_file(num_of_blocks)
 952         tmpFile.seek(0, 2)
 953         file_size = tmpFile.tell()
 954         info = dict(object_info)
 955         info['content-length'] = file_size
 956         block_size = container_info['x-container-block-size']
 957         with patch.object(
 958                 PithosClient, 'get_object_info',
 959                 return_value=info) as GOI:
 960             for start, end in (
 961                     (0, file_size + 1),
 962                     (file_size + 1, file_size + 2)):
 963                 tmpFile.seek(0, 0)
 964                 self.assertRaises(
 965                     ClientError,
 966                     self.client.overwrite_object,
 967                     obj, start, end, tmpFile)
 968             for start, end in ((0, 144), (144, 233), (233, file_size)):
 969                 tmpFile.seek(0, 0)
 970                 owr_gen = None
 971                 exp_size = end - start + 1
 972                 if not start or exp_size > block_size:
 973                     try:
 974                         from progress.bar import ShadyBar
 975                         owr_bar = ShadyBar('Mock append')
 976                     except ImportError:
 977                         owr_bar = None
 978
 979                     if owr_bar:
 980
 981                         def owr_gen(n):
 982                             for i in owr_bar.iter(range(n)):
 983                                 yield
 984                             yield
 985
 986                     if exp_size > block_size:
 987                         exp_size = exp_size % block_size or block_size
 988
 989                 self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
 990                 self.assertEqual(GOI.mock_calls[-1], call(obj))
 991                 self.assertEqual(GCI.mock_calls[-1], call())
 992                 (args, kwargs) = post.mock_calls[-1][1:3]
 993                 self.assertEqual(args, (obj,))
 994                 self.assertEqual(len(kwargs['data']), exp_size)
 995                 self.assertEqual(kwargs['content_length'], exp_size)
 996                 self.assertEqual(kwargs['update'], True)
 997                 exp = 'application/octet-stream'
 998                 self.assertEqual(kwargs['content_type'], exp)
 999
1000     @patch('%s.set_param' % pithos_pkg)
1001     @patch('%s.get' % pithos_pkg, return_value=FR())
1002     def test_get_sharing_accounts(self, get, SP):
1003         FR.json = sharers
1004         for kws in (
1005                 dict(),
1006                 dict(limit='50m3-11m17'),
1007                 dict(marker='X'),
1008                 dict(limit='50m3-11m17', marker='X')):
1009             r = self.client.get_sharing_accounts(**kws)
1010             self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1011             self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
1012             limit, marker = kws.get('limit', None), kws.get('marker', None)
1013             self.assertEqual(SP.mock_calls[-2], call(
1014                 'limit', limit,
1015                 iff=limit is not None))
1016             self.assertEqual(SP.mock_calls[-1], call(
1017                 'marker', marker,
1018                 iff=marker is not None))
1019             for i in range(len(r)):
1020                 self.assert_dicts_are_equal(r[i], sharers[i])
1021
1022     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1023     def test_get_object_versionlist(self, get):
1024         info = dict(object_info)
1025         info['versions'] = ['v1', 'v2']
1026         FR.json = info
1027         r = self.client.get_object_versionlist(obj)
1028         get.assert_called_once_with(obj, format='json', version='list')
1029         self.assertEqual(r, info['versions'])
1030
1031 if __name__ == '__main__':
1032     from sys import argv
1033     from kamaki.clients.test import runTestCase
1034     not_found = True
1035     if not argv[1:] or argv[1] == 'Pithos':
1036         not_found = False
1037         runTestCase(Pithos, 'Pithos Client', argv[2:])
1038     if not argv[1:] or argv[1] == 'PithosRest':
1039         not_found = False
1040         runTestCase(PithosRest, 'PithosRest Client', argv[2:])
1041     if not_found:
1042         print('TestCase %s not found' % argv[1])