code.grnet.gr Git - kamaki/blob - kamaki/clients/pithos/test.py

   1 # Copyright 2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from unittest import TestCase
  35 from mock import patch, call
  36 from tempfile import NamedTemporaryFile
  37 from os import urandom
  38 from itertools import product
  39 from random import randint
  40
  41 try:
  42     from collections import OrderedDict
  43 except ImportError:
  44     from kamaki.clients.commisioning.utils.ordereddict import OrderedDict
  45
  46 from kamaki.clients import pithos, ClientError
  47
  48
  49 rest_pkg = 'kamaki.clients.pithos.rest_api.PithosRestClient'
  50 pithos_pkg = 'kamaki.clients.pithos.PithosClient'
  51
  52 user_id = 'ac0un7-1d-5tr1ng'
  53 obj = 'obj3c7N4m3'
  54
  55 account_info = {
  56     'content-language': 'en-us',
  57     'content-type': 'text/html; charset=utf-8',
  58     'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
  59     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  60     'server': 'gunicorn/0.14.5',
  61     'vary': 'Accept-Language',
  62     'x-account-bytes-used': '751615526',
  63     'x-account-container-count': 7,
  64     'x-account-policy-quota': 53687091200,
  65     'x-account-policy-versioning': 'auto'}
  66 container_info = {
  67     'content-language': 'en-us',
  68     'content-type': 'text/html; charset=utf-8',
  69     'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
  70     'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
  71     'server': 'gunicorn/0.14.5',
  72     'vary': 'Accept-Language',
  73     'x-container-block-hash': 'sha256',
  74     'x-container-block-size': 4194304,
  75     'x-container-bytes-used': 309528938,
  76     'x-container-object-count': 14,
  77     'x-container-object-meta': '',
  78     'x-container-policy-quota': 53687091200,
  79     'x-container-policy-versioning': 'auto'}
  80 object_info = {
  81     'content-language': 'en-us',
  82     'content-length': 254965,
  83     'content-type': 'application/octet-stream',
  84     'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
  85     'etag': '',
  86     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  87     'server': 'gunicorn/0.14.5',
  88     'vary': 'Accept-Language',
  89     'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
  90     'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
  91     'x-object-version': '525996',
  92     'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
  93     'x-object-meta-k1': 'v1',
  94     'x-object-meta-k2': 'v2'}
  95 container_list = [
  96     dict(
  97         count=2,
  98         last_modified="2013-02-27T11:56:09.893033+00:00",
  99         bytes=677076979,
 100         name="pithos",
 101         x_container_policy=dict(quota="21474836480", versioning="auto")),
 102     dict(
 103         count=0,
 104         last_modified="2012-10-23T12:25:17.229187+00:00",
 105         bytes=0,
 106         name="trash",
 107         x_container_policy=dict(quota="21474836480", versioning="auto"))]
 108 object_list = [
 109     dict(hash="",
 110         name="The_Secret_Garden.zip",
 111         x_object_public="/public/wdp9p",
 112         bytes=203304947,
 113         x_object_version_timestamp="1360237915.7027509",
 114         x_object_uuid="s0m3uu1df0r0bj0n3",
 115         last_modified="2013-02-07T11:51:55.702751+00:00",
 116         content_type="application/octet-stream",
 117         x_object_hash="0afdf29f71cd53126225c3f54ca",
 118         x_object_version=17737,
 119         x_object_modified_by=user_id),
 120     dict(hash="",
 121         name="The_Revealed_Garden.zip",
 122         x_object_public="/public/wpd7p",
 123         bytes=20330947,
 124         x_object_version_timestamp="13602915.7027509",
 125         x_object_uuid="s0m3uu1df0r0bj70w",
 126         last_modified="2013-02-07T11:51:55.702751+00:00",
 127         content_type="application/octet-stream",
 128         x_object_hash="0afdf29f71cd53126225c3f54ca",
 129         x_object_version=17737,
 130         x_object_modified_by=user_id)]
 131 object_hashmap = dict(
 132     block_hash="sha256", block_size=4194304, bytes=33554432,
 133     hashes=[
 134         "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
 135         "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
 136         "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
 137         "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
 138         "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
 139         "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
 140         "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
 141         "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
 142 sharers = [
 143     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
 144     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
 145     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
 146
 147
 148 class FR(object):
 149     """FR stands for Fake Response"""
 150     json = dict()
 151     headers = dict()
 152     content = json
 153     status = None
 154     status_code = 200
 155
 156     def release(self):
 157         pass
 158
 159
 160 class PithosRestClient(TestCase):
 161
 162     def setUp(self):
 163         self.url = 'https://www.example.com/pithos'
 164         self.token = 'p17h0570k3n'
 165         self.client = pithos.PithosRestClient(self.url, self.token)
 166         self.client.account = user_id
 167         self.client.container = 'c0nt@1n3r_i'
 168
 169     def tearDown(self):
 170         FR.headers = dict()
 171         FR.json = dict()
 172         FR.content = FR.json
 173
 174     @patch('%s.set_param' % rest_pkg)
 175     @patch('%s.set_header' % rest_pkg)
 176     @patch('%s.head' % rest_pkg, return_value=FR())
 177     def test_account_head(self, head, SH, SP):
 178         for params in product(
 179                 (None, '50m3-d473'),
 180                 (None, '50m3-07h3r-d473'),
 181                 (None, 'y37-4n7h3r-d473'),
 182                 ((), ('someval',), ('v1', 'v2',)),
 183                 (dict(), dict(success=200), dict(k='v', v='k'))):
 184             args, kwargs = params[-2], params[-1]
 185             params = params[:-2]
 186             self.client.account_head(*(params + args), **kwargs)
 187             unt = params[0]
 188             self.assertEqual(SP.mock_calls[-1], call('until', unt, iff=unt))
 189             IMS, IUS = params[1], params[2]
 190             self.assertEqual(SH.mock_calls[-2:], [
 191                 call('If-Modified-Since', IMS),
 192                 call('If-Unmodified-Since', IUS)])
 193             self.assertEqual(head.mock_calls[-1], call(
 194                 '/%s' % self.client.account,
 195                 *args,
 196                 success=kwargs.pop('success', 204),
 197                 **kwargs))
 198
 199     @patch('%s.set_param' % rest_pkg)
 200     @patch('%s.set_header' % rest_pkg)
 201     @patch('%s.get' % rest_pkg, return_value=FR())
 202     def test_account_get(self, get, SH, SP):
 203         keys = ('limit', 'marker', 'format', 'shared', 'until')
 204         for params in product(
 205                 (None, 42),
 206                 (None, 'X'),
 207                 ('json', 'xml'),
 208                 (False, True),
 209                 (None, '50m3-d473'),
 210                 (None, '50m3-07h3r-d473'),
 211                 (None, 'y37-4n7h3r-d473'),
 212                 ((), ('someval',), ('v1', 'v2',)),
 213                 (dict(), dict(success=200), dict(k='v', v='k'))):
 214             args, kwargs = params[-2], params[-1]
 215             params = params[:-2]
 216             self.client.account_get(*(params + args), **kwargs)
 217             self.assertEqual(SP.mock_calls[-5:],
 218                 [call(keys[i], iff=X) if (
 219                     i == 3) else call(
 220                         keys[i], X, iff=X) for i, X in enumerate(params[:5])])
 221             IMS, IUS = params[5], params[6]
 222             self.assertEqual(SH.mock_calls[-2:], [
 223                 call('If-Modified-Since', IMS),
 224                 call('If-Unmodified-Since', IUS)])
 225             self.assertEqual(get.mock_calls[-1], call(
 226                 '/%s' % self.client.account,
 227                 *args,
 228                 success=kwargs.pop('success', (200, 204)),
 229                 **kwargs))
 230
 231     @patch('%s.set_param' % rest_pkg)
 232     @patch('%s.set_header' % rest_pkg)
 233     @patch('%s.post' % rest_pkg, return_value=FR())
 234     def test_account_post(self, post, SH, SP):
 235         #keys = ('update', 'groups', 'metadata', 'quota', 'versioning')
 236         for pm in product(
 237                 (True, False),
 238                 ({}, dict(g=['u1', 'u2']), dict(g1=[], g2=['u1', 'u2'])),
 239                 (None, dict(k1='v1', k2='v2', k3='v2'), dict(k='v')),
 240                 (None, 42),
 241                 (None, 'v3r510n1ng'),
 242                 ((), ('someval',), ('v1', 'v2',)),
 243                 (dict(), dict(success=200), dict(k='v', v='k'))):
 244             args, kwargs = pm[-2:]
 245             pm = pm[:-2]
 246             self.client.account_post(*(pm + args), **kwargs)
 247             upd = pm[0]
 248             self.assertEqual(SP.mock_calls[-1], call('update', iff=upd))
 249             expected = []
 250             if pm[1]:
 251                 expected += [
 252                 call('X-Account-Group-%s' % k, v) for k, v in pm[1].items()]
 253             if pm[2]:
 254                 expected = [
 255                 call('X-Account-Meta-%s' % k, v) for k, v in pm[2].items()]
 256             expected = [
 257                 call('X-Account-Policy-Quota', pm[3]),
 258                 call('X-Account-Policy-Versioning', pm[4])]
 259             self.assertEqual(SH.mock_calls[- len(expected):], expected)
 260             self.assertEqual(post.mock_calls[-1], call(
 261                 '/%s' % self.client.account,
 262                 *args,
 263                 success=kwargs.pop('success', 202),
 264                 **kwargs))
 265
 266     @patch('%s.set_param' % rest_pkg)
 267     @patch('%s.set_header' % rest_pkg)
 268     @patch('%s.head' % rest_pkg, return_value=FR())
 269     def test_container_head(self, head, SH, SP):
 270         for pm in product(
 271                 (None, '4-d473'),
 272                 (None, '47h3r-d473'),
 273                 (None, 'y37-4n47h3r'),
 274                 ((), ('someval',)),
 275                 (dict(), dict(success=200), dict(k='v', v='k'))):
 276             args, kwargs = pm[-2:]
 277             pm = pm[:-2]
 278             self.client.container_head(*(pm + args), **kwargs)
 279             unt, ims, ius = pm[0:3]
 280             self.assertEqual(SP.mock_calls[-1], call('until', unt, iff=unt))
 281             self.assertEqual(SH.mock_calls[-2:], [
 282                 call('If-Modified-Since', ims),
 283                 call('If-Unmodified-Since', ius)])
 284             self.assertEqual(head.mock_calls[-1], call(
 285                 '/%s/%s' % (self.client.account, self.client.container),
 286                 *args,
 287                 success=kwargs.pop('success', 204),
 288                 **kwargs))
 289
 290     @patch('%s.set_param' % rest_pkg)
 291     @patch('%s.set_header' % rest_pkg)
 292     @patch('%s.get' % rest_pkg, return_value=FR())
 293     def test_container_get(self, get, SH, SP):
 294         for pm in product(
 295                 (None, 42),
 296                 (None, 'X'),
 297                 (None, 'some/prefix'),
 298                 (None, 'delimiter'),
 299                 (None, '/some/path'),
 300                 ('json', 'some-format'),
 301                 ([], ['k1', 'k2', 'k3']),
 302                 (False, True),
 303                 (None, 'unt1l-d473'),
 304                 (None, 'y37-4n47h3r'),
 305                 (None, '4n47h3r-d473'),
 306                 ((), ('someval',)),
 307                 (dict(), dict(success=400), dict(k='v', v='k'))):
 308             args, kwargs = pm[-2:]
 309             pm = pm[:-2]
 310             self.client.container_get(*(pm + args), **kwargs)
 311             lmt, mrk, prfx, dlm, path, frmt, meta, shr, unt = pm[:-2]
 312             exp = [call('limit', lmt, iff=lmt), call('marker', mrk, iff=mrk)]
 313             exp += [call('path', path)] if path else [
 314                 call('prefix', prfx, iff=prfx),
 315                 call('delimiter', dlm, iff=dlm)]
 316             exp += [call('format', frmt, iff=frmt), call('shared', iff=shr)]
 317             if meta:
 318                 exp += [call('meta', ','.join(meta))]
 319             exp += [call('until', unt, iff=unt)]
 320             self.assertEqual(SP.mock_calls[- len(exp):], exp)
 321             ims, ius = pm[-2:]
 322             self.assertEqual(SH.mock_calls[-2:], [
 323                 call('If-Modified-Since', ims),
 324                 call('If-Unmodified-Since', ius)])
 325             self.assertEqual(get.mock_calls[-1], call(
 326                 '/%s/%s' % (self.client.account, self.client.container),
 327                 *args,
 328                 success=kwargs.pop('success', 200),
 329                 **kwargs))
 330
 331     @patch('%s.set_header' % rest_pkg)
 332     @patch('%s.put' % rest_pkg, return_value=FR())
 333     def test_container_put(self, put, SH):
 334         for pm in product(
 335                 (None, 42),
 336                 (None, 'v3r51on1ng'),
 337                 (dict(), dict(k1='v2'), dict(k2='v2', k3='v3')),
 338                 ((), ('someval',)),
 339                 (dict(), dict(success=400), dict(k='v', v='k'))):
 340             args, kwargs = pm[-2:]
 341             pm = pm[:-2]
 342             self.client.container_put(*(pm + args), **kwargs)
 343             quota, versioning, metas = pm[-3:]
 344             exp = [
 345                 call('X-Container-Policy-Quota', quota),
 346                 call('X-Container-Policy-Versioning', versioning)] + [
 347                 call('X-Container-Meta-%s' % k, v) for k, v in metas.items()]
 348             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 349             self.assertEqual(put.mock_calls[-1], call(
 350                 '/%s/%s' % (self.client.account, self.client.container),
 351                 *args,
 352                 success=kwargs.pop('success', (201, 202)),
 353                 **kwargs))
 354
 355     @patch('%s.set_param' % rest_pkg)
 356     @patch('%s.set_header' % rest_pkg)
 357     @patch('%s.post' % rest_pkg, return_value=FR())
 358     def test_container_post(self, post, SH, SP):
 359         for pm in product(
 360                 (True, False),
 361                 ('json', 'some-format'),
 362                 (None, 'quota'),
 363                 (None, 'v3r51on1ng'),
 364                 (dict(), dict(k1='v2'), dict(k2='v2', k3='v3')),
 365                 (None, 'content-type'),
 366                 (None, 42),
 367                 (None, 'transfer-encoding'),
 368                 ((), ('someval',)),
 369                 (dict(), dict(success=400), dict(k='v', v='k'))):
 370             args, kwargs = pm[-2:]
 371             pm = pm[:-2]
 372             self.client.container_post(*(pm + args), **kwargs)
 373             upd, frmt = pm[:2]
 374             self.assertEqual(SP.mock_calls[-2:], [
 375                 call('update', iff=upd),
 376                 call('format', frmt, iff=frmt)])
 377             qta, vrs, metas, ctype, clen, trenc = pm[2:]
 378             prfx = 'X-Container-Meta-'
 379             exp = [
 380                 call('X-Container-Policy-Quota', qta),
 381                 call('X-Container-Policy-Versioning', vrs)] + [
 382                 call('%s%s' % (prfx, k), v) for k, v in metas.items()] + [
 383                 call('Content-Type', ctype),
 384                 call('Content-Length', clen),
 385                 call('Transfer-Encoding', trenc)]
 386             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 387             ims, ius = pm[-2:]
 388             self.assertEqual(post.mock_calls[-1], call(
 389                 '/%s/%s' % (self.client.account, self.client.container),
 390                 *args,
 391                 success=kwargs.pop('success', 202),
 392                 **kwargs))
 393
 394     @patch('%s.set_param' % rest_pkg)
 395     @patch('%s.delete' % rest_pkg, return_value=FR())
 396     def test_container_delete(self, delete, SP):
 397         for pm in product(
 398                 (None, 'd473'),
 399                 (None, 'd3l1m'),
 400                 ((), ('someval',)),
 401                 (dict(), dict(success=400), dict(k='v', v='k'))):
 402             args, kwargs = pm[-2:]
 403             pm = pm[:-2]
 404             self.client.container_delete(*(pm + args), **kwargs)
 405             unt, dlm = pm[-2:]
 406             self.assertEqual(SP.mock_calls[-2:], [
 407                 call('until', unt, iff=unt),
 408                 call('delimiter', dlm, iff=dlm)])
 409             self.assertEqual(delete.mock_calls[-1], call(
 410                 '/%s/%s' % (self.client.account, self.client.container),
 411                 *args,
 412                 success=kwargs.pop('success', 204),
 413                 **kwargs))
 414
 415     @patch('%s.set_param' % rest_pkg)
 416     @patch('%s.set_header' % rest_pkg)
 417     @patch('%s.head' % rest_pkg, return_value=FR())
 418     def test_object_head(self, head, SH, SP):
 419         for pm in product(
 420                 (None, 'v3r510n'),
 421                 (None, '1f-374g'),
 422                 (None, '1f-n0-74g'),
 423                 (None, '1f-m0d-51nc3'),
 424                 (None, '1f-unm0d-51nc3'),
 425                 ((), ('someval',)),
 426                 (dict(), dict(success=400), dict(k='v', v='k'))):
 427             args, kwargs = pm[-2:]
 428             pm = pm[:-2]
 429             self.client.object_head(obj, *(pm + args), **kwargs)
 430             vrs, etag, netag, ims, ius = pm[:5]
 431             self.assertEqual(
 432                 SP.mock_calls[-1],
 433                 call('version', vrs, iff=vrs))
 434             self.assertEqual(SH.mock_calls[-4:], [
 435                 call('If-Match', etag),
 436                 call('If-None-Match', netag),
 437                 call('If-Modified-Since', ims),
 438                 call('If-Unmodified-Since', ius)])
 439             acc, cont = self.client.account, self.client.container
 440             self.assertEqual(head.mock_calls[-1], call(
 441                 '/%s/%s/%s' % (acc, cont, obj),
 442                 *args,
 443                 success=kwargs.pop('success', 200),
 444                 **kwargs))
 445
 446     @patch('%s.set_param' % rest_pkg)
 447     @patch('%s.set_header' % rest_pkg)
 448     @patch('%s.get' % rest_pkg, return_value=FR())
 449     def test_object_get(self, get, SH, SP):
 450         for pm in product(
 451                 ('json', 'f0rm47'),
 452                 (False, True),
 453                 (None, 'v3r510n'),
 454                 (None, 'range=74-63'),
 455                 (False, True),
 456                 (None, '3746'),
 457                 (None, 'non-3746'),
 458                 (None, '1f-m0d'),
 459                 (None, '1f-unm0d'),
 460                 ((), ('someval',)),
 461                 (dict(), dict(success=400), dict(k='v', v='k'))):
 462             args, kwargs = pm[-2:]
 463             pm = pm[:-2]
 464             self.client.object_get(obj, *(pm + args), **kwargs)
 465             format, hashmap, version = pm[:3]
 466             self.assertEqual(SP.mock_calls[-3:], [
 467                 call('format', format, iff=format),
 468                 call('hashmap', hashmap, iff=hashmap),
 469                 call('version', version, iff=version)])
 470             rng, ifrng, im, inm, ims, ius = pm[-6:]
 471             self.assertEqual(SH.mock_calls[-6:], [
 472                 call('Range', rng),
 473                 call('If-Range', '', ifrng and rng),
 474                 call('If-Match', im),
 475                 call('If-None-Match', inm),
 476                 call('If-Modified-Since', ims),
 477                 call('If-Unmodified-Since', ius)])
 478             acc, cont = self.client.account, self.client.container
 479             self.assertEqual(get.mock_calls[-1], call(
 480                 '/%s/%s/%s' % (acc, cont, obj),
 481                 *args,
 482                 success=kwargs.pop('success', 200),
 483                 **kwargs))
 484
 485     @patch('%s.set_param' % rest_pkg)
 486     @patch('%s.set_header' % rest_pkg)
 487     @patch('%s.put' % rest_pkg, return_value=FR())
 488     def test_object_put(self, put, SH, SP):
 489         for pm in product(
 490                 ('json', 'f0rm47'),
 491                 (False, True),
 492                 (None, 'delim',),
 493                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 494                 (False, True),
 495                 (dict(), dict(k2='v2', k3='v3')),
 496                 ((), ('someval',)),
 497                 (dict(), dict(success=400), dict(k='v', v='k'))):
 498             args, kwargs = pm[-2:]
 499             pm = pm[:-2]
 500             terms = [None] * 13
 501             for i in range(len(terms)):
 502                 if randint(0, 2):
 503                     terms[i] = 'val_%s' % randint(13, 1024)
 504             self.client.object_put(
 505                 obj,
 506                 *(pm[:3] + tuple(terms) + pm[3:] + args),
 507                 **kwargs)
 508             format, hashmap, delimiter = pm[:3]
 509             self.assertEqual(SP.mock_calls[-3:], [
 510                 call('format', format, iff=format),
 511                 call('hashmap', hashmap, iff=hashmap),
 512                 call('delimiter', delimiter, iff=delimiter)])
 513             (
 514                 im, inm, etag, clen, ctype, trenc,
 515                 cp, mv, srcacc, srcvrs, conenc, condis, mnf) = terms
 516             perms, public, metas = pm[3:]
 517             exp = [
 518                 call('If-Match', im),
 519                 call('If-None-Match', inm),
 520                 call('ETag', etag),
 521                 call('Content-Length', clen),
 522                 call('Content-Type', ctype),
 523                 call('Transfer-Encoding', trenc),
 524                 call('X-Copy-From', cp),
 525                 call('X-Move-From', mv),
 526                 call('X-Source-Account', srcacc),
 527                 call('X-Source-Version', srcvrs),
 528                 call('Content-Encoding', conenc),
 529                 call('Content-Disposition', condis),
 530                 call('X-Object-Manifest', mnf)]
 531             if perms:
 532                 perm_str = ''
 533                 for ptype, pval in perms.items():
 534                     if pval:
 535                         perm_str += ';' if perm_str else ''
 536                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 537                 exp += [call('X-Object-Sharing', perm_str)]
 538             exp += [call('X-Object-Public', public)]
 539             for k, v in metas.items():
 540                 exp += [call('X-Object-Meta-%s' % k, v)]
 541             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 542             acc, cont = self.client.account, self.client.container
 543             self.assertEqual(put.mock_calls[-1], call(
 544                 '/%s/%s/%s' % (acc, cont, obj),
 545                 *args,
 546                 success=kwargs.pop('success', 201),
 547                 **kwargs))
 548
 549     @patch('%s.set_param' % rest_pkg)
 550     @patch('%s.set_header' % rest_pkg)
 551     @patch('%s.copy' % rest_pkg, return_value=FR())
 552     def test_object_copy(self, copy, SH, SP):
 553         dest = 'dest1n4710n'
 554         for pm in product(
 555                 ('json', 'f0rm47'),
 556                 (False, True),
 557                 (None, 'ifmatch'),
 558                 (None, 'ifnonematch'),
 559                 (None, 'destinationaccount'),
 560                 (None, 'content-type'),
 561                 (None, 'content-encoding'),
 562                 (None, 'content-disp'),
 563                 (None, 'source-version'),
 564                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 565                 (False, True),
 566                 (dict(), dict(k2='v2', k3='v3')),
 567                 ((), ('someval',)),
 568                 (dict(), dict(success=400), dict(k='v', v='k'))):
 569             args, kwargs = pm[-2:]
 570             pm = pm[:-2]
 571             self.client.object_copy(obj, dest, *(pm + args), **kwargs)
 572             format, ict = pm[:2]
 573             self.assertEqual(SP.mock_calls[-2:], [
 574                 call('format', format, iff=format),
 575                 call('ignore_content_type', iff=ict)])
 576             im, inm, da, ct, ce, cd, sv, perms, public, metas = pm[2:]
 577             exp = [call('If-Match', im),
 578                 call('If-None-Match', inm),
 579                 call('Destination', dest),
 580                 call('Destination-Account', da),
 581                 call('Content-Type', ct),
 582                 call('Content-Encoding', ce),
 583                 call('Content-Disposition', cd),
 584                 call('X-Source-Version', sv)]
 585             if perms:
 586                 perm_str = ''
 587                 for ptype, pval in perms.items():
 588                     if pval:
 589                         perm_str += ';' if perm_str else ''
 590                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 591                 exp += [call('X-Object-Sharing', perm_str)]
 592             exp += [call('X-Object-Public', public)]
 593             for k, v in metas.items():
 594                 exp += [call('X-Object-Meta-%s' % k, v)]
 595             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 596             acc, cont = self.client.account, self.client.container
 597             self.assertEqual(copy.mock_calls[-1], call(
 598                 '/%s/%s/%s' % (acc, cont, obj),
 599                 *args,
 600                 success=kwargs.pop('success', 201),
 601                 **kwargs))
 602
 603     @patch('%s.set_param' % rest_pkg)
 604     @patch('%s.set_header' % rest_pkg)
 605     @patch('%s.move' % rest_pkg, return_value=FR())
 606     def test_object_move(self, move, SH, SP):
 607         for pm in product(
 608                 ('json', 'f0rm47'),
 609                 (False, True),
 610                 (None, 'ifmatch'),
 611                 (None, 'ifnonematch'),
 612                 (None, 'destination'),
 613                 (None, 'destinationaccount'),
 614                 (None, 'content-type'),
 615                 (None, 'content-encoding'),
 616                 (None, 'content-disp'),
 617                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 618                 (False, True),
 619                 (dict(), dict(k2='v2', k3='v3')),
 620                 ((), ('someval',)),
 621                 (dict(), dict(success=400), dict(k='v', v='k'))):
 622             args, kwargs = pm[-2:]
 623             pm = pm[:-2]
 624             self.client.object_move(obj, *(pm + args), **kwargs)
 625             format, ict = pm[:2]
 626             self.assertEqual(SP.mock_calls[-2:], [
 627                 call('format', format, iff=format),
 628                 call('ignore_content_type', iff=ict)])
 629             im, inm, d, da, ct, ce, cd, perms, public, metas = pm[2:]
 630             exp = [call('If-Match', im),
 631                 call('If-None-Match', inm),
 632                 call('Destination', d),
 633                 call('Destination-Account', da),
 634                 call('Content-Type', ct),
 635                 call('Content-Encoding', ce),
 636                 call('Content-Disposition', cd)]
 637             if perms:
 638                 perm_str = ''
 639                 for ptype, pval in perms.items():
 640                     if pval:
 641                         perm_str += ';' if perm_str else ''
 642                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 643                 exp += [call('X-Object-Sharing', perm_str)]
 644             exp += [call('X-Object-Public', public)]
 645             for k, v in metas.items():
 646                 exp += [call('X-Object-Meta-%s' % k, v)]
 647             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 648             acc, cont = self.client.account, self.client.container
 649             self.assertEqual(move.mock_calls[-1], call(
 650                 '/%s/%s/%s' % (acc, cont, obj),
 651                 *args,
 652                 success=kwargs.pop('success', 201),
 653                 **kwargs))
 654
 655     @patch('%s.set_param' % rest_pkg)
 656     @patch('%s.set_header' % rest_pkg)
 657     @patch('%s.post' % rest_pkg, return_value=FR())
 658     def test_object_post(self, post, SH, SP):
 659         for pm in product(
 660                 ('json', 'f0rm47'),
 661                 (False, True),
 662                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 663                 (False, True),
 664                 (dict(), dict(k2='v2', k3='v3')),
 665                 ((), ('someval',)),
 666                 (dict(), dict(success=400), dict(k='v', v='k'))):
 667             args, kwargs = pm[-2:]
 668             pm = pm[:-2]
 669             terms = [None] * 13
 670             for i in range(len(terms)):
 671                 if randint(0, 2):
 672                     terms[i] = 'val_%s' % randint(13, 1024)
 673             self.client.object_post(
 674                 obj,
 675                 *(pm[:2] + tuple(terms) + pm[2:] + args),
 676                 **kwargs)
 677             format, update = pm[:2]
 678             self.assertEqual(SP.mock_calls[-2:], [
 679                 call('format', format, iff=format),
 680                 call('update', iff=update)])
 681             (
 682                 im, inm, clen, ctype, crng, trenc, cenc,
 683                 condis, srcobj, srcacc, srcvrs, obytes, mnfs) = terms
 684             exp = [
 685                 call('If-Match', im),
 686                 call('If-None-Match', inm),
 687                 call('Content-Length', clen, iff=not trenc),
 688                 call('Content-Type', ctype),
 689                 call('Content-Range', crng),
 690                 call('Transfer-Encoding', trenc),
 691                 call('Content-Encoding', cenc),
 692                 call('Content-Disposition', condis),
 693                 call('X-Source-Object', srcobj),
 694                 call('X-Source-Account', srcacc),
 695                 call('X-Source-Version', srcvrs),
 696                 call('X-Object-Bytes', obytes),
 697                 call('X-Object-Manifest', mnfs)]
 698             perms, public, metas = pm[2:]
 699             if perms:
 700                 perm_str = ''
 701                 for ptype, pval in perms.items():
 702                     if pval:
 703                         perm_str += ';' if perm_str else ''
 704                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 705                 exp += [call('X-Object-Sharing', perm_str)]
 706             exp += [call('X-Object-Public', public)]
 707             for k, v in metas.items():
 708                 exp += [call('X-Object-Meta-%s' % k, v)]
 709             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 710             acc, cont = self.client.account, self.client.container
 711             self.assertEqual(post.mock_calls[-1], call(
 712                 '/%s/%s/%s' % (acc, cont, obj),
 713                 *args,
 714                 success=kwargs.pop('success', (202, 204)),
 715                 **kwargs))
 716
 717     @patch('%s.set_param' % rest_pkg)
 718     @patch('%s.delete' % rest_pkg, return_value=FR())
 719     def test_object_delete(self, delete, SP):
 720         for pm in product(
 721                 (None, 'until'),
 722                 (None, 'delim'),
 723                 ((), ('someval',)),
 724                 (dict(), dict(success=400), dict(k='v', v='k'))):
 725             args, kwargs = pm[-2:]
 726             pm = pm[:-2]
 727             self.client.object_delete(
 728                 obj,
 729                 *(pm + args),
 730                 **kwargs)
 731             until, dlm = pm[-2:]
 732             self.assertEqual(SP.mock_calls[-2:], [
 733                 call('until', until, iff=until),
 734                 call('delimiter', dlm, iff=dlm)])
 735             acc, cont = self.client.account, self.client.container
 736             self.assertEqual(delete.mock_calls[-1], call(
 737                 '/%s/%s/%s' % (acc, cont, obj),
 738                 *args,
 739                 success=kwargs.pop('success', 204),
 740                 **kwargs))
 741
 742
 743 class PithosClient(TestCase):
 744
 745     files = []
 746
 747     def _create_temp_file(self, num_of_blocks):
 748         self.files.append(NamedTemporaryFile())
 749         tmpFile = self.files[-1]
 750         file_size = num_of_blocks * 4 * 1024 * 1024
 751         print('\n\tCreate tmp file')
 752         tmpFile.write(urandom(file_size))
 753         tmpFile.flush()
 754         tmpFile.seek(0)
 755         print('\t\tDone')
 756         return tmpFile
 757
 758     def assert_dicts_are_equal(self, d1, d2):
 759         for k, v in d1.items():
 760             self.assertTrue(k in d2)
 761             if isinstance(v, dict):
 762                 self.assert_dicts_are_equal(v, d2[k])
 763             else:
 764                 self.assertEqual(unicode(v), unicode(d2[k]))
 765
 766     def setUp(self):
 767         self.url = 'https://www.example.com/pithos'
 768         self.token = 'p17h0570k3n'
 769         self.client = pithos.PithosClient(self.url, self.token)
 770         self.client.account = user_id
 771         self.client.container = 'c0nt@1n3r_i'
 772
 773     def tearDown(self):
 774         FR.headers = dict()
 775         FR.status_code = 200
 776         FR.json = dict()
 777         FR.content = FR.json
 778         for f in self.files:
 779             f.close()
 780
 781     #  Pithos+ methods that extend storage API
 782
 783     @patch('%s.account_head' % pithos_pkg, return_value=FR())
 784     def test_get_account_info(self, AH):
 785         FR.headers = account_info
 786         for until in (None, 'un71L-d473'):
 787             r = self.client.get_account_info(until=until)
 788             self.assert_dicts_are_equal(r, account_info)
 789             self.assertEqual(AH.mock_calls[-1], call(until=until))
 790         FR.status_code = 401
 791         self.assertRaises(ClientError, self.client.get_account_info)
 792
 793     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 794     def test_del_account_meta(self, AP):
 795         keys = ['k1', 'k2', 'k3']
 796         for key in keys:
 797             self.client.del_account_meta(key)
 798             self.assertEqual(
 799                 AP.mock_calls[-1],
 800                 call(update=True, metadata={key: ''}))
 801
 802     @patch('%s.container_head' % pithos_pkg, return_value=FR())
 803     def test_get_container_info(self, CH):
 804         FR.headers = container_info
 805         r = self.client.get_container_info()
 806         self.assert_dicts_are_equal(r, container_info)
 807         u = 'some date'
 808         r = self.client.get_container_info(until=u)
 809         self.assertEqual(CH.mock_calls, [call(until=None), call(until=u)])
 810
 811     @patch('%s.account_get' % pithos_pkg, return_value=FR())
 812     def test_list_containers(self, get):
 813         FR.json = container_list
 814         r = self.client.list_containers()
 815         get.assert_called_once_with()
 816         for i in range(len(r)):
 817             self.assert_dicts_are_equal(r[i], container_list[i])
 818
 819     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 820     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 821     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 822     def test_upload_object(self, OP, CP, GCI):
 823         num_of_blocks = 8
 824         tmpFile = self._create_temp_file(num_of_blocks)
 825
 826         # Without kwargs
 827         self.client.upload_object(obj, tmpFile)
 828         self.assertEqual(GCI.mock_calls[-1], call())
 829         [call1, call2] = OP.mock_calls
 830
 831         (args1, kwargs1) = call1[1:3]
 832         (args2, kwargs2) = call2[1:3]
 833         self.assertEqual(args1, (obj,))
 834         expected1 = dict(
 835             hashmap=True,
 836             success=(201, 409),
 837             format='json',
 838             json=dict(
 839                 hashes=['s0m3h@5h'] * num_of_blocks,
 840                 bytes=num_of_blocks * 4 * 1024 * 1024),
 841             etag=None,
 842             content_encoding=None,
 843             content_type='application/octet-stream',
 844             content_disposition=None,
 845             public=None,
 846             permissions=None)
 847         for k, v in expected1.items():
 848             if k == 'json':
 849                 self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
 850                 self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
 851             else:
 852                 self.assertEqual(v, kwargs1[k])
 853
 854         (args2, kwargs2) = call2[1:3]
 855         self.assertEqual(args2, (obj,))
 856         expected2 = dict(
 857             json=dict(
 858                 hashes=['s0m3h@5h'] * num_of_blocks,
 859                 bytes=num_of_blocks * 4 * 1024 * 1024),
 860             content_type='application/octet-stream',
 861             hashmap=True,
 862             success=201,
 863             format='json')
 864         for k, v in expected2.items():
 865             if k == 'json':
 866                 self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
 867                 self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
 868             else:
 869                 self.assertEqual(v, kwargs2[k])
 870
 871         mock_offset = 2
 872
 873         #  With progress bars
 874         try:
 875             from progress.bar import ShadyBar
 876             blck_bar = ShadyBar('Mock blck calc.')
 877             upld_bar = ShadyBar('Mock uplds')
 878         except ImportError:
 879             blck_bar = None
 880             upld_bar = None
 881
 882         if blck_bar and upld_bar:
 883
 884             def blck_gen(n):
 885                 for i in blck_bar.iter(range(n)):
 886                     yield
 887                 yield
 888
 889             def upld_gen(n):
 890                 for i in upld_bar.iter(range(n)):
 891                     yield
 892                 yield
 893
 894             tmpFile.seek(0)
 895             self.client.upload_object(
 896                 obj, tmpFile,
 897                 hash_cb=blck_gen, upload_cb=upld_gen)
 898
 899             for i, c in enumerate(OP.mock_calls[-mock_offset:]):
 900                 self.assertEqual(OP.mock_calls[i], c)
 901
 902         #  With content-type
 903         tmpFile.seek(0)
 904         ctype = 'video/mpeg'
 905         sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
 906         self.client.upload_object(obj, tmpFile,
 907             content_type=ctype, sharing=sharing)
 908         self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
 909         self.assert_dicts_are_equal(
 910             OP.mock_calls[-2][2]['permissions'],
 911             sharing)
 912
 913         # With other args
 914         tmpFile.seek(0)
 915         kwargs = dict(
 916             etag='s0m3E74g',
 917             content_type=ctype,
 918             content_disposition=ctype + 'd15p051710n',
 919             public=True,
 920             content_encoding='802.11')
 921         self.client.upload_object(obj, tmpFile, **kwargs)
 922         for arg, val in kwargs.items():
 923             self.assertEqual(OP.mock_calls[-2][2][arg], val)
 924
 925     def test_get_object_info(self):
 926         FR.headers = object_info
 927         version = 'v3r510n'
 928         with patch.object(
 929                 pithos.PithosClient, 'object_head',
 930                 return_value=FR()) as head:
 931             r = self.client.get_object_info(obj)
 932             self.assertEqual(r, object_info)
 933             r = self.client.get_object_info(obj, version=version)
 934             self.assertEqual(head.mock_calls, [
 935                 call(obj, version=None),
 936                 call(obj, version=version)])
 937         with patch.object(
 938                 pithos.PithosClient, 'object_head',
 939                 side_effect=ClientError('Obj not found', 404)):
 940             self.assertRaises(
 941                 ClientError,
 942                 self.client.get_object_info,
 943                 obj, version=version)
 944
 945     @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
 946     def test_get_object_meta(self, GOI):
 947         for version in (None, 'v3r510n'):
 948             r = self.client.get_object_meta(obj, version)
 949             for k in [k for k in object_info if k.startswith('x-object-meta')]:
 950                 self.assertEqual(r.pop(k), object_info[k])
 951             self.assertFalse(len(r))
 952             self.assertEqual(GOI.mock_calls[-1], call(obj, version=version))
 953
 954     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 955     def test_del_object_meta(self, post):
 956         metakey = '50m3m3t4k3y'
 957         self.client.del_object_meta(obj, metakey)
 958         post.assert_called_once_with(obj, update=True, metadata={metakey: ''})
 959
 960     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 961     def test_copy_object(self, put):
 962         src_cont = 'src-c0nt41n3r'
 963         src_obj = 'src-0bj'
 964         dst_cont = 'dst-c0nt41n3r'
 965         dst_obj = 'dst-0bj'
 966         expected = call(
 967             src_obj,
 968             content_length=0,
 969             source_account=None,
 970             success=201,
 971             copy_from='/%s/%s' % (src_cont, src_obj),
 972             delimiter=None,
 973             content_type=None,
 974             source_version=None,
 975             public=False)
 976         self.client.copy_object(src_cont, src_obj, dst_cont)
 977         self.assertEqual(put.mock_calls[-1], expected)
 978         self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
 979         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 980         kwargs = dict(
 981             source_version='src-v3r510n',
 982             source_account='src-4cc0un7',
 983             public=True,
 984             content_type='c0n73n7Typ3',
 985             delimiter='5')
 986         self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
 987         for k, v in kwargs.items():
 988             self.assertEqual(v, put.mock_calls[-1][2][k])
 989
 990     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 991     def test_move_object(self, put):
 992         src_cont = 'src-c0nt41n3r'
 993         src_obj = 'src-0bj'
 994         dst_cont = 'dst-c0nt41n3r'
 995         dst_obj = 'dst-0bj'
 996         expected = call(
 997             src_obj,
 998             content_length=0,
 999             source_account=None,
1000             success=201,
1001             move_from='/%s/%s' % (src_cont, src_obj),
1002             delimiter=None,
1003             content_type=None,
1004             source_version=None,
1005             public=False)
1006         self.client.move_object(src_cont, src_obj, dst_cont)
1007         self.assertEqual(put.mock_calls[-1], expected)
1008         self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
1009         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
1010         kwargs = dict(
1011             source_version='src-v3r510n',
1012             source_account='src-4cc0un7',
1013             public=True,
1014             content_type='c0n73n7Typ3',
1015             delimiter='5')
1016         self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
1017         for k, v in kwargs.items():
1018             self.assertEqual(v, put.mock_calls[-1][2][k])
1019
1020     #  Pithos+ only methods
1021
1022     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
1023     def test_purge_container(self, CD):
1024         self.client.purge_container()
1025         self.assertTrue('until' in CD.mock_calls[-1][2])
1026         cont = self.client.container
1027         self.client.purge_container('another-container')
1028         self.assertEqual(self.client.container, cont)
1029
1030     @patch('%s.object_put' % pithos_pkg, return_value=FR())
1031     def test_upload_object_unchunked(self, put):
1032         num_of_blocks = 8
1033         tmpFile = self._create_temp_file(num_of_blocks)
1034         expected = dict(
1035                 success=201,
1036                 data=num_of_blocks * 4 * 1024 * 1024,
1037                 etag='some-etag',
1038                 content_encoding='some content_encoding',
1039                 content_type='some content-type',
1040                 content_disposition='some content_disposition',
1041                 public=True,
1042                 permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
1043         self.client.upload_object_unchunked(obj, tmpFile)
1044         self.assertEqual(put.mock_calls[-1][1], (obj,))
1045         self.assertEqual(
1046             sorted(put.mock_calls[-1][2].keys()),
1047             sorted(expected.keys()))
1048         kwargs = dict(expected)
1049         kwargs.pop('success')
1050         kwargs['size'] = kwargs.pop('data')
1051         kwargs['sharing'] = kwargs.pop('permissions')
1052         tmpFile.seek(0)
1053         self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
1054         pmc = put.mock_calls[-1][2]
1055         for k, v in expected.items():
1056             if k == 'data':
1057                 self.assertEqual(len(pmc[k]), v)
1058             else:
1059                 self.assertEqual(pmc[k], v)
1060         self.assertRaises(
1061             ClientError,
1062             self.client.upload_object_unchunked,
1063             obj, tmpFile, withHashFile=True)
1064
1065     @patch('%s.object_put' % pithos_pkg, return_value=FR())
1066     def test_create_object_by_manifestation(self, put):
1067         manifest = '%s/%s' % (self.client.container, obj)
1068         kwargs = dict(
1069                 etag='some-etag',
1070                 content_encoding='some content_encoding',
1071                 content_type='some content-type',
1072                 content_disposition='some content_disposition',
1073                 public=True,
1074                 sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
1075         self.client.create_object_by_manifestation(obj)
1076         expected = dict(content_length=0, manifest=manifest)
1077         for k in kwargs:
1078             expected['permissions' if k == 'sharing' else k] = None
1079         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
1080         self.client.create_object_by_manifestation(obj, **kwargs)
1081         expected.update(kwargs)
1082         expected['permissions'] = expected.pop('sharing')
1083         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
1084
1085     @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
1086     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1087     def test_download_object(self, GET, GOH):
1088         num_of_blocks = 8
1089         tmpFile = self._create_temp_file(num_of_blocks)
1090         FR.content = tmpFile.read(4 * 1024 * 1024)
1091         tmpFile = self._create_temp_file(num_of_blocks)
1092         num_of_blocks = len(object_hashmap['hashes'])
1093         kwargs = dict(
1094             resume=True,
1095             version='version',
1096             range_str='10-20',
1097             if_match='if and only if',
1098             if_none_match='if and only not',
1099             if_modified_since='what if not?',
1100             if_unmodified_since='this happens if not!',
1101             async_headers=dict(Range='bytes=0-88888888'))
1102
1103         self.client.download_object(obj, tmpFile)
1104         self.assertEqual(len(GET.mock_calls), num_of_blocks)
1105         self.assertEqual(GET.mock_calls[-1][1], (obj,))
1106         for k, v in kwargs.items():
1107             if k == 'async_headers':
1108                 self.assertTrue('Range' in GET.mock_calls[-1][2][k])
1109             elif k in ('resume', 'range_str'):
1110                 continue
1111             else:
1112                 self.assertEqual(GET.mock_calls[-1][2][k], None)
1113
1114         #  Check ranges are consecutive
1115         starts = []
1116         ends = []
1117         for c in GET.mock_calls:
1118             rng_str = c[2]['async_headers']['Range']
1119             (start, rng_str) = rng_str.split('=')
1120             (start, end) = rng_str.split('-')
1121             starts.append(start)
1122             ends.append(end)
1123         ends = sorted(ends)
1124         for i, start in enumerate(sorted(starts)):
1125             if i:
1126                 int(ends[i - 1]) == int(start) - 1
1127
1128         #  With progress bars
1129         try:
1130             from progress.bar import ShadyBar
1131             dl_bar = ShadyBar('Mock dl')
1132         except ImportError:
1133             dl_bar = None
1134
1135         if dl_bar:
1136
1137             def blck_gen(n):
1138                 for i in dl_bar.iter(range(n)):
1139                     yield
1140                 yield
1141
1142             tmpFile.seek(0)
1143             self.client.download_object(obj, tmpFile, download_cb=blck_gen)
1144             self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
1145
1146         tmpFile.seek(0)
1147         kwargs.pop('async_headers')
1148         kwargs.pop('resume')
1149         self.client.download_object(obj, tmpFile, **kwargs)
1150         for k, v in kwargs.items():
1151             if k == 'range_str':
1152                 self.assertEqual(
1153                     GET.mock_calls[-1][2]['data_range'],
1154                     'bytes=%s' % v)
1155             else:
1156                 self.assertEqual(GET.mock_calls[-1][2][k], v)
1157
1158         #  ALl options on no tty
1159
1160         def foo():
1161             return True
1162
1163         tmpFile.seek(0)
1164         tmpFile.isatty = foo
1165         self.client.download_object(obj, tmpFile, **kwargs)
1166         for k, v in kwargs.items():
1167             if k == 'range_str':
1168                 self.assertTrue('data_range' in GET.mock_calls[-1][2])
1169             else:
1170                 self.assertEqual(GET.mock_calls[-1][2][k], v)
1171
1172     def test_get_object_hashmap(self):
1173         FR.json = object_hashmap
1174         for empty in (304, 412):
1175             with patch.object(
1176                     pithos.PithosClient, 'object_get',
1177                     side_effect=ClientError('Empty', status=empty)):
1178                 r = self.client.get_object_hashmap(obj)
1179                 self.assertEqual(r, {})
1180         exp_args = dict(
1181             hashmap=True,
1182             data_range=None,
1183             version=None,
1184             if_etag_match=None,
1185             if_etag_not_match=None,
1186             if_modified_since=None,
1187             if_unmodified_since=None)
1188         kwargs = dict(
1189             version='s0m3v3r51on',
1190             if_match='if match',
1191             if_none_match='if non match',
1192             if_modified_since='some date here',
1193             if_unmodified_since='some date here',
1194             data_range='10-20')
1195         with patch.object(
1196                 pithos.PithosClient, 'object_get',
1197                 return_value=FR()) as get:
1198             r = self.client.get_object_hashmap(obj)
1199             self.assertEqual(r, object_hashmap)
1200             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
1201             r = self.client.get_object_hashmap(obj, **kwargs)
1202             exp_args['if_etag_match'] = kwargs.pop('if_match')
1203             exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
1204             exp_args.update(kwargs)
1205             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
1206
1207     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1208     def test_set_account_group(self, post):
1209         (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
1210         self.client.set_account_group(group, usernames)
1211         post.assert_called_once_with(update=True, groups={group: usernames})
1212
1213     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1214     def test_del_account_group(self, post):
1215         group = 'aU53rGr0up'
1216         self.client.del_account_group(group)
1217         post.assert_called_once_with(update=True, groups={group: []})
1218
1219     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
1220     def test_get_account_quota(self, GAI):
1221         key = 'x-account-policy-quota'
1222         r = self.client.get_account_quota()
1223         GAI.assert_called_once_with()
1224         self.assertEqual(r[key], account_info[key])
1225
1226     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
1227     def test_get_account_versioning(self, GAI):
1228         key = 'x-account-policy-versioning'
1229         r = self.client.get_account_versioning()
1230         GAI.assert_called_once_with()
1231         self.assertEqual(r[key], account_info[key])
1232
1233     def test_get_account_meta(self):
1234         key = 'x-account-meta-'
1235         with patch.object(
1236                 pithos.PithosClient, 'get_account_info',
1237                 return_value=account_info):
1238             r = self.client.get_account_meta()
1239             keys = [k for k in r if k.startswith(key)]
1240             self.assertFalse(keys)
1241         acc_info = dict(account_info)
1242         acc_info['%sk1' % key] = 'v1'
1243         acc_info['%sk2' % key] = 'v2'
1244         acc_info['%sk3' % key] = 'v3'
1245         with patch.object(
1246                 pithos.PithosClient, 'get_account_info',
1247                 return_value=acc_info):
1248             r = self.client.get_account_meta()
1249             for k in [k for k in acc_info if k.startswith(key)]:
1250                 self.assertEqual(r[k], acc_info[k])
1251
1252     def test_get_account_group(self):
1253         key = 'x-account-group-'
1254         with patch.object(
1255                 pithos.PithosClient, 'get_account_info',
1256                 return_value=account_info):
1257             r = self.client.get_account_group()
1258             keys = [k for k in r if k.startswith(key)]
1259             self.assertFalse(keys)
1260         acc_info = dict(account_info)
1261         acc_info['%sk1' % key] = 'g1'
1262         acc_info['%sk2' % key] = 'g2'
1263         acc_info['%sk3' % key] = 'g3'
1264         with patch.object(
1265                 pithos.PithosClient, 'get_account_info',
1266                 return_value=acc_info):
1267             r = self.client.get_account_group()
1268             for k in [k for k in acc_info if k.startswith(key)]:
1269                 self.assertEqual(r[k], acc_info[k])
1270
1271     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1272     def test_set_account_meta(self, post):
1273         metas = dict(k1='v1', k2='v2', k3='v3')
1274         self.client.set_account_meta(metas)
1275         post.assert_called_once_with(update=True, metadata=metas)
1276
1277     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1278     def test_set_account_quota(self, post):
1279         qu = 1024
1280         self.client.set_account_quota(qu)
1281         post.assert_called_once_with(update=True, quota=qu)
1282
1283     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1284     def test_set_account_versioning(self, post):
1285         vrs = 'n3wV3r51on1ngTyp3'
1286         self.client.set_account_versioning(vrs)
1287         post.assert_called_once_with(update=True, versioning=vrs)
1288
1289     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
1290     def test_del_container(self, delete):
1291         for kwarg in (
1292                 dict(delimiter=None, until=None),
1293                 dict(delimiter='X', until='50m3d473')):
1294             self.client.del_container(**kwarg)
1295             expected = dict(kwarg)
1296             expected['success'] = (204, 404, 409)
1297             self.assertEqual(delete.mock_calls[-1], call(**expected))
1298         for status_code in (404, 409):
1299             FR.status_code = status_code
1300             self.assertRaises(ClientError, self.client.del_container)
1301
1302     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1303     def test_get_container_versioning(self, GCI):
1304         key = 'x-container-policy-versioning'
1305         cont = 'c0n7-417'
1306         bu_cnt = self.client.container
1307         for container in (None, cont):
1308             r = self.client.get_container_versioning(container=container)
1309             self.assertEqual(r[key], container_info[key])
1310             self.assertEqual(GCI.mock_calls[-1], call())
1311             self.assertEqual(bu_cnt, self.client.container)
1312
1313     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1314     def test_get_container_quota(self, GCI):
1315         key = 'x-container-policy-quota'
1316         cont = 'c0n7-417'
1317         bu_cnt = self.client.container
1318         for container in (None, cont):
1319             r = self.client.get_container_quota(container=container)
1320             self.assertEqual(r[key], container_info[key])
1321             self.assertEqual(GCI.mock_calls[-1], call())
1322             self.assertEqual(bu_cnt, self.client.container)
1323
1324     def test_get_container_meta(self):
1325         somedate = '50m3d473'
1326         key = 'x-container-meta'
1327         metaval = '50m3m374v41'
1328         container_plus = dict(container_info)
1329         container_plus[key] = metaval
1330         for ret in ((container_info, {}), (container_plus, {key: metaval})):
1331             with patch.object(
1332                     pithos.PithosClient,
1333                     'get_container_info',
1334                     return_value=ret[0]) as GCI:
1335                 for until in (None, somedate):
1336                     r = self.client.get_container_meta(until=until)
1337                     self.assertEqual(r, ret[1])
1338                     self.assertEqual(GCI.mock_calls[-1], call(until=until))
1339
1340     def test_get_container_object_meta(self):
1341         somedate = '50m3d473'
1342         key = 'x-container-object-meta'
1343         metaval = '50m3m374v41'
1344         container_plus = dict(container_info)
1345         container_plus[key] = metaval
1346         for ret in (
1347                 (container_info, {key: ''}),
1348                 (container_plus, {key: metaval})):
1349             with patch.object(
1350                     pithos.PithosClient,
1351                     'get_container_info',
1352                     return_value=ret[0]) as GCI:
1353                 for until in (None, somedate):
1354                     r = self.client.get_container_object_meta(until=until)
1355                     self.assertEqual(r, ret[1])
1356                     self.assertEqual(GCI.mock_calls[-1], call(until=until))
1357
1358     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1359     def test_set_container_meta(self, post):
1360         metas = dict(k1='v1', k2='v2', k3='v3')
1361         self.client.set_container_meta(metas)
1362         post.assert_called_once_with(update=True, metadata=metas)
1363
1364     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1365     def test_del_container_meta(self, AP):
1366         self.client.del_container_meta('somekey')
1367         AP.assert_called_once_with(update=True, metadata={'somekey': ''})
1368
1369     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1370     def test_set_container_quota(self, post):
1371         qu = 1024
1372         self.client.set_container_quota(qu)
1373         post.assert_called_once_with(update=True, quota=qu)
1374
1375     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1376     def test_set_container_versioning(self, post):
1377         vrs = 'n3wV3r51on1ngTyp3'
1378         self.client.set_container_versioning(vrs)
1379         post.assert_called_once_with(update=True, versioning=vrs)
1380
1381     @patch('%s.object_delete' % pithos_pkg, return_value=FR())
1382     def test_del_object(self, delete):
1383         for kwarg in (
1384                 dict(delimiter=None, until=None),
1385                 dict(delimiter='X', until='50m3d473')):
1386             self.client.del_object(obj, **kwarg)
1387             self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
1388
1389     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1390     def test_set_object_meta(self, post):
1391         metas = dict(k1='v1', k2='v2', k3='v3')
1392         self.assertRaises(
1393             AssertionError,
1394             self.client.set_object_meta,
1395             obj, 'Non dict arg')
1396         self.client.set_object_meta(obj, metas)
1397         post.assert_called_once_with(obj, update=True, metadata=metas)
1398
1399     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1400     def test_publish_object(self, post):
1401         oinfo = dict(object_info)
1402         val = 'pubL1c'
1403         oinfo['x-object-public'] = val
1404         with patch.object(
1405                 pithos.PithosClient, 'get_object_info',
1406                 return_value=oinfo) as GOF:
1407             r = self.client.publish_object(obj)
1408             self.assertEqual(
1409                 post.mock_calls[-1],
1410                 call(obj, public=True, update=True))
1411             self.assertEqual(GOF.mock_calls[-1], call(obj))
1412             self.assertEqual(r, '%s%s' % (self.url[:-6], val))
1413
1414     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1415     def test_unpublish_object(self, post):
1416         self.client.unpublish_object(obj)
1417         post.assert_called_once_with(obj, public=False, update=True)
1418
1419     def test_get_object_sharing(self):
1420         info = dict(object_info)
1421         expected = dict(read='u1,g1,u2', write='u1')
1422         info['x-object-sharing'] = '; '.join(
1423             ['%s=%s' % (k, v) for k, v in expected.items()])
1424         with patch.object(
1425                 pithos.PithosClient, 'get_object_info',
1426                 return_value=info) as GOF:
1427             r = self.client.get_object_sharing(obj)
1428             self.assertEqual(GOF.mock_calls[-1], call(obj))
1429             self.assert_dicts_are_equal(r, expected)
1430             info['x-object-sharing'] = '//'.join(
1431                 ['%s=%s' % (k, v) for k, v in expected.items()])
1432             self.assertRaises(
1433                 ValueError,
1434                 self.client.get_object_sharing,
1435                 obj)
1436             info['x-object-sharing'] = '; '.join(
1437                 ['%s:%s' % (k, v) for k, v in expected.items()])
1438             self.assertRaises(
1439                 ClientError,
1440                 self.client.get_object_sharing,
1441                 obj)
1442             info['x-object-sharing'] = 'read=%s' % expected['read']
1443             r = self.client.get_object_sharing(obj)
1444             expected.pop('write')
1445             self.assert_dicts_are_equal(r, expected)
1446
1447     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1448     def test_set_object_sharing(self, OP):
1449         read_perms = ['u1', 'g1', 'u2', 'g2']
1450         write_perms = ['u1', 'g1']
1451         for kwargs in (
1452                 dict(read_permition=read_perms, write_permition=write_perms),
1453                 dict(read_permition=read_perms),
1454                 dict(write_permition=write_perms),
1455                 dict()):
1456             self.client.set_object_sharing(obj, **kwargs)
1457             kwargs['read'] = kwargs.pop('read_permition', '')
1458             kwargs['write'] = kwargs.pop('write_permition', '')
1459             self.assertEqual(
1460                 OP.mock_calls[-1],
1461                 call(obj, update=True, permissions=kwargs))
1462
1463     @patch('%s.set_object_sharing' % pithos_pkg)
1464     def test_del_object_sharing(self, SOS):
1465         self.client.del_object_sharing(obj)
1466         SOS.assert_called_once_with(obj)
1467
1468     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1469     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1470     def test_append_object(self, post, GCI):
1471         num_of_blocks = 4
1472         tmpFile = self._create_temp_file(num_of_blocks)
1473         tmpFile.seek(0, 2)
1474         file_size = tmpFile.tell()
1475         for turn in range(2):
1476             tmpFile.seek(0, 0)
1477
1478             try:
1479                 from progress.bar import ShadyBar
1480                 apn_bar = ShadyBar('Mock append')
1481             except ImportError:
1482                 apn_bar = None
1483
1484             if apn_bar:
1485
1486                 def append_gen(n):
1487                     for i in apn_bar.iter(range(n)):
1488                         yield
1489                     yield
1490
1491             else:
1492                 append_gen = None
1493
1494             self.client.append_object(
1495                 obj, tmpFile,
1496                 upload_cb=append_gen if turn else None)
1497             self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
1498             (args, kwargs) = post.mock_calls[-1][1:3]
1499             self.assertEqual(args, (obj,))
1500             self.assertEqual(kwargs['content_length'], len(kwargs['data']))
1501             fsize = num_of_blocks * int(kwargs['content_length'])
1502             self.assertEqual(fsize, file_size)
1503             self.assertEqual(kwargs['content_range'], 'bytes */*')
1504             exp = 'application/octet-stream'
1505             self.assertEqual(kwargs['content_type'], exp)
1506             self.assertEqual(kwargs['update'], True)
1507
1508     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1509     def test_truncate_object(self, post):
1510         upto_bytes = 377
1511         self.client.truncate_object(obj, upto_bytes)
1512         post.assert_called_once_with(
1513             obj,
1514             update=True,
1515             object_bytes=upto_bytes,
1516             content_range='bytes 0-%s/*' % upto_bytes,
1517             content_type='application/octet-stream',
1518             source_object='/%s/%s' % (self.client.container, obj))
1519
1520     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1521     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1522     def test_overwrite_object(self, post, GCI):
1523         num_of_blocks = 4
1524         tmpFile = self._create_temp_file(num_of_blocks)
1525         tmpFile.seek(0, 2)
1526         file_size = tmpFile.tell()
1527         info = dict(object_info)
1528         info['content-length'] = file_size
1529         block_size = container_info['x-container-block-size']
1530         with patch.object(
1531                 pithos.PithosClient, 'get_object_info',
1532                 return_value=info) as GOI:
1533             for start, end in (
1534                     (0, file_size + 1),
1535                     (file_size + 1, file_size + 2)):
1536                 tmpFile.seek(0, 0)
1537                 self.assertRaises(
1538                     ClientError,
1539                     self.client.overwrite_object,
1540                     obj, start, end, tmpFile)
1541             for start, end in ((0, 144), (144, 233), (233, file_size)):
1542                 tmpFile.seek(0, 0)
1543                 owr_gen = None
1544                 exp_size = end - start + 1
1545                 if not start or exp_size > block_size:
1546                     try:
1547                         from progress.bar import ShadyBar
1548                         owr_bar = ShadyBar('Mock append')
1549                     except ImportError:
1550                         owr_bar = None
1551
1552                     if owr_bar:
1553
1554                         def owr_gen(n):
1555                             for i in owr_bar.iter(range(n)):
1556                                 yield
1557                             yield
1558
1559                     if exp_size > block_size:
1560                         exp_size = exp_size % block_size or block_size
1561
1562                 self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
1563                 self.assertEqual(GOI.mock_calls[-1], call(obj))
1564                 self.assertEqual(GCI.mock_calls[-1], call())
1565                 (args, kwargs) = post.mock_calls[-1][1:3]
1566                 self.assertEqual(args, (obj,))
1567                 self.assertEqual(len(kwargs['data']), exp_size)
1568                 self.assertEqual(kwargs['content_length'], exp_size)
1569                 self.assertEqual(kwargs['update'], True)
1570                 exp = 'application/octet-stream'
1571                 self.assertEqual(kwargs['content_type'], exp)
1572
1573     @patch('%s.set_param' % pithos_pkg)
1574     @patch('%s.get' % pithos_pkg, return_value=FR())
1575     def test_get_sharing_accounts(self, get, SP):
1576         FR.json = sharers
1577         for kws in (
1578                 dict(),
1579                 dict(limit='50m3-11m17'),
1580                 dict(marker='X'),
1581                 dict(limit='50m3-11m17', marker='X')):
1582             r = self.client.get_sharing_accounts(**kws)
1583             self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1584             self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
1585             limit, marker = kws.get('limit', None), kws.get('marker', None)
1586             self.assertEqual(SP.mock_calls[-2], call(
1587                 'limit', limit,
1588                 iff=limit is not None))
1589             self.assertEqual(SP.mock_calls[-1], call(
1590                 'marker', marker,
1591                 iff=marker is not None))
1592             for i in range(len(r)):
1593                 self.assert_dicts_are_equal(r[i], sharers[i])
1594
1595     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1596     def test_get_object_versionlist(self, get):
1597         info = dict(object_info)
1598         info['versions'] = ['v1', 'v2']
1599         FR.json = info
1600         r = self.client.get_object_versionlist(obj)
1601         get.assert_called_once_with(obj, format='json', version='list')
1602         self.assertEqual(r, info['versions'])
1603
1604 if __name__ == '__main__':
1605     from sys import argv
1606     from kamaki.clients.test import runTestCase
1607     not_found = True
1608     if not argv[1:] or argv[1] == 'PithosClient':
1609         not_found = False
1610         runTestCase(PithosClient, 'Pithos Client', argv[2:])
1611     if not argv[1:] or argv[1] == 'PithosRestClient':
1612         not_found = False
1613         runTestCase(PithosRestClient, 'PithosRest Client', argv[2:])
1614     if not_found:
1615         print('TestCase %s not found' % argv[1])