code.grnet.gr Git - kamaki/blob - kamaki/clients/pithos/test.py

   1 # Copyright 2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from unittest import TestCase
  35 from mock import patch, call
  36 from tempfile import NamedTemporaryFile
  37 from os import urandom
  38 from itertools import product
  39 from random import randint
  40
  41 try:
  42     from collections import OrderedDict
  43 except ImportError:
  44     from kamaki.clients.commisioning.utils.ordereddict import OrderedDict
  45
  46 from kamaki.clients import ClientError
  47 from kamaki.clients.pithos import PithosClient, PithosRestClient
  48
  49
  50 rest_pkg = 'kamaki.clients.pithos.rest_api.PithosRestClient'
  51 pithos_pkg = 'kamaki.clients.pithos.PithosClient'
  52
  53 user_id = 'ac0un7-1d-5tr1ng'
  54 obj = 'obj3c7N4m3'
  55
  56 account_info = {
  57     'content-language': 'en-us',
  58     'content-type': 'text/html; charset=utf-8',
  59     'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
  60     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  61     'server': 'gunicorn/0.14.5',
  62     'vary': 'Accept-Language',
  63     'x-account-bytes-used': '751615526',
  64     'x-account-container-count': 7,
  65     'x-account-policy-quota': 53687091200,
  66     'x-account-policy-versioning': 'auto'}
  67 container_info = {
  68     'content-language': 'en-us',
  69     'content-type': 'text/html; charset=utf-8',
  70     'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
  71     'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
  72     'server': 'gunicorn/0.14.5',
  73     'vary': 'Accept-Language',
  74     'x-container-block-hash': 'sha256',
  75     'x-container-block-size': 4194304,
  76     'x-container-bytes-used': 309528938,
  77     'x-container-object-count': 14,
  78     'x-container-object-meta': '',
  79     'x-container-policy-quota': 53687091200,
  80     'x-container-policy-versioning': 'auto'}
  81 object_info = {
  82     'content-language': 'en-us',
  83     'content-length': 254965,
  84     'content-type': 'application/octet-stream',
  85     'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
  86     'etag': '',
  87     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  88     'server': 'gunicorn/0.14.5',
  89     'vary': 'Accept-Language',
  90     'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
  91     'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
  92     'x-object-version': '525996',
  93     'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
  94     'x-object-meta-k1': 'v1',
  95     'x-object-meta-k2': 'v2'}
  96 container_list = [
  97     dict(
  98         count=2,
  99         last_modified="2013-02-27T11:56:09.893033+00:00",
 100         bytes=677076979,
 101         name="pithos",
 102         x_container_policy=dict(quota="21474836480", versioning="auto")),
 103     dict(
 104         count=0,
 105         last_modified="2012-10-23T12:25:17.229187+00:00",
 106         bytes=0,
 107         name="trash",
 108         x_container_policy=dict(quota="21474836480", versioning="auto"))]
 109 object_list = [
 110     dict(hash="",
 111         name="The_Secret_Garden.zip",
 112         x_object_public="/public/wdp9p",
 113         bytes=203304947,
 114         x_object_version_timestamp="1360237915.7027509",
 115         x_object_uuid="s0m3uu1df0r0bj0n3",
 116         last_modified="2013-02-07T11:51:55.702751+00:00",
 117         content_type="application/octet-stream",
 118         x_object_hash="0afdf29f71cd53126225c3f54ca",
 119         x_object_version=17737,
 120         x_object_modified_by=user_id),
 121     dict(hash="",
 122         name="The_Revealed_Garden.zip",
 123         x_object_public="/public/wpd7p",
 124         bytes=20330947,
 125         x_object_version_timestamp="13602915.7027509",
 126         x_object_uuid="s0m3uu1df0r0bj70w",
 127         last_modified="2013-02-07T11:51:55.702751+00:00",
 128         content_type="application/octet-stream",
 129         x_object_hash="0afdf29f71cd53126225c3f54ca",
 130         x_object_version=17737,
 131         x_object_modified_by=user_id)]
 132 object_hashmap = dict(
 133     block_hash="sha256", block_size=4194304, bytes=33554432,
 134     hashes=[
 135         "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
 136         "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
 137         "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
 138         "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
 139         "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
 140         "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
 141         "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
 142         "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
 143 sharers = [
 144     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
 145     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
 146     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
 147
 148
 149 class FR(object):
 150     """FR stands for Fake Response"""
 151     json = dict()
 152     headers = dict()
 153     content = json
 154     status = None
 155     status_code = 200
 156
 157     def release(self):
 158         pass
 159
 160
 161 class PithosRest(TestCase):
 162
 163     def setUp(self):
 164         self.url = 'https://www.example.com/pithos'
 165         self.token = 'p17h0570k3n'
 166         self.client = PithosRestClient(self.url, self.token)
 167         self.client.account = user_id
 168         self.client.container = 'c0nt@1n3r_i'
 169
 170     def tearDown(self):
 171         FR.headers = dict()
 172         FR.json = dict()
 173         FR.content = FR.json
 174
 175     @patch('%s.set_param' % rest_pkg)
 176     @patch('%s.set_header' % rest_pkg)
 177     @patch('%s.head' % rest_pkg, return_value=FR())
 178     def test_account_head(self, head, SH, SP):
 179         for params in product(
 180                 (None, '50m3-d473'),
 181                 (None, '50m3-07h3r-d473'),
 182                 (None, 'y37-4n7h3r-d473'),
 183                 ((), ('someval',), ('v1', 'v2',)),
 184                 (dict(), dict(success=200), dict(k='v', v='k'))):
 185             args, kwargs = params[-2], params[-1]
 186             params = params[:-2]
 187             self.client.account_head(*(params + args), **kwargs)
 188             unt = params[0]
 189             self.assertEqual(SP.mock_calls[-1], call('until', unt, iff=unt))
 190             IMS, IUS = params[1], params[2]
 191             self.assertEqual(SH.mock_calls[-2:], [
 192                 call('If-Modified-Since', IMS),
 193                 call('If-Unmodified-Since', IUS)])
 194             self.assertEqual(head.mock_calls[-1], call(
 195                 '/%s' % self.client.account,
 196                 *args,
 197                 success=kwargs.pop('success', 204),
 198                 **kwargs))
 199
 200     @patch('%s.set_param' % rest_pkg)
 201     @patch('%s.set_header' % rest_pkg)
 202     @patch('%s.get' % rest_pkg, return_value=FR())
 203     def test_account_get(self, get, SH, SP):
 204         keys = ('limit', 'marker', 'format', 'shared', 'until')
 205         for params in product(
 206                 (None, 42),
 207                 (None, 'X'),
 208                 ('json', 'xml'),
 209                 (False, True),
 210                 (None, '50m3-d473'),
 211                 (None, '50m3-07h3r-d473'),
 212                 (None, 'y37-4n7h3r-d473'),
 213                 ((), ('someval',), ('v1', 'v2',)),
 214                 (dict(), dict(success=200), dict(k='v', v='k'))):
 215             args, kwargs = params[-2], params[-1]
 216             params = params[:-2]
 217             self.client.account_get(*(params + args), **kwargs)
 218             self.assertEqual(SP.mock_calls[-5:],
 219                 [call(keys[i], iff=X) if (
 220                     i == 3) else call(
 221                         keys[i], X, iff=X) for i, X in enumerate(params[:5])])
 222             IMS, IUS = params[5], params[6]
 223             self.assertEqual(SH.mock_calls[-2:], [
 224                 call('If-Modified-Since', IMS),
 225                 call('If-Unmodified-Since', IUS)])
 226             self.assertEqual(get.mock_calls[-1], call(
 227                 '/%s' % self.client.account,
 228                 *args,
 229                 success=kwargs.pop('success', (200, 204)),
 230                 **kwargs))
 231
 232     @patch('%s.set_param' % rest_pkg)
 233     @patch('%s.set_header' % rest_pkg)
 234     @patch('%s.post' % rest_pkg, return_value=FR())
 235     def test_account_post(self, post, SH, SP):
 236         #keys = ('update', 'groups', 'metadata', 'quota', 'versioning')
 237         for pm in product(
 238                 (True, False),
 239                 ({}, dict(g=['u1', 'u2']), dict(g1=[], g2=['u1', 'u2'])),
 240                 (None, dict(k1='v1', k2='v2', k3='v2'), dict(k='v')),
 241                 (None, 42),
 242                 (None, 'v3r510n1ng'),
 243                 ((), ('someval',), ('v1', 'v2',)),
 244                 (dict(), dict(success=200), dict(k='v', v='k'))):
 245             args, kwargs = pm[-2:]
 246             pm = pm[:-2]
 247             self.client.account_post(*(pm + args), **kwargs)
 248             upd = pm[0]
 249             self.assertEqual(SP.mock_calls[-1], call('update', iff=upd))
 250             expected = []
 251             if pm[1]:
 252                 expected += [
 253                 call('X-Account-Group-%s' % k, v) for k, v in pm[1].items()]
 254             if pm[2]:
 255                 expected = [
 256                 call('X-Account-Meta-%s' % k, v) for k, v in pm[2].items()]
 257             expected = [
 258                 call('X-Account-Policy-Quota', pm[3]),
 259                 call('X-Account-Policy-Versioning', pm[4])]
 260             self.assertEqual(SH.mock_calls[- len(expected):], expected)
 261             self.assertEqual(post.mock_calls[-1], call(
 262                 '/%s' % self.client.account,
 263                 *args,
 264                 success=kwargs.pop('success', 202),
 265                 **kwargs))
 266
 267     @patch('%s.set_param' % rest_pkg)
 268     @patch('%s.set_header' % rest_pkg)
 269     @patch('%s.head' % rest_pkg, return_value=FR())
 270     def test_container_head(self, head, SH, SP):
 271         for pm in product(
 272                 (None, '4-d473'),
 273                 (None, '47h3r-d473'),
 274                 (None, 'y37-4n47h3r'),
 275                 ((), ('someval',)),
 276                 (dict(), dict(success=200), dict(k='v', v='k'))):
 277             args, kwargs = pm[-2:]
 278             pm = pm[:-2]
 279             self.client.container_head(*(pm + args), **kwargs)
 280             unt, ims, ius = pm[0:3]
 281             self.assertEqual(SP.mock_calls[-1], call('until', unt, iff=unt))
 282             self.assertEqual(SH.mock_calls[-2:], [
 283                 call('If-Modified-Since', ims),
 284                 call('If-Unmodified-Since', ius)])
 285             self.assertEqual(head.mock_calls[-1], call(
 286                 '/%s/%s' % (self.client.account, self.client.container),
 287                 *args,
 288                 success=kwargs.pop('success', 204),
 289                 **kwargs))
 290
 291     @patch('%s.set_param' % rest_pkg)
 292     @patch('%s.set_header' % rest_pkg)
 293     @patch('%s.get' % rest_pkg, return_value=FR())
 294     def test_container_get(self, get, SH, SP):
 295         for pm in product(
 296                 (None, 42),
 297                 (None, 'X'),
 298                 (None, 'some/prefix'),
 299                 (None, 'delimiter'),
 300                 (None, '/some/path'),
 301                 ('json', 'some-format'),
 302                 ([], ['k1', 'k2', 'k3']),
 303                 (False, True),
 304                 (None, 'unt1l-d473'),
 305                 (None, 'y37-4n47h3r'),
 306                 (None, '4n47h3r-d473'),
 307                 ((), ('someval',)),
 308                 (dict(), dict(success=400), dict(k='v', v='k'))):
 309             args, kwargs = pm[-2:]
 310             pm = pm[:-2]
 311             self.client.container_get(*(pm + args), **kwargs)
 312             lmt, mrk, prfx, dlm, path, frmt, meta, shr, unt = pm[:-2]
 313             exp = [call('limit', lmt, iff=lmt), call('marker', mrk, iff=mrk)]
 314             exp += [call('path', path)] if path else [
 315                 call('prefix', prfx, iff=prfx),
 316                 call('delimiter', dlm, iff=dlm)]
 317             exp += [call('format', frmt, iff=frmt), call('shared', iff=shr)]
 318             if meta:
 319                 exp += [call('meta', ','.join(meta))]
 320             exp += [call('until', unt, iff=unt)]
 321             self.assertEqual(SP.mock_calls[- len(exp):], exp)
 322             ims, ius = pm[-2:]
 323             self.assertEqual(SH.mock_calls[-2:], [
 324                 call('If-Modified-Since', ims),
 325                 call('If-Unmodified-Since', ius)])
 326             self.assertEqual(get.mock_calls[-1], call(
 327                 '/%s/%s' % (self.client.account, self.client.container),
 328                 *args,
 329                 success=kwargs.pop('success', 200),
 330                 **kwargs))
 331
 332     @patch('%s.set_header' % rest_pkg)
 333     @patch('%s.put' % rest_pkg, return_value=FR())
 334     def test_container_put(self, put, SH):
 335         for pm in product(
 336                 (None, 42),
 337                 (None, 'v3r51on1ng'),
 338                 (dict(), dict(k1='v2'), dict(k2='v2', k3='v3')),
 339                 ((), ('someval',)),
 340                 (dict(), dict(success=400), dict(k='v', v='k'))):
 341             args, kwargs = pm[-2:]
 342             pm = pm[:-2]
 343             self.client.container_put(*(pm + args), **kwargs)
 344             quota, versioning, metas = pm[-3:]
 345             exp = [
 346                 call('X-Container-Policy-Quota', quota),
 347                 call('X-Container-Policy-Versioning', versioning)] + [
 348                 call('X-Container-Meta-%s' % k, v) for k, v in metas.items()]
 349             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 350             self.assertEqual(put.mock_calls[-1], call(
 351                 '/%s/%s' % (self.client.account, self.client.container),
 352                 *args,
 353                 success=kwargs.pop('success', (201, 202)),
 354                 **kwargs))
 355
 356     @patch('%s.set_param' % rest_pkg)
 357     @patch('%s.set_header' % rest_pkg)
 358     @patch('%s.post' % rest_pkg, return_value=FR())
 359     def test_container_post(self, post, SH, SP):
 360         for pm in product(
 361                 (True, False),
 362                 ('json', 'some-format'),
 363                 (None, 'quota'),
 364                 (None, 'v3r51on1ng'),
 365                 (dict(), dict(k1='v2'), dict(k2='v2', k3='v3')),
 366                 (None, 'content-type'),
 367                 (None, 42),
 368                 (None, 'transfer-encoding'),
 369                 ((), ('someval',)),
 370                 (dict(), dict(success=400), dict(k='v', v='k'))):
 371             args, kwargs = pm[-2:]
 372             pm = pm[:-2]
 373             self.client.container_post(*(pm + args), **kwargs)
 374             upd, frmt = pm[:2]
 375             self.assertEqual(SP.mock_calls[-2:], [
 376                 call('update', iff=upd),
 377                 call('format', frmt, iff=frmt)])
 378             qta, vrs, metas, ctype, clen, trenc = pm[2:]
 379             prfx = 'X-Container-Meta-'
 380             exp = [
 381                 call('X-Container-Policy-Quota', qta),
 382                 call('X-Container-Policy-Versioning', vrs)] + [
 383                 call('%s%s' % (prfx, k), v) for k, v in metas.items()] + [
 384                 call('Content-Type', ctype),
 385                 call('Content-Length', clen),
 386                 call('Transfer-Encoding', trenc)]
 387             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 388             ims, ius = pm[-2:]
 389             self.assertEqual(post.mock_calls[-1], call(
 390                 '/%s/%s' % (self.client.account, self.client.container),
 391                 *args,
 392                 success=kwargs.pop('success', 202),
 393                 **kwargs))
 394
 395     @patch('%s.set_param' % rest_pkg)
 396     @patch('%s.delete' % rest_pkg, return_value=FR())
 397     def test_container_delete(self, delete, SP):
 398         for pm in product(
 399                 (None, 'd473'),
 400                 (None, 'd3l1m'),
 401                 ((), ('someval',)),
 402                 (dict(), dict(success=400), dict(k='v', v='k'))):
 403             args, kwargs = pm[-2:]
 404             pm = pm[:-2]
 405             self.client.container_delete(*(pm + args), **kwargs)
 406             unt, dlm = pm[-2:]
 407             self.assertEqual(SP.mock_calls[-2:], [
 408                 call('until', unt, iff=unt),
 409                 call('delimiter', dlm, iff=dlm)])
 410             self.assertEqual(delete.mock_calls[-1], call(
 411                 '/%s/%s' % (self.client.account, self.client.container),
 412                 *args,
 413                 success=kwargs.pop('success', 204),
 414                 **kwargs))
 415
 416     @patch('%s.set_param' % rest_pkg)
 417     @patch('%s.set_header' % rest_pkg)
 418     @patch('%s.head' % rest_pkg, return_value=FR())
 419     def test_object_head(self, head, SH, SP):
 420         for pm in product(
 421                 (None, 'v3r510n'),
 422                 (None, '1f-374g'),
 423                 (None, '1f-n0-74g'),
 424                 (None, '1f-m0d-51nc3'),
 425                 (None, '1f-unm0d-51nc3'),
 426                 ((), ('someval',)),
 427                 (dict(), dict(success=400), dict(k='v', v='k'))):
 428             args, kwargs = pm[-2:]
 429             pm = pm[:-2]
 430             self.client.object_head(obj, *(pm + args), **kwargs)
 431             vrs, etag, netag, ims, ius = pm[:5]
 432             self.assertEqual(
 433                 SP.mock_calls[-1],
 434                 call('version', vrs, iff=vrs))
 435             self.assertEqual(SH.mock_calls[-4:], [
 436                 call('If-Match', etag),
 437                 call('If-None-Match', netag),
 438                 call('If-Modified-Since', ims),
 439                 call('If-Unmodified-Since', ius)])
 440             acc, cont = self.client.account, self.client.container
 441             self.assertEqual(head.mock_calls[-1], call(
 442                 '/%s/%s/%s' % (acc, cont, obj),
 443                 *args,
 444                 success=kwargs.pop('success', 200),
 445                 **kwargs))
 446
 447     @patch('%s.set_param' % rest_pkg)
 448     @patch('%s.set_header' % rest_pkg)
 449     @patch('%s.get' % rest_pkg, return_value=FR())
 450     def test_object_get(self, get, SH, SP):
 451         for pm in product(
 452                 ('json', 'f0rm47'),
 453                 (False, True),
 454                 (None, 'v3r510n'),
 455                 (None, 'range=74-63'),
 456                 (False, True),
 457                 (None, '3746'),
 458                 (None, 'non-3746'),
 459                 (None, '1f-m0d'),
 460                 (None, '1f-unm0d'),
 461                 ((), ('someval',)),
 462                 (dict(), dict(success=400), dict(k='v', v='k'))):
 463             args, kwargs = pm[-2:]
 464             pm = pm[:-2]
 465             self.client.object_get(obj, *(pm + args), **kwargs)
 466             format, hashmap, version = pm[:3]
 467             self.assertEqual(SP.mock_calls[-3:], [
 468                 call('format', format, iff=format),
 469                 call('hashmap', hashmap, iff=hashmap),
 470                 call('version', version, iff=version)])
 471             rng, ifrng, im, inm, ims, ius = pm[-6:]
 472             self.assertEqual(SH.mock_calls[-6:], [
 473                 call('Range', rng),
 474                 call('If-Range', '', ifrng and rng),
 475                 call('If-Match', im),
 476                 call('If-None-Match', inm),
 477                 call('If-Modified-Since', ims),
 478                 call('If-Unmodified-Since', ius)])
 479             acc, cont = self.client.account, self.client.container
 480             self.assertEqual(get.mock_calls[-1], call(
 481                 '/%s/%s/%s' % (acc, cont, obj),
 482                 *args,
 483                 success=kwargs.pop('success', 200),
 484                 **kwargs))
 485
 486     @patch('%s.set_param' % rest_pkg)
 487     @patch('%s.set_header' % rest_pkg)
 488     @patch('%s.put' % rest_pkg, return_value=FR())
 489     def test_object_put(self, put, SH, SP):
 490         for pm in product(
 491                 ('json', 'f0rm47'),
 492                 (False, True),
 493                 (None, 'delim',),
 494                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 495                 (False, True),
 496                 (dict(), dict(k2='v2', k3='v3')),
 497                 ((), ('someval',)),
 498                 (dict(), dict(success=400), dict(k='v', v='k'))):
 499             args, kwargs = pm[-2:]
 500             pm = pm[:-2]
 501             terms = [None] * 13
 502             for i in range(len(terms)):
 503                 if randint(0, 2):
 504                     terms[i] = 'val_%s' % randint(13, 1024)
 505             self.client.object_put(
 506                 obj,
 507                 *(pm[:3] + tuple(terms) + pm[3:] + args),
 508                 **kwargs)
 509             format, hashmap, delimiter = pm[:3]
 510             self.assertEqual(SP.mock_calls[-3:], [
 511                 call('format', format, iff=format),
 512                 call('hashmap', hashmap, iff=hashmap),
 513                 call('delimiter', delimiter, iff=delimiter)])
 514             (
 515                 im, inm, etag, clen, ctype, trenc,
 516                 cp, mv, srcacc, srcvrs, conenc, condis, mnf) = terms
 517             perms, public, metas = pm[3:]
 518             exp = [
 519                 call('If-Match', im),
 520                 call('If-None-Match', inm),
 521                 call('ETag', etag),
 522                 call('Content-Length', clen),
 523                 call('Content-Type', ctype),
 524                 call('Transfer-Encoding', trenc),
 525                 call('X-Copy-From', cp),
 526                 call('X-Move-From', mv),
 527                 call('X-Source-Account', srcacc),
 528                 call('X-Source-Version', srcvrs),
 529                 call('Content-Encoding', conenc),
 530                 call('Content-Disposition', condis),
 531                 call('X-Object-Manifest', mnf)]
 532             if perms:
 533                 perm_str = ''
 534                 for ptype, pval in perms.items():
 535                     if pval:
 536                         perm_str += ';' if perm_str else ''
 537                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 538                 exp += [call('X-Object-Sharing', perm_str)]
 539             exp += [call('X-Object-Public', public)]
 540             for k, v in metas.items():
 541                 exp += [call('X-Object-Meta-%s' % k, v)]
 542             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 543             acc, cont = self.client.account, self.client.container
 544             self.assertEqual(put.mock_calls[-1], call(
 545                 '/%s/%s/%s' % (acc, cont, obj),
 546                 *args,
 547                 success=kwargs.pop('success', 201),
 548                 **kwargs))
 549
 550     @patch('%s.set_param' % rest_pkg)
 551     @patch('%s.set_header' % rest_pkg)
 552     @patch('%s.copy' % rest_pkg, return_value=FR())
 553     def test_object_copy(self, copy, SH, SP):
 554         dest = 'dest1n4710n'
 555         for pm in product(
 556                 ('json', 'f0rm47'),
 557                 (False, True),
 558                 (None, 'ifmatch'),
 559                 (None, 'ifnonematch'),
 560                 (None, 'destinationaccount'),
 561                 (None, 'content-type'),
 562                 (None, 'content-encoding'),
 563                 (None, 'content-disp'),
 564                 (None, 'source-version'),
 565                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 566                 (False, True),
 567                 (dict(), dict(k2='v2', k3='v3')),
 568                 ((), ('someval',)),
 569                 (dict(), dict(success=400), dict(k='v', v='k'))):
 570             args, kwargs = pm[-2:]
 571             pm = pm[:-2]
 572             self.client.object_copy(obj, dest, *(pm + args), **kwargs)
 573             format, ict = pm[:2]
 574             self.assertEqual(SP.mock_calls[-2:], [
 575                 call('format', format, iff=format),
 576                 call('ignore_content_type', iff=ict)])
 577             im, inm, da, ct, ce, cd, sv, perms, public, metas = pm[2:]
 578             exp = [call('If-Match', im),
 579                 call('If-None-Match', inm),
 580                 call('Destination', dest),
 581                 call('Destination-Account', da),
 582                 call('Content-Type', ct),
 583                 call('Content-Encoding', ce),
 584                 call('Content-Disposition', cd),
 585                 call('X-Source-Version', sv)]
 586             if perms:
 587                 perm_str = ''
 588                 for ptype, pval in perms.items():
 589                     if pval:
 590                         perm_str += ';' if perm_str else ''
 591                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 592                 exp += [call('X-Object-Sharing', perm_str)]
 593             exp += [call('X-Object-Public', public)]
 594             for k, v in metas.items():
 595                 exp += [call('X-Object-Meta-%s' % k, v)]
 596             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 597             acc, cont = self.client.account, self.client.container
 598             self.assertEqual(copy.mock_calls[-1], call(
 599                 '/%s/%s/%s' % (acc, cont, obj),
 600                 *args,
 601                 success=kwargs.pop('success', 201),
 602                 **kwargs))
 603
 604     @patch('%s.set_param' % rest_pkg)
 605     @patch('%s.set_header' % rest_pkg)
 606     @patch('%s.move' % rest_pkg, return_value=FR())
 607     def test_object_move(self, move, SH, SP):
 608         for pm in product(
 609                 ('json', 'f0rm47'),
 610                 (False, True),
 611                 (None, 'ifmatch'),
 612                 (None, 'ifnonematch'),
 613                 (None, 'destination'),
 614                 (None, 'destinationaccount'),
 615                 (None, 'content-type'),
 616                 (None, 'content-encoding'),
 617                 (None, 'content-disp'),
 618                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 619                 (False, True),
 620                 (dict(), dict(k2='v2', k3='v3')),
 621                 ((), ('someval',)),
 622                 (dict(), dict(success=400), dict(k='v', v='k'))):
 623             args, kwargs = pm[-2:]
 624             pm = pm[:-2]
 625             self.client.object_move(obj, *(pm + args), **kwargs)
 626             format, ict = pm[:2]
 627             self.assertEqual(SP.mock_calls[-2:], [
 628                 call('format', format, iff=format),
 629                 call('ignore_content_type', iff=ict)])
 630             im, inm, d, da, ct, ce, cd, perms, public, metas = pm[2:]
 631             exp = [call('If-Match', im),
 632                 call('If-None-Match', inm),
 633                 call('Destination', d),
 634                 call('Destination-Account', da),
 635                 call('Content-Type', ct),
 636                 call('Content-Encoding', ce),
 637                 call('Content-Disposition', cd)]
 638             if perms:
 639                 perm_str = ''
 640                 for ptype, pval in perms.items():
 641                     if pval:
 642                         perm_str += ';' if perm_str else ''
 643                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 644                 exp += [call('X-Object-Sharing', perm_str)]
 645             exp += [call('X-Object-Public', public)]
 646             for k, v in metas.items():
 647                 exp += [call('X-Object-Meta-%s' % k, v)]
 648             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 649             acc, cont = self.client.account, self.client.container
 650             self.assertEqual(move.mock_calls[-1], call(
 651                 '/%s/%s/%s' % (acc, cont, obj),
 652                 *args,
 653                 success=kwargs.pop('success', 201),
 654                 **kwargs))
 655
 656     @patch('%s.set_param' % rest_pkg)
 657     @patch('%s.set_header' % rest_pkg)
 658     @patch('%s.post' % rest_pkg, return_value=FR())
 659     def test_object_post(self, post, SH, SP):
 660         for pm in product(
 661                 ('json', 'f0rm47'),
 662                 (False, True),
 663                 (dict(), dict(read=['u1', 'g2'], write=['u1'])),
 664                 (False, True),
 665                 (dict(), dict(k2='v2', k3='v3')),
 666                 ((), ('someval',)),
 667                 (dict(), dict(success=400), dict(k='v', v='k'))):
 668             args, kwargs = pm[-2:]
 669             pm = pm[:-2]
 670             terms = [None] * 13
 671             for i in range(len(terms)):
 672                 if randint(0, 2):
 673                     terms[i] = 'val_%s' % randint(13, 1024)
 674             self.client.object_post(
 675                 obj,
 676                 *(pm[:2] + tuple(terms) + pm[2:] + args),
 677                 **kwargs)
 678             format, update = pm[:2]
 679             self.assertEqual(SP.mock_calls[-2:], [
 680                 call('format', format, iff=format),
 681                 call('update', iff=update)])
 682             (
 683                 im, inm, clen, ctype, crng, trenc, cenc,
 684                 condis, srcobj, srcacc, srcvrs, obytes, mnfs) = terms
 685             exp = [
 686                 call('If-Match', im),
 687                 call('If-None-Match', inm),
 688                 call('Content-Length', clen, iff=not trenc),
 689                 call('Content-Type', ctype),
 690                 call('Content-Range', crng),
 691                 call('Transfer-Encoding', trenc),
 692                 call('Content-Encoding', cenc),
 693                 call('Content-Disposition', condis),
 694                 call('X-Source-Object', srcobj),
 695                 call('X-Source-Account', srcacc),
 696                 call('X-Source-Version', srcvrs),
 697                 call('X-Object-Bytes', obytes),
 698                 call('X-Object-Manifest', mnfs)]
 699             perms, public, metas = pm[2:]
 700             if perms:
 701                 perm_str = ''
 702                 for ptype, pval in perms.items():
 703                     if pval:
 704                         perm_str += ';' if perm_str else ''
 705                         perm_str += '%s=%s' % (ptype, ','.join(pval))
 706                 exp += [call('X-Object-Sharing', perm_str)]
 707             exp += [call('X-Object-Public', public)]
 708             for k, v in metas.items():
 709                 exp += [call('X-Object-Meta-%s' % k, v)]
 710             self.assertEqual(SH.mock_calls[- len(exp):], exp)
 711             acc, cont = self.client.account, self.client.container
 712             self.assertEqual(post.mock_calls[-1], call(
 713                 '/%s/%s/%s' % (acc, cont, obj),
 714                 *args,
 715                 success=kwargs.pop('success', (202, 204)),
 716                 **kwargs))
 717
 718     @patch('%s.set_param' % rest_pkg)
 719     @patch('%s.delete' % rest_pkg, return_value=FR())
 720     def test_object_delete(self, delete, SP):
 721         for pm in product(
 722                 (None, 'until'),
 723                 (None, 'delim'),
 724                 ((), ('someval',)),
 725                 (dict(), dict(success=400), dict(k='v', v='k'))):
 726             args, kwargs = pm[-2:]
 727             pm = pm[:-2]
 728             self.client.object_delete(
 729                 obj,
 730                 *(pm + args),
 731                 **kwargs)
 732             until, dlm = pm[-2:]
 733             self.assertEqual(SP.mock_calls[-2:], [
 734                 call('until', until, iff=until),
 735                 call('delimiter', dlm, iff=dlm)])
 736             acc, cont = self.client.account, self.client.container
 737             self.assertEqual(delete.mock_calls[-1], call(
 738                 '/%s/%s/%s' % (acc, cont, obj),
 739                 *args,
 740                 success=kwargs.pop('success', 204),
 741                 **kwargs))
 742
 743
 744 class Pithos(TestCase):
 745
 746     files = []
 747
 748     def _create_temp_file(self, num_of_blocks):
 749         self.files.append(NamedTemporaryFile())
 750         tmpFile = self.files[-1]
 751         file_size = num_of_blocks * 4 * 1024 * 1024
 752         print('\n\tCreate tmp file')
 753         tmpFile.write(urandom(file_size))
 754         tmpFile.flush()
 755         tmpFile.seek(0)
 756         print('\t\tDone')
 757         return tmpFile
 758
 759     def assert_dicts_are_equal(self, d1, d2):
 760         for k, v in d1.items():
 761             self.assertTrue(k in d2)
 762             if isinstance(v, dict):
 763                 self.assert_dicts_are_equal(v, d2[k])
 764             else:
 765                 self.assertEqual(unicode(v), unicode(d2[k]))
 766
 767     def setUp(self):
 768         self.url = 'https://www.example.com/pithos'
 769         self.token = 'p17h0570k3n'
 770         self.client = PithosClient(self.url, self.token)
 771         self.client.account = user_id
 772         self.client.container = 'c0nt@1n3r_i'
 773
 774     def tearDown(self):
 775         FR.headers = dict()
 776         FR.status_code = 200
 777         FR.json = dict()
 778         FR.content = FR.json
 779         for f in self.files:
 780             f.close()
 781
 782     #  Pithos+ methods that extend storage API
 783
 784     @patch('%s.account_head' % pithos_pkg, return_value=FR())
 785     def test_get_account_info(self, AH):
 786         FR.headers = account_info
 787         for until in (None, 'un71L-d473'):
 788             r = self.client.get_account_info(until=until)
 789             self.assert_dicts_are_equal(r, account_info)
 790             self.assertEqual(AH.mock_calls[-1], call(until=until))
 791         FR.status_code = 401
 792         self.assertRaises(ClientError, self.client.get_account_info)
 793
 794     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 795     def test_del_account_meta(self, AP):
 796         keys = ['k1', 'k2', 'k3']
 797         for key in keys:
 798             self.client.del_account_meta(key)
 799             self.assertEqual(
 800                 AP.mock_calls[-1],
 801                 call(update=True, metadata={key: ''}))
 802
 803     @patch('%s.container_head' % pithos_pkg, return_value=FR())
 804     def test_get_container_info(self, CH):
 805         FR.headers = container_info
 806         r = self.client.get_container_info()
 807         self.assert_dicts_are_equal(r, container_info)
 808         u = 'some date'
 809         r = self.client.get_container_info(until=u)
 810         self.assertEqual(CH.mock_calls, [call(until=None), call(until=u)])
 811
 812     @patch('%s.account_get' % pithos_pkg, return_value=FR())
 813     def test_list_containers(self, get):
 814         FR.json = container_list
 815         r = self.client.list_containers()
 816         get.assert_called_once_with()
 817         for i in range(len(r)):
 818             self.assert_dicts_are_equal(r[i], container_list[i])
 819
 820     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 821     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 822     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 823     def test_upload_object(self, OP, CP, GCI):
 824         num_of_blocks = 8
 825         tmpFile = self._create_temp_file(num_of_blocks)
 826
 827         # Without kwargs
 828         self.client.upload_object(obj, tmpFile)
 829         self.assertEqual(GCI.mock_calls[-1], call())
 830         [call1, call2] = OP.mock_calls
 831
 832         (args1, kwargs1) = call1[1:3]
 833         (args2, kwargs2) = call2[1:3]
 834         self.assertEqual(args1, (obj,))
 835         expected1 = dict(
 836             hashmap=True,
 837             success=(201, 409),
 838             format='json',
 839             json=dict(
 840                 hashes=['s0m3h@5h'] * num_of_blocks,
 841                 bytes=num_of_blocks * 4 * 1024 * 1024),
 842             etag=None,
 843             content_encoding=None,
 844             content_type='application/octet-stream',
 845             content_disposition=None,
 846             public=None,
 847             permissions=None)
 848         for k, v in expected1.items():
 849             if k == 'json':
 850                 self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
 851                 self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
 852             else:
 853                 self.assertEqual(v, kwargs1[k])
 854
 855         (args2, kwargs2) = call2[1:3]
 856         self.assertEqual(args2, (obj,))
 857         expected2 = dict(
 858             json=dict(
 859                 hashes=['s0m3h@5h'] * num_of_blocks,
 860                 bytes=num_of_blocks * 4 * 1024 * 1024),
 861             content_type='application/octet-stream',
 862             hashmap=True,
 863             success=201,
 864             format='json')
 865         for k, v in expected2.items():
 866             if k == 'json':
 867                 self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
 868                 self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
 869             else:
 870                 self.assertEqual(v, kwargs2[k])
 871
 872         mock_offset = 2
 873
 874         #  With progress bars
 875         try:
 876             from progress.bar import ShadyBar
 877             blck_bar = ShadyBar('Mock blck calc.')
 878             upld_bar = ShadyBar('Mock uplds')
 879         except ImportError:
 880             blck_bar = None
 881             upld_bar = None
 882
 883         if blck_bar and upld_bar:
 884
 885             def blck_gen(n):
 886                 for i in blck_bar.iter(range(n)):
 887                     yield
 888                 yield
 889
 890             def upld_gen(n):
 891                 for i in upld_bar.iter(range(n)):
 892                     yield
 893                 yield
 894
 895             tmpFile.seek(0)
 896             self.client.upload_object(
 897                 obj, tmpFile,
 898                 hash_cb=blck_gen, upload_cb=upld_gen)
 899
 900             for i, c in enumerate(OP.mock_calls[-mock_offset:]):
 901                 self.assertEqual(OP.mock_calls[i], c)
 902
 903         #  With content-type
 904         tmpFile.seek(0)
 905         ctype = 'video/mpeg'
 906         sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
 907         self.client.upload_object(obj, tmpFile,
 908             content_type=ctype, sharing=sharing)
 909         self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
 910         self.assert_dicts_are_equal(
 911             OP.mock_calls[-2][2]['permissions'],
 912             sharing)
 913
 914         # With other args
 915         tmpFile.seek(0)
 916         kwargs = dict(
 917             etag='s0m3E74g',
 918             content_type=ctype,
 919             content_disposition=ctype + 'd15p051710n',
 920             public=True,
 921             content_encoding='802.11')
 922         self.client.upload_object(obj, tmpFile, **kwargs)
 923         for arg, val in kwargs.items():
 924             self.assertEqual(OP.mock_calls[-2][2][arg], val)
 925
 926     def test_get_object_info(self):
 927         FR.headers = object_info
 928         version = 'v3r510n'
 929         with patch.object(
 930                 PithosClient, 'object_head',
 931                 return_value=FR()) as head:
 932             r = self.client.get_object_info(obj)
 933             self.assertEqual(r, object_info)
 934             r = self.client.get_object_info(obj, version=version)
 935             self.assertEqual(head.mock_calls, [
 936                 call(obj, version=None),
 937                 call(obj, version=version)])
 938         with patch.object(
 939                 PithosClient, 'object_head',
 940                 side_effect=ClientError('Obj not found', 404)):
 941             self.assertRaises(
 942                 ClientError,
 943                 self.client.get_object_info,
 944                 obj, version=version)
 945
 946     @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
 947     def test_get_object_meta(self, GOI):
 948         for version in (None, 'v3r510n'):
 949             r = self.client.get_object_meta(obj, version)
 950             for k in [k for k in object_info if k.startswith('x-object-meta')]:
 951                 self.assertEqual(r.pop(k), object_info[k])
 952             self.assertFalse(len(r))
 953             self.assertEqual(GOI.mock_calls[-1], call(obj, version=version))
 954
 955     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 956     def test_del_object_meta(self, post):
 957         metakey = '50m3m3t4k3y'
 958         self.client.del_object_meta(obj, metakey)
 959         post.assert_called_once_with(obj, update=True, metadata={metakey: ''})
 960
 961     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 962     def test_copy_object(self, put):
 963         src_cont = 'src-c0nt41n3r'
 964         src_obj = 'src-0bj'
 965         dst_cont = 'dst-c0nt41n3r'
 966         dst_obj = 'dst-0bj'
 967         expected = call(
 968             src_obj,
 969             content_length=0,
 970             source_account=None,
 971             success=201,
 972             copy_from='/%s/%s' % (src_cont, src_obj),
 973             delimiter=None,
 974             content_type=None,
 975             source_version=None,
 976             public=False)
 977         self.client.copy_object(src_cont, src_obj, dst_cont)
 978         self.assertEqual(put.mock_calls[-1], expected)
 979         self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
 980         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 981         kwargs = dict(
 982             source_version='src-v3r510n',
 983             source_account='src-4cc0un7',
 984             public=True,
 985             content_type='c0n73n7Typ3',
 986             delimiter='5')
 987         self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
 988         for k, v in kwargs.items():
 989             self.assertEqual(v, put.mock_calls[-1][2][k])
 990
 991     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 992     def test_move_object(self, put):
 993         src_cont = 'src-c0nt41n3r'
 994         src_obj = 'src-0bj'
 995         dst_cont = 'dst-c0nt41n3r'
 996         dst_obj = 'dst-0bj'
 997         expected = call(
 998             src_obj,
 999             content_length=0,
1000             source_account=None,
1001             success=201,
1002             move_from='/%s/%s' % (src_cont, src_obj),
1003             delimiter=None,
1004             content_type=None,
1005             source_version=None,
1006             public=False)
1007         self.client.move_object(src_cont, src_obj, dst_cont)
1008         self.assertEqual(put.mock_calls[-1], expected)
1009         self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
1010         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
1011         kwargs = dict(
1012             source_version='src-v3r510n',
1013             source_account='src-4cc0un7',
1014             public=True,
1015             content_type='c0n73n7Typ3',
1016             delimiter='5')
1017         self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
1018         for k, v in kwargs.items():
1019             self.assertEqual(v, put.mock_calls[-1][2][k])
1020
1021     #  Pithos+ only methods
1022
1023     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
1024     def test_purge_container(self, CD):
1025         self.client.purge_container()
1026         self.assertTrue('until' in CD.mock_calls[-1][2])
1027         cont = self.client.container
1028         self.client.purge_container('another-container')
1029         self.assertEqual(self.client.container, cont)
1030
1031     @patch('%s.object_put' % pithos_pkg, return_value=FR())
1032     def test_upload_object_unchunked(self, put):
1033         num_of_blocks = 8
1034         tmpFile = self._create_temp_file(num_of_blocks)
1035         expected = dict(
1036                 success=201,
1037                 data=num_of_blocks * 4 * 1024 * 1024,
1038                 etag='some-etag',
1039                 content_encoding='some content_encoding',
1040                 content_type='some content-type',
1041                 content_disposition='some content_disposition',
1042                 public=True,
1043                 permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
1044         self.client.upload_object_unchunked(obj, tmpFile)
1045         self.assertEqual(put.mock_calls[-1][1], (obj,))
1046         self.assertEqual(
1047             sorted(put.mock_calls[-1][2].keys()),
1048             sorted(expected.keys()))
1049         kwargs = dict(expected)
1050         kwargs.pop('success')
1051         kwargs['size'] = kwargs.pop('data')
1052         kwargs['sharing'] = kwargs.pop('permissions')
1053         tmpFile.seek(0)
1054         self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
1055         pmc = put.mock_calls[-1][2]
1056         for k, v in expected.items():
1057             if k == 'data':
1058                 self.assertEqual(len(pmc[k]), v)
1059             else:
1060                 self.assertEqual(pmc[k], v)
1061         self.assertRaises(
1062             ClientError,
1063             self.client.upload_object_unchunked,
1064             obj, tmpFile, withHashFile=True)
1065
1066     @patch('%s.object_put' % pithos_pkg, return_value=FR())
1067     def test_create_object_by_manifestation(self, put):
1068         manifest = '%s/%s' % (self.client.container, obj)
1069         kwargs = dict(
1070                 etag='some-etag',
1071                 content_encoding='some content_encoding',
1072                 content_type='some content-type',
1073                 content_disposition='some content_disposition',
1074                 public=True,
1075                 sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
1076         self.client.create_object_by_manifestation(obj)
1077         expected = dict(content_length=0, manifest=manifest)
1078         for k in kwargs:
1079             expected['permissions' if k == 'sharing' else k] = None
1080         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
1081         self.client.create_object_by_manifestation(obj, **kwargs)
1082         expected.update(kwargs)
1083         expected['permissions'] = expected.pop('sharing')
1084         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
1085
1086     @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
1087     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1088     def test_download_object(self, GET, GOH):
1089         num_of_blocks = 8
1090         tmpFile = self._create_temp_file(num_of_blocks)
1091         FR.content = tmpFile.read(4 * 1024 * 1024)
1092         tmpFile = self._create_temp_file(num_of_blocks)
1093         num_of_blocks = len(object_hashmap['hashes'])
1094         kwargs = dict(
1095             resume=True,
1096             version='version',
1097             range_str='10-20',
1098             if_match='if and only if',
1099             if_none_match='if and only not',
1100             if_modified_since='what if not?',
1101             if_unmodified_since='this happens if not!',
1102             async_headers=dict(Range='bytes=0-88888888'))
1103
1104         self.client.download_object(obj, tmpFile)
1105         self.assertEqual(len(GET.mock_calls), num_of_blocks)
1106         self.assertEqual(GET.mock_calls[-1][1], (obj,))
1107         for k, v in kwargs.items():
1108             if k == 'async_headers':
1109                 self.assertTrue('Range' in GET.mock_calls[-1][2][k])
1110             elif k in ('resume', 'range_str'):
1111                 continue
1112             else:
1113                 self.assertEqual(GET.mock_calls[-1][2][k], None)
1114
1115         #  Check ranges are consecutive
1116         starts = []
1117         ends = []
1118         for c in GET.mock_calls:
1119             rng_str = c[2]['async_headers']['Range']
1120             (start, rng_str) = rng_str.split('=')
1121             (start, end) = rng_str.split('-')
1122             starts.append(start)
1123             ends.append(end)
1124         ends = sorted(ends)
1125         for i, start in enumerate(sorted(starts)):
1126             if i:
1127                 int(ends[i - 1]) == int(start) - 1
1128
1129         #  With progress bars
1130         try:
1131             from progress.bar import ShadyBar
1132             dl_bar = ShadyBar('Mock dl')
1133         except ImportError:
1134             dl_bar = None
1135
1136         if dl_bar:
1137
1138             def blck_gen(n):
1139                 for i in dl_bar.iter(range(n)):
1140                     yield
1141                 yield
1142
1143             tmpFile.seek(0)
1144             self.client.download_object(obj, tmpFile, download_cb=blck_gen)
1145             self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
1146
1147         tmpFile.seek(0)
1148         kwargs.pop('async_headers')
1149         kwargs.pop('resume')
1150         self.client.download_object(obj, tmpFile, **kwargs)
1151         for k, v in kwargs.items():
1152             if k == 'range_str':
1153                 self.assertEqual(
1154                     GET.mock_calls[-1][2]['data_range'],
1155                     'bytes=%s' % v)
1156             else:
1157                 self.assertEqual(GET.mock_calls[-1][2][k], v)
1158
1159         #  ALl options on no tty
1160
1161         def foo():
1162             return True
1163
1164         tmpFile.seek(0)
1165         tmpFile.isatty = foo
1166         self.client.download_object(obj, tmpFile, **kwargs)
1167         for k, v in kwargs.items():
1168             if k == 'range_str':
1169                 self.assertTrue('data_range' in GET.mock_calls[-1][2])
1170             else:
1171                 self.assertEqual(GET.mock_calls[-1][2][k], v)
1172
1173     def test_get_object_hashmap(self):
1174         FR.json = object_hashmap
1175         for empty in (304, 412):
1176             with patch.object(
1177                     PithosClient, 'object_get',
1178                     side_effect=ClientError('Empty', status=empty)):
1179                 r = self.client.get_object_hashmap(obj)
1180                 self.assertEqual(r, {})
1181         exp_args = dict(
1182             hashmap=True,
1183             data_range=None,
1184             version=None,
1185             if_etag_match=None,
1186             if_etag_not_match=None,
1187             if_modified_since=None,
1188             if_unmodified_since=None)
1189         kwargs = dict(
1190             version='s0m3v3r51on',
1191             if_match='if match',
1192             if_none_match='if non match',
1193             if_modified_since='some date here',
1194             if_unmodified_since='some date here',
1195             data_range='10-20')
1196         with patch.object(
1197                 PithosClient, 'object_get',
1198                 return_value=FR()) as get:
1199             r = self.client.get_object_hashmap(obj)
1200             self.assertEqual(r, object_hashmap)
1201             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
1202             r = self.client.get_object_hashmap(obj, **kwargs)
1203             exp_args['if_etag_match'] = kwargs.pop('if_match')
1204             exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
1205             exp_args.update(kwargs)
1206             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
1207
1208     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1209     def test_set_account_group(self, post):
1210         (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
1211         self.client.set_account_group(group, usernames)
1212         post.assert_called_once_with(update=True, groups={group: usernames})
1213
1214     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1215     def test_del_account_group(self, post):
1216         group = 'aU53rGr0up'
1217         self.client.del_account_group(group)
1218         post.assert_called_once_with(update=True, groups={group: []})
1219
1220     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
1221     def test_get_account_quota(self, GAI):
1222         key = 'x-account-policy-quota'
1223         r = self.client.get_account_quota()
1224         GAI.assert_called_once_with()
1225         self.assertEqual(r[key], account_info[key])
1226
1227     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
1228     def test_get_account_versioning(self, GAI):
1229         key = 'x-account-policy-versioning'
1230         r = self.client.get_account_versioning()
1231         GAI.assert_called_once_with()
1232         self.assertEqual(r[key], account_info[key])
1233
1234     def test_get_account_meta(self):
1235         key = 'x-account-meta-'
1236         with patch.object(
1237                 PithosClient, 'get_account_info',
1238                 return_value=account_info):
1239             r = self.client.get_account_meta()
1240             keys = [k for k in r if k.startswith(key)]
1241             self.assertFalse(keys)
1242         acc_info = dict(account_info)
1243         acc_info['%sk1' % key] = 'v1'
1244         acc_info['%sk2' % key] = 'v2'
1245         acc_info['%sk3' % key] = 'v3'
1246         with patch.object(
1247                 PithosClient, 'get_account_info',
1248                 return_value=acc_info):
1249             r = self.client.get_account_meta()
1250             for k in [k for k in acc_info if k.startswith(key)]:
1251                 self.assertEqual(r[k], acc_info[k])
1252
1253     def test_get_account_group(self):
1254         key = 'x-account-group-'
1255         with patch.object(
1256                 PithosClient, 'get_account_info',
1257                 return_value=account_info):
1258             r = self.client.get_account_group()
1259             keys = [k for k in r if k.startswith(key)]
1260             self.assertFalse(keys)
1261         acc_info = dict(account_info)
1262         acc_info['%sk1' % key] = 'g1'
1263         acc_info['%sk2' % key] = 'g2'
1264         acc_info['%sk3' % key] = 'g3'
1265         with patch.object(
1266                 PithosClient, 'get_account_info',
1267                 return_value=acc_info):
1268             r = self.client.get_account_group()
1269             for k in [k for k in acc_info if k.startswith(key)]:
1270                 self.assertEqual(r[k], acc_info[k])
1271
1272     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1273     def test_set_account_meta(self, post):
1274         metas = dict(k1='v1', k2='v2', k3='v3')
1275         self.client.set_account_meta(metas)
1276         post.assert_called_once_with(update=True, metadata=metas)
1277
1278     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1279     def test_set_account_quota(self, post):
1280         qu = 1024
1281         self.client.set_account_quota(qu)
1282         post.assert_called_once_with(update=True, quota=qu)
1283
1284     @patch('%s.account_post' % pithos_pkg, return_value=FR())
1285     def test_set_account_versioning(self, post):
1286         vrs = 'n3wV3r51on1ngTyp3'
1287         self.client.set_account_versioning(vrs)
1288         post.assert_called_once_with(update=True, versioning=vrs)
1289
1290     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
1291     def test_del_container(self, delete):
1292         for kwarg in (
1293                 dict(delimiter=None, until=None),
1294                 dict(delimiter='X', until='50m3d473')):
1295             self.client.del_container(**kwarg)
1296             expected = dict(kwarg)
1297             expected['success'] = (204, 404, 409)
1298             self.assertEqual(delete.mock_calls[-1], call(**expected))
1299         for status_code in (404, 409):
1300             FR.status_code = status_code
1301             self.assertRaises(ClientError, self.client.del_container)
1302
1303     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1304     def test_get_container_versioning(self, GCI):
1305         key = 'x-container-policy-versioning'
1306         cont = 'c0n7-417'
1307         bu_cnt = self.client.container
1308         for container in (None, cont):
1309             r = self.client.get_container_versioning(container=container)
1310             self.assertEqual(r[key], container_info[key])
1311             self.assertEqual(GCI.mock_calls[-1], call())
1312             self.assertEqual(bu_cnt, self.client.container)
1313
1314     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1315     def test_get_container_quota(self, GCI):
1316         key = 'x-container-policy-quota'
1317         cont = 'c0n7-417'
1318         bu_cnt = self.client.container
1319         for container in (None, cont):
1320             r = self.client.get_container_quota(container=container)
1321             self.assertEqual(r[key], container_info[key])
1322             self.assertEqual(GCI.mock_calls[-1], call())
1323             self.assertEqual(bu_cnt, self.client.container)
1324
1325     def test_get_container_meta(self):
1326         somedate = '50m3d473'
1327         key = 'x-container-meta'
1328         metaval = '50m3m374v41'
1329         container_plus = dict(container_info)
1330         container_plus[key] = metaval
1331         for ret in ((container_info, {}), (container_plus, {key: metaval})):
1332             with patch.object(
1333                     PithosClient,
1334                     'get_container_info',
1335                     return_value=ret[0]) as GCI:
1336                 for until in (None, somedate):
1337                     r = self.client.get_container_meta(until=until)
1338                     self.assertEqual(r, ret[1])
1339                     self.assertEqual(GCI.mock_calls[-1], call(until=until))
1340
1341     def test_get_container_object_meta(self):
1342         somedate = '50m3d473'
1343         key = 'x-container-object-meta'
1344         metaval = '50m3m374v41'
1345         container_plus = dict(container_info)
1346         container_plus[key] = metaval
1347         for ret in (
1348                 (container_info, {key: ''}),
1349                 (container_plus, {key: metaval})):
1350             with patch.object(
1351                     PithosClient,
1352                     'get_container_info',
1353                     return_value=ret[0]) as GCI:
1354                 for until in (None, somedate):
1355                     r = self.client.get_container_object_meta(until=until)
1356                     self.assertEqual(r, ret[1])
1357                     self.assertEqual(GCI.mock_calls[-1], call(until=until))
1358
1359     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1360     def test_set_container_meta(self, post):
1361         metas = dict(k1='v1', k2='v2', k3='v3')
1362         self.client.set_container_meta(metas)
1363         post.assert_called_once_with(update=True, metadata=metas)
1364
1365     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1366     def test_del_container_meta(self, AP):
1367         self.client.del_container_meta('somekey')
1368         AP.assert_called_once_with(update=True, metadata={'somekey': ''})
1369
1370     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1371     def test_set_container_quota(self, post):
1372         qu = 1024
1373         self.client.set_container_quota(qu)
1374         post.assert_called_once_with(update=True, quota=qu)
1375
1376     @patch('%s.container_post' % pithos_pkg, return_value=FR())
1377     def test_set_container_versioning(self, post):
1378         vrs = 'n3wV3r51on1ngTyp3'
1379         self.client.set_container_versioning(vrs)
1380         post.assert_called_once_with(update=True, versioning=vrs)
1381
1382     @patch('%s.object_delete' % pithos_pkg, return_value=FR())
1383     def test_del_object(self, delete):
1384         for kwarg in (
1385                 dict(delimiter=None, until=None),
1386                 dict(delimiter='X', until='50m3d473')):
1387             self.client.del_object(obj, **kwarg)
1388             self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
1389
1390     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1391     def test_set_object_meta(self, post):
1392         metas = dict(k1='v1', k2='v2', k3='v3')
1393         self.assertRaises(
1394             AssertionError,
1395             self.client.set_object_meta,
1396             obj, 'Non dict arg')
1397         self.client.set_object_meta(obj, metas)
1398         post.assert_called_once_with(obj, update=True, metadata=metas)
1399
1400     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1401     def test_publish_object(self, post):
1402         oinfo = dict(object_info)
1403         val = 'pubL1c'
1404         oinfo['x-object-public'] = val
1405         with patch.object(
1406                 PithosClient, 'get_object_info',
1407                 return_value=oinfo) as GOF:
1408             r = self.client.publish_object(obj)
1409             self.assertEqual(
1410                 post.mock_calls[-1],
1411                 call(obj, public=True, update=True))
1412             self.assertEqual(GOF.mock_calls[-1], call(obj))
1413             self.assertEqual(r, '%s%s' % (self.url[:-6], val))
1414
1415     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1416     def test_unpublish_object(self, post):
1417         self.client.unpublish_object(obj)
1418         post.assert_called_once_with(obj, public=False, update=True)
1419
1420     def test_get_object_sharing(self):
1421         info = dict(object_info)
1422         expected = dict(read='u1,g1,u2', write='u1')
1423         info['x-object-sharing'] = '; '.join(
1424             ['%s=%s' % (k, v) for k, v in expected.items()])
1425         with patch.object(
1426                 PithosClient, 'get_object_info',
1427                 return_value=info) as GOF:
1428             r = self.client.get_object_sharing(obj)
1429             self.assertEqual(GOF.mock_calls[-1], call(obj))
1430             self.assert_dicts_are_equal(r, expected)
1431             info['x-object-sharing'] = '//'.join(
1432                 ['%s=%s' % (k, v) for k, v in expected.items()])
1433             self.assertRaises(
1434                 ValueError,
1435                 self.client.get_object_sharing,
1436                 obj)
1437             info['x-object-sharing'] = '; '.join(
1438                 ['%s:%s' % (k, v) for k, v in expected.items()])
1439             self.assertRaises(
1440                 ClientError,
1441                 self.client.get_object_sharing,
1442                 obj)
1443             info['x-object-sharing'] = 'read=%s' % expected['read']
1444             r = self.client.get_object_sharing(obj)
1445             expected.pop('write')
1446             self.assert_dicts_are_equal(r, expected)
1447
1448     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1449     def test_set_object_sharing(self, OP):
1450         read_perms = ['u1', 'g1', 'u2', 'g2']
1451         write_perms = ['u1', 'g1']
1452         for kwargs in (
1453                 dict(read_permition=read_perms, write_permition=write_perms),
1454                 dict(read_permition=read_perms),
1455                 dict(write_permition=write_perms),
1456                 dict()):
1457             self.client.set_object_sharing(obj, **kwargs)
1458             kwargs['read'] = kwargs.pop('read_permition', '')
1459             kwargs['write'] = kwargs.pop('write_permition', '')
1460             self.assertEqual(
1461                 OP.mock_calls[-1],
1462                 call(obj, update=True, permissions=kwargs))
1463
1464     @patch('%s.set_object_sharing' % pithos_pkg)
1465     def test_del_object_sharing(self, SOS):
1466         self.client.del_object_sharing(obj)
1467         SOS.assert_called_once_with(obj)
1468
1469     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1470     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1471     def test_append_object(self, post, GCI):
1472         num_of_blocks = 4
1473         tmpFile = self._create_temp_file(num_of_blocks)
1474         tmpFile.seek(0, 2)
1475         file_size = tmpFile.tell()
1476         for turn in range(2):
1477             tmpFile.seek(0, 0)
1478
1479             try:
1480                 from progress.bar import ShadyBar
1481                 apn_bar = ShadyBar('Mock append')
1482             except ImportError:
1483                 apn_bar = None
1484
1485             if apn_bar:
1486
1487                 def append_gen(n):
1488                     for i in apn_bar.iter(range(n)):
1489                         yield
1490                     yield
1491
1492             else:
1493                 append_gen = None
1494
1495             self.client.append_object(
1496                 obj, tmpFile,
1497                 upload_cb=append_gen if turn else None)
1498             self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
1499             (args, kwargs) = post.mock_calls[-1][1:3]
1500             self.assertEqual(args, (obj,))
1501             self.assertEqual(kwargs['content_length'], len(kwargs['data']))
1502             fsize = num_of_blocks * int(kwargs['content_length'])
1503             self.assertEqual(fsize, file_size)
1504             self.assertEqual(kwargs['content_range'], 'bytes */*')
1505             exp = 'application/octet-stream'
1506             self.assertEqual(kwargs['content_type'], exp)
1507             self.assertEqual(kwargs['update'], True)
1508
1509     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1510     def test_truncate_object(self, post):
1511         upto_bytes = 377
1512         self.client.truncate_object(obj, upto_bytes)
1513         post.assert_called_once_with(
1514             obj,
1515             update=True,
1516             object_bytes=upto_bytes,
1517             content_range='bytes 0-%s/*' % upto_bytes,
1518             content_type='application/octet-stream',
1519             source_object='/%s/%s' % (self.client.container, obj))
1520
1521     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1522     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1523     def test_overwrite_object(self, post, GCI):
1524         num_of_blocks = 4
1525         tmpFile = self._create_temp_file(num_of_blocks)
1526         tmpFile.seek(0, 2)
1527         file_size = tmpFile.tell()
1528         info = dict(object_info)
1529         info['content-length'] = file_size
1530         block_size = container_info['x-container-block-size']
1531         with patch.object(
1532                 PithosClient, 'get_object_info',
1533                 return_value=info) as GOI:
1534             for start, end in (
1535                     (0, file_size + 1),
1536                     (file_size + 1, file_size + 2)):
1537                 tmpFile.seek(0, 0)
1538                 self.assertRaises(
1539                     ClientError,
1540                     self.client.overwrite_object,
1541                     obj, start, end, tmpFile)
1542             for start, end in ((0, 144), (144, 233), (233, file_size)):
1543                 tmpFile.seek(0, 0)
1544                 owr_gen = None
1545                 exp_size = end - start + 1
1546                 if not start or exp_size > block_size:
1547                     try:
1548                         from progress.bar import ShadyBar
1549                         owr_bar = ShadyBar('Mock append')
1550                     except ImportError:
1551                         owr_bar = None
1552
1553                     if owr_bar:
1554
1555                         def owr_gen(n):
1556                             for i in owr_bar.iter(range(n)):
1557                                 yield
1558                             yield
1559
1560                     if exp_size > block_size:
1561                         exp_size = exp_size % block_size or block_size
1562
1563                 self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
1564                 self.assertEqual(GOI.mock_calls[-1], call(obj))
1565                 self.assertEqual(GCI.mock_calls[-1], call())
1566                 (args, kwargs) = post.mock_calls[-1][1:3]
1567                 self.assertEqual(args, (obj,))
1568                 self.assertEqual(len(kwargs['data']), exp_size)
1569                 self.assertEqual(kwargs['content_length'], exp_size)
1570                 self.assertEqual(kwargs['update'], True)
1571                 exp = 'application/octet-stream'
1572                 self.assertEqual(kwargs['content_type'], exp)
1573
1574     @patch('%s.set_param' % pithos_pkg)
1575     @patch('%s.get' % pithos_pkg, return_value=FR())
1576     def test_get_sharing_accounts(self, get, SP):
1577         FR.json = sharers
1578         for kws in (
1579                 dict(),
1580                 dict(limit='50m3-11m17'),
1581                 dict(marker='X'),
1582                 dict(limit='50m3-11m17', marker='X')):
1583             r = self.client.get_sharing_accounts(**kws)
1584             self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1585             self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
1586             limit, marker = kws.get('limit', None), kws.get('marker', None)
1587             self.assertEqual(SP.mock_calls[-2], call(
1588                 'limit', limit,
1589                 iff=limit is not None))
1590             self.assertEqual(SP.mock_calls[-1], call(
1591                 'marker', marker,
1592                 iff=marker is not None))
1593             for i in range(len(r)):
1594                 self.assert_dicts_are_equal(r[i], sharers[i])
1595
1596     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1597     def test_get_object_versionlist(self, get):
1598         info = dict(object_info)
1599         info['versions'] = ['v1', 'v2']
1600         FR.json = info
1601         r = self.client.get_object_versionlist(obj)
1602         get.assert_called_once_with(obj, format='json', version='list')
1603         self.assertEqual(r, info['versions'])
1604
1605 if __name__ == '__main__':
1606     from sys import argv
1607     from kamaki.clients.test import runTestCase
1608     not_found = True
1609     if not argv[1:] or argv[1] == 'Pithos':
1610         not_found = False
1611         runTestCase(Pithos, 'Pithos Client', argv[2:])
1612     if not argv[1:] or argv[1] == 'PithosRest':
1613         not_found = False
1614         runTestCase(PithosRest, 'PithosRest Client', argv[2:])
1615     if not_found:
1616         print('TestCase %s not found' % argv[1])