code.grnet.gr Git - kamaki/blob - kamaki/clients/pithos/test.py

   1 # Copyright 2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from unittest import TestCase
  35 from mock import patch, call
  36 from tempfile import NamedTemporaryFile
  37 from os import urandom
  38
  39 from kamaki.clients import ClientError
  40 from kamaki.clients.pithos import PithosClient as PC
  41 from kamaki.clients.connection.kamakicon import KamakiHTTPConnection as C
  42
  43 client_pkg = 'kamaki.clients.Client'
  44 pithos_pkg = 'kamaki.clients.pithos.PithosClient'
  45
  46 user_id = 'ac0un7-1d-5tr1ng'
  47 obj = 'obj3c7N4m3'
  48
  49 account_info = {
  50     'content-language': 'en-us',
  51     'content-type': 'text/html; charset=utf-8',
  52     'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
  53     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  54     'server': 'gunicorn/0.14.5',
  55     'vary': 'Accept-Language',
  56     'x-account-bytes-used': '751615526',
  57     'x-account-container-count': 7,
  58     'x-account-policy-quota': 53687091200,
  59     'x-account-policy-versioning': 'auto'}
  60 container_info = {
  61     'content-language': 'en-us',
  62     'content-type': 'text/html; charset=utf-8',
  63     'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
  64     'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
  65     'server': 'gunicorn/0.14.5',
  66     'vary': 'Accept-Language',
  67     'x-container-block-hash': 'sha256',
  68     'x-container-block-size': 4194304,
  69     'x-container-bytes-used': 309528938,
  70     'x-container-object-count': 14,
  71     'x-container-object-meta': '',
  72     'x-container-policy-quota': 53687091200,
  73     'x-container-policy-versioning': 'auto'}
  74 object_info = {
  75     'content-language': 'en-us',
  76     'content-length': 254965,
  77     'content-type': 'application/octet-stream',
  78     'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
  79     'etag': '',
  80     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  81     'server': 'gunicorn/0.14.5',
  82     'vary': 'Accept-Language',
  83     'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
  84     'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
  85     'x-object-version': '525996',
  86     'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
  87     'x-object-meta-k1': 'v1',
  88     'x-object-meta-k2': 'v2'}
  89 container_list = [
  90     dict(
  91         count=2,
  92         last_modified="2013-02-27T11:56:09.893033+00:00",
  93         bytes=677076979,
  94         name="pithos",
  95         x_container_policy=dict(quota="21474836480", versioning="auto")),
  96     dict(
  97         count=0,
  98         last_modified="2012-10-23T12:25:17.229187+00:00",
  99         bytes=0,
 100         name="trash",
 101         x_container_policy=dict(quota="21474836480", versioning="auto"))]
 102 object_list = [
 103     dict(hash="",
 104         name="The_Secret_Garden.zip",
 105         x_object_public="/public/wdp9p",
 106         bytes=203304947,
 107         x_object_version_timestamp="1360237915.7027509",
 108         x_object_uuid="s0m3uu1df0r0bj0n3",
 109         last_modified="2013-02-07T11:51:55.702751+00:00",
 110         content_type="application/octet-stream",
 111         x_object_hash="0afdf29f71cd53126225c3f54ca",
 112         x_object_version=17737,
 113         x_object_modified_by=user_id),
 114     dict(hash="",
 115         name="The_Revealed_Garden.zip",
 116         x_object_public="/public/wpd7p",
 117         bytes=20330947,
 118         x_object_version_timestamp="13602915.7027509",
 119         x_object_uuid="s0m3uu1df0r0bj70w",
 120         last_modified="2013-02-07T11:51:55.702751+00:00",
 121         content_type="application/octet-stream",
 122         x_object_hash="0afdf29f71cd53126225c3f54ca",
 123         x_object_version=17737,
 124         x_object_modified_by=user_id)]
 125 object_hashmap = dict(
 126     block_hash="sha256", block_size=4194304, bytes=33554432,
 127     hashes=[
 128         "4988438cc1c0292c085d289649b28cf547ba3db71c6efaac9f2df7e193d4d0af",
 129         "b214244aa56df7d1df7c6cac066e7cef268d9c2beb4dcf7ce68af667b0626f91",
 130         "17f365f25e0682565ded30576066bb13377a3d306967e4d74e06bb6bbc20f75f",
 131         "2524ae208932669fff89adf8a2fc0df3b67736ca0d3aadce7a2ce640f142af37",
 132         "5d807a2129d2fcd3c221c3da418ed52af3fc48d0817b62e0bb437acffccd3514",
 133         "609de22ce842d997f645fc49d5f14e0e3766dd51a6cbe66383b2bab82c8dfcd0",
 134         "3102851ac168c78be70e35ff5178c7b1ebebd589e5106d565ca1094d1ca8ff59",
 135         "bfe306dd24e92a8d85caf7055643f250fd319e8c4cdd4755ddabbf3ff97e83c7"])
 136 sharers = [
 137     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b1a-82d5"),
 138     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="0b2a-f2d5"),
 139     dict(last_modified="2013-01-29T16:50:06.084674+00:00", name="2b1a-82d6")]
 140
 141
 142 class FR(object):
 143     """FR stands for Fake Response"""
 144     json = dict()
 145     headers = dict()
 146     content = json
 147     status = None
 148     status_code = 200
 149
 150     def release(self):
 151         pass
 152
 153
 154 class Pithos(TestCase):
 155
 156     files = []
 157
 158     def _create_temp_file(self, num_of_blocks):
 159         self.files.append(NamedTemporaryFile())
 160         tmpFile = self.files[-1]
 161         file_size = num_of_blocks * 4 * 1024 * 1024
 162         print('\n\tCreate tmp file')
 163         tmpFile.write(urandom(file_size))
 164         tmpFile.flush()
 165         tmpFile.seek(0)
 166         print('\t\tDone')
 167         return tmpFile
 168
 169     def assert_dicts_are_equal(self, d1, d2):
 170         for k, v in d1.items():
 171             self.assertTrue(k in d2)
 172             if isinstance(v, dict):
 173                 self.assert_dicts_are_equal(v, d2[k])
 174             else:
 175                 self.assertEqual(unicode(v), unicode(d2[k]))
 176
 177     def setUp(self):
 178         self.url = 'https://www.example.com/pithos'
 179         self.token = 'p17h0570k3n'
 180         self.client = PC(self.url, self.token)
 181         self.client.account = user_id
 182         self.client.container = 'c0nt@1n3r_i'
 183
 184     def tearDown(self):
 185         FR.headers = dict()
 186         FR.status_code = 200
 187         FR.json = dict()
 188         FR.content = FR.json
 189         for f in self.files:
 190             f.close()
 191
 192     #  Pithos+ methods that extend storage API
 193
 194     @patch('%s.account_head' % pithos_pkg, return_value=FR())
 195     def test_get_account_info(self, AH):
 196         FR.headers = account_info
 197         r = self.client.get_account_info()
 198         self.assert_dicts_are_equal(r, account_info)
 199         self.assertEqual(AH.mock_calls[-1], call(until=None))
 200         unt = 'un71L-d473'
 201         r = self.client.get_account_info(until=unt)
 202         self.assert_dicts_are_equal(r, account_info)
 203         self.assertEqual(AH.mock_calls[-1], call(until=unt))
 204         FR.status_code = 401
 205         self.assertRaises(ClientError, self.client.get_account_info)
 206
 207     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 208     def test_del_account_meta(self, ap):
 209         keys = ['k1', 'k2', 'k3']
 210         expected = []
 211         for key in keys:
 212             self.client.del_account_meta(key)
 213             expected.append(call(update=True, metadata={key: ''}))
 214         self.assertEqual(ap.mock_calls, expected)
 215
 216     @patch('%s.container_head' % pithos_pkg, return_value=FR())
 217     def test_get_container_info(self, ch):
 218         FR.headers = container_info
 219         r = self.client.get_container_info()
 220         self.assert_dicts_are_equal(r, container_info)
 221         u = 'some date'
 222         r = self.client.get_container_info(until=u)
 223         self.assertEqual(ch.mock_calls, [call(until=None), call(until=u)])
 224
 225     @patch('%s.account_get' % pithos_pkg, return_value=FR())
 226     def test_list_containers(self, get):
 227         FR.json = container_list
 228         r = self.client.list_containers()
 229         for i in range(len(r)):
 230             self.assert_dicts_are_equal(r[i], container_list[i])
 231
 232     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 233     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 234     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 235     def test_upload_object(self, CI, CP, OP):
 236         num_of_blocks = 8
 237         tmpFile = self._create_temp_file(num_of_blocks)
 238
 239         # Without kwargs
 240         self.client.upload_object(obj, tmpFile)
 241         self.assertEqual(PC.get_container_info.mock_calls, [call()])
 242         [call1, call2] = PC.object_put.mock_calls
 243
 244         (args1, kwargs1) = call1[1:3]
 245         (args2, kwargs2) = call2[1:3]
 246         self.assertEqual(args1, (obj,))
 247         expected1 = dict(
 248             hashmap=True,
 249             success=(201, 409),
 250             format='json',
 251             json=dict(
 252                 hashes=['s0m3h@5h'] * num_of_blocks,
 253                 bytes=num_of_blocks * 4 * 1024 * 1024),
 254             etag=None,
 255             content_encoding=None,
 256             content_type='application/octet-stream',
 257             content_disposition=None,
 258             public=None,
 259             permissions=None)
 260         for k, v in expected1.items():
 261             if k == 'json':
 262                 self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
 263                 self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
 264             else:
 265                 self.assertEqual(v, kwargs1[k])
 266
 267         (args2, kwargs2) = call2[1:3]
 268         self.assertEqual(args2, (obj,))
 269         expected2 = dict(
 270             json=dict(
 271                 hashes=['s0m3h@5h'] * num_of_blocks,
 272                 bytes=num_of_blocks * 4 * 1024 * 1024),
 273             content_type='application/octet-stream',
 274             hashmap=True,
 275             success=201,
 276             format='json')
 277         for k, v in expected2.items():
 278             if k == 'json':
 279                 self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
 280                 self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
 281             else:
 282                 self.assertEqual(v, kwargs2[k])
 283
 284         OP = PC.object_put
 285         mock_offset = 2
 286
 287         #  With progress bars
 288         try:
 289             from progress.bar import ShadyBar
 290             blck_bar = ShadyBar('Mock blck calc.')
 291             upld_bar = ShadyBar('Mock uplds')
 292         except ImportError:
 293             blck_bar = None
 294             upld_bar = None
 295
 296         if blck_bar and upld_bar:
 297
 298             def blck_gen(n):
 299                 for i in blck_bar.iter(range(n)):
 300                     yield
 301                 yield
 302
 303             def upld_gen(n):
 304                 for i in upld_bar.iter(range(n)):
 305                     yield
 306                 yield
 307
 308             tmpFile.seek(0)
 309             self.client.upload_object(
 310                 obj, tmpFile,
 311                 hash_cb=blck_gen, upload_cb=upld_gen)
 312
 313             for i, c in enumerate(OP.mock_calls[-mock_offset:]):
 314                 self.assertEqual(OP.mock_calls[i], c)
 315
 316         #  With content-type
 317         tmpFile.seek(0)
 318         ctype = 'video/mpeg'
 319         sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
 320         self.client.upload_object(obj, tmpFile,
 321             content_type=ctype, sharing=sharing)
 322         self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
 323         self.assert_dicts_are_equal(
 324             OP.mock_calls[-2][2]['permissions'],
 325             sharing)
 326
 327         # With other args
 328         tmpFile.seek(0)
 329         kwargs = dict(
 330             etag='s0m3E74g',
 331             content_type=ctype,
 332             content_disposition=ctype + 'd15p051710n',
 333             public=True,
 334             content_encoding='802.11')
 335         self.client.upload_object(obj, tmpFile, **kwargs)
 336         for arg, val in kwargs.items():
 337             self.assertEqual(OP.mock_calls[-2][2][arg], val)
 338
 339     @patch('%s.put' % pithos_pkg, return_value=FR())
 340     @patch('%s.set_header' % client_pkg)
 341     def test_create_directory(self, SH, put):
 342         cont = self.client.container
 343         exp_shd = [
 344             call('Content-Type', 'application/directory'),
 345             call('Content-length', '0')]
 346         exp_put = [call('/%s/%s/%s' % (user_id, cont, obj), success=201)]
 347         self.client.create_directory(obj)
 348         self.assertEqual(PC.set_header.mock_calls, exp_shd)
 349         self.assertEqual(put.mock_calls, exp_put)
 350
 351     def test_get_object_info(self):
 352         FR.headers = object_info
 353         version = 'v3r510n'
 354         with patch.object(PC, 'object_head', return_value=FR()) as head:
 355             r = self.client.get_object_info(obj)
 356             self.assertEqual(r, object_info)
 357             r = self.client.get_object_info(obj, version=version)
 358             self.assertEqual(head.mock_calls, [
 359                 call(obj, version=None),
 360                 call(obj, version=version)])
 361         with patch.object(
 362                 PC,
 363                 'object_head',
 364                 side_effect=ClientError('Obj not found', 404)):
 365             self.assertRaises(
 366                 ClientError,
 367                 self.client.get_object_info,
 368                 obj, version=version)
 369
 370     @patch('%s.get_object_info' % pithos_pkg, return_value=object_info)
 371     def test_get_object_meta(self, GOI):
 372         expected = dict()
 373         for k, v in object_info.items():
 374             expected[k] = v
 375         r = self.client.get_object_meta(obj)
 376         self.assert_dicts_are_equal(r, expected)
 377
 378     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 379     def test_del_object_meta(self, post):
 380         metakey = '50m3m3t4k3y'
 381         self.client.del_object_meta(obj, metakey)
 382         expected = call(obj, update=True, metadata={metakey: ''})
 383         self.assertEqual(post.mock_calls[-1], expected)
 384
 385     @patch('%s.post' % client_pkg, return_value=FR())
 386     @patch('%s.set_header' % client_pkg)
 387     def test_replace_object_meta(self, SH, post):
 388         metas = dict(k1='new1', k2='new2', k3='new3')
 389         cont = self.client.container
 390         self.client.replace_object_meta(metas)
 391         expected = call('/%s/%s' % (user_id, cont), success=202)
 392         self.assertEqual(post.mock_calls[-1], expected)
 393         prfx = 'X-Object-Meta-'
 394         expected = [call('%s%s' % (prfx, k), v) for k, v in metas.items()]
 395         self.assertEqual(PC.set_header.mock_calls, expected)
 396
 397     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 398     def test_copy_object(self, put):
 399         src_cont = 'src-c0nt41n3r'
 400         src_obj = 'src-0bj'
 401         dst_cont = 'dst-c0nt41n3r'
 402         dst_obj = 'dst-0bj'
 403         expected = call(
 404             src_obj,
 405             content_length=0,
 406             source_account=None,
 407             success=201,
 408             copy_from='/%s/%s' % (src_cont, src_obj),
 409             delimiter=None,
 410             content_type=None,
 411             source_version=None,
 412             public=False)
 413         self.client.copy_object(src_cont, src_obj, dst_cont)
 414         self.assertEqual(put.mock_calls[-1], expected)
 415         self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
 416         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 417         kwargs = dict(
 418             source_version='src-v3r510n',
 419             source_account='src-4cc0un7',
 420             public=True,
 421             content_type='c0n73n7Typ3',
 422             delimiter='5')
 423         self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
 424         for k, v in kwargs.items():
 425             self.assertEqual(v, put.mock_calls[-1][2][k])
 426
 427     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 428     def test_move_object(self, put):
 429         src_cont = 'src-c0nt41n3r'
 430         src_obj = 'src-0bj'
 431         dst_cont = 'dst-c0nt41n3r'
 432         dst_obj = 'dst-0bj'
 433         expected = call(
 434             src_obj,
 435             content_length=0,
 436             source_account=None,
 437             success=201,
 438             move_from='/%s/%s' % (src_cont, src_obj),
 439             delimiter=None,
 440             content_type=None,
 441             source_version=None,
 442             public=False)
 443         self.client.move_object(src_cont, src_obj, dst_cont)
 444         self.assertEqual(put.mock_calls[-1], expected)
 445         self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
 446         self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 447         kwargs = dict(
 448             source_version='src-v3r510n',
 449             source_account='src-4cc0un7',
 450             public=True,
 451             content_type='c0n73n7Typ3',
 452             delimiter='5')
 453         self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
 454         for k, v in kwargs.items():
 455             self.assertEqual(v, put.mock_calls[-1][2][k])
 456
 457     @patch('%s.delete' % client_pkg, return_value=FR())
 458     def test_delete_object(self, delete):
 459         cont = self.client.container
 460         self.client.delete_object(obj)
 461         self.assertEqual(
 462             delete.mock_calls[-1],
 463             call('/%s/%s/%s' % (user_id, cont, obj), success=(204, 404)))
 464         FR.status_code = 404
 465         self.assertRaises(ClientError, self.client.delete_object, obj)
 466
 467     @patch('%s.get' % client_pkg, return_value=FR())
 468     @patch('%s.set_param' % client_pkg)
 469     def test_list_objects(self, SP, get):
 470         FR.json = object_list
 471         acc = self.client.account
 472         cont = self.client.container
 473         SP = PC.set_param
 474         r = self.client.list_objects()
 475         for i in range(len(r)):
 476             self.assert_dicts_are_equal(r[i], object_list[i])
 477         self.assertEqual(get.mock_calls, [
 478             call('/%s/%s' % (acc, cont), success=(200, 204, 304, 404))])
 479         self.assertEqual(SP.mock_calls, [call('format', 'json')])
 480         FR.status_code = 304
 481         self.assertEqual(self.client.list_objects(), [])
 482         FR.status_code = 404
 483         self.assertRaises(ClientError, self.client.list_objects)
 484
 485     @patch('%s.get' % client_pkg, return_value=FR())
 486     @patch('%s.set_param' % client_pkg)
 487     def test_list_objects_in_path(self, SP, get):
 488         FR.json = object_list
 489         path = '/some/awsome/path'
 490         acc = self.client.account
 491         cont = self.client.container
 492         SP = PC.set_param
 493         self.client.list_objects_in_path(path)
 494         self.assertEqual(get.mock_calls, [
 495             call('/%s/%s' % (acc, cont), success=(200, 204, 404))])
 496         self.assertEqual(SP.mock_calls, [
 497             call('format', 'json'), call('path', path)])
 498         FR.status_code = 404
 499         self.assertRaises(ClientError, self.client.list_objects)
 500
 501     #  Pithos+ only methods
 502
 503     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 504     def test_purge_container(self, cd):
 505         self.client.purge_container()
 506         self.assertTrue('until' in cd.mock_calls[-1][2])
 507         cont = self.client.container
 508         self.client.purge_container('another-container')
 509         self.assertEqual(self.client.container, cont)
 510
 511     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 512     def test_upload_object_unchunked(self, put):
 513         num_of_blocks = 8
 514         tmpFile = self._create_temp_file(num_of_blocks)
 515         expected = dict(
 516                 success=201,
 517                 data=num_of_blocks * 4 * 1024 * 1024,
 518                 etag='some-etag',
 519                 content_encoding='some content_encoding',
 520                 content_type='some content-type',
 521                 content_disposition='some content_disposition',
 522                 public=True,
 523                 permissions=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 524         self.client.upload_object_unchunked(obj, tmpFile)
 525         self.assertEqual(put.mock_calls[-1][1], (obj,))
 526         self.assertEqual(
 527             sorted(put.mock_calls[-1][2].keys()),
 528             sorted(expected.keys()))
 529         kwargs = dict(expected)
 530         kwargs.pop('success')
 531         kwargs['size'] = kwargs.pop('data')
 532         kwargs['sharing'] = kwargs.pop('permissions')
 533         tmpFile.seek(0)
 534         self.client.upload_object_unchunked(obj, tmpFile, **kwargs)
 535         pmc = put.mock_calls[-1][2]
 536         for k, v in expected.items():
 537             if k == 'data':
 538                 self.assertEqual(len(pmc[k]), v)
 539             else:
 540                 self.assertEqual(pmc[k], v)
 541         self.assertRaises(
 542             ClientError,
 543             self.client.upload_object_unchunked,
 544             obj, tmpFile, withHashFile=True)
 545
 546     @patch('%s.object_put' % pithos_pkg, return_value=FR())
 547     def test_create_object_by_manifestation(self, put):
 548         manifest = '%s/%s' % (self.client.container, obj)
 549         kwargs = dict(
 550                 etag='some-etag',
 551                 content_encoding='some content_encoding',
 552                 content_type='some content-type',
 553                 content_disposition='some content_disposition',
 554                 public=True,
 555                 sharing=dict(read=['u1', 'g1', 'u2'], write=['u1']))
 556         self.client.create_object_by_manifestation(obj)
 557         expected = dict(content_length=0, manifest=manifest)
 558         for k in kwargs:
 559             expected['permissions' if k == 'sharing' else k] = None
 560         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 561         self.client.create_object_by_manifestation(obj, **kwargs)
 562         expected.update(kwargs)
 563         expected['permissions'] = expected.pop('sharing')
 564         self.assertEqual(put.mock_calls[-1], call(obj, **expected))
 565
 566     @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
 567     @patch('%s.object_get' % pithos_pkg, return_value=FR())
 568     def test_download_object(self, GOH, GET):
 569         num_of_blocks = 8
 570         tmpFile = self._create_temp_file(num_of_blocks)
 571         FR.content = tmpFile.read(4 * 1024 * 1024)
 572         tmpFile = self._create_temp_file(num_of_blocks)
 573         GET = PC.object_get
 574         num_of_blocks = len(object_hashmap['hashes'])
 575         kwargs = dict(
 576             resume=True,
 577             version='version',
 578             range_str='10-20',
 579             if_match='if and only if',
 580             if_none_match='if and only not',
 581             if_modified_since='what if not?',
 582             if_unmodified_since='this happens if not!',
 583             async_headers=dict(Range='bytes=0-88888888'))
 584
 585         self.client.download_object(obj, tmpFile)
 586         self.assertEqual(len(GET.mock_calls), num_of_blocks)
 587         self.assertEqual(GET.mock_calls[-1][1], (obj,))
 588         for k, v in kwargs.items():
 589             if k == 'async_headers':
 590                 self.assertTrue('Range' in GET.mock_calls[-1][2][k])
 591             elif k in ('resume', 'range_str'):
 592                 continue
 593             else:
 594                 self.assertEqual(GET.mock_calls[-1][2][k], None)
 595
 596         #  Check ranges are consecutive
 597         starts = []
 598         ends = []
 599         for c in GET.mock_calls:
 600             rng_str = c[2]['async_headers']['Range']
 601             (start, rng_str) = rng_str.split('=')
 602             (start, end) = rng_str.split('-')
 603             starts.append(start)
 604             ends.append(end)
 605         ends = sorted(ends)
 606         for i, start in enumerate(sorted(starts)):
 607             if i:
 608                 int(ends[i - 1]) == int(start) - 1
 609
 610         #  With progress bars
 611         try:
 612             from progress.bar import ShadyBar
 613             dl_bar = ShadyBar('Mock dl')
 614         except ImportError:
 615             dl_bar = None
 616
 617         if dl_bar:
 618
 619             def blck_gen(n):
 620                 for i in dl_bar.iter(range(n)):
 621                     yield
 622                 yield
 623
 624             tmpFile.seek(0)
 625             self.client.download_object(obj, tmpFile, download_cb=blck_gen)
 626             self.assertEqual(len(GET.mock_calls), 2 * num_of_blocks)
 627
 628         tmpFile.seek(0)
 629         kwargs.pop('async_headers')
 630         kwargs.pop('resume')
 631         self.client.download_object(obj, tmpFile, **kwargs)
 632         for k, v in kwargs.items():
 633             if k == 'range_str':
 634                 self.assertEqual(
 635                     GET.mock_calls[-1][2]['data_range'],
 636                     'bytes=%s' % v)
 637             else:
 638                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 639
 640         #  ALl options on no tty
 641
 642         def foo():
 643             return True
 644
 645         tmpFile.seek(0)
 646         tmpFile.isatty = foo
 647         self.client.download_object(obj, tmpFile, **kwargs)
 648         for k, v in kwargs.items():
 649             if k == 'range_str':
 650                 self.assertTrue('data_range' in GET.mock_calls[-1][2])
 651             else:
 652                 self.assertEqual(GET.mock_calls[-1][2][k], v)
 653
 654     def test_get_object_hashmap(self):
 655         FR.json = object_hashmap
 656         for empty in (304, 412):
 657             with patch.object(
 658                     PC, 'object_get',
 659                     side_effect=ClientError('Empty', status=empty)):
 660                 r = self.client.get_object_hashmap(obj)
 661                 self.assertEqual(r, {})
 662         exp_args = dict(
 663             hashmap=True,
 664             data_range=None,
 665             version=None,
 666             if_etag_match=None,
 667             if_etag_not_match=None,
 668             if_modified_since=None,
 669             if_unmodified_since=None)
 670         kwargs = dict(
 671             version='s0m3v3r51on',
 672             if_match='if match',
 673             if_none_match='if non match',
 674             if_modified_since='some date here',
 675             if_unmodified_since='some date here',
 676             data_range='10-20')
 677         with patch.object(PC, 'object_get', return_value=FR()) as get:
 678             r = self.client.get_object_hashmap(obj)
 679             self.assertEqual(r, object_hashmap)
 680             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 681             r = self.client.get_object_hashmap(obj, **kwargs)
 682             exp_args['if_etag_match'] = kwargs.pop('if_match')
 683             exp_args['if_etag_not_match'] = kwargs.pop('if_none_match')
 684             exp_args.update(kwargs)
 685             self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
 686
 687     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 688     def test_set_account_group(self, post):
 689         (group, usernames) = ('aU53rGr0up', ['u1', 'u2', 'u3'])
 690         self.client.set_account_group(group, usernames)
 691         self.assertEqual(
 692             post.mock_calls[-1],
 693             call(update=True, groups={group: usernames}))
 694
 695     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 696     def test_del_account_group(self, post):
 697         group = 'aU53rGr0up'
 698         self.client.del_account_group(group)
 699         self.assertEqual(
 700             post.mock_calls[-1],
 701             call(update=True, groups={group: []}))
 702
 703     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 704     def test_get_account_quota(self, GAI):
 705         key = 'x-account-policy-quota'
 706         r = self.client.get_account_quota()
 707         self.assertEqual(r[key], account_info[key])
 708
 709     @patch('%s.get_account_info' % pithos_pkg, return_value=account_info)
 710     def test_get_account_versioning(self, GAI):
 711         key = 'x-account-policy-versioning'
 712         r = self.client.get_account_versioning()
 713         self.assertEqual(r[key], account_info[key])
 714
 715     def test_get_account_meta(self):
 716         key = 'x-account-meta-'
 717         with patch.object(PC, 'get_account_info', return_value=account_info):
 718             r = self.client.get_account_meta()
 719             keys = [k for k in r if k.startswith(key)]
 720             self.assertFalse(keys)
 721         acc_info = dict(account_info)
 722         acc_info['%sk1' % key] = 'v1'
 723         acc_info['%sk2' % key] = 'v2'
 724         acc_info['%sk3' % key] = 'v3'
 725         with patch.object(PC, 'get_account_info', return_value=acc_info):
 726             r = self.client.get_account_meta()
 727             for k in [k for k in acc_info if k.startswith(key)]:
 728                 self.assertEqual(r[k], acc_info[k])
 729
 730     def test_get_account_group(self):
 731         key = 'x-account-group-'
 732         with patch.object(PC, 'get_account_info', return_value=account_info):
 733             r = self.client.get_account_group()
 734             keys = [k for k in r if k.startswith(key)]
 735             self.assertFalse(keys)
 736         acc_info = dict(account_info)
 737         acc_info['%sk1' % key] = 'g1'
 738         acc_info['%sk2' % key] = 'g2'
 739         acc_info['%sk3' % key] = 'g3'
 740         with patch.object(PC, 'get_account_info', return_value=acc_info):
 741             r = self.client.get_account_group()
 742             for k in [k for k in acc_info if k.startswith(key)]:
 743                 self.assertEqual(r[k], acc_info[k])
 744
 745     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 746     def test_set_account_meta(self, post):
 747         metas = dict(k1='v1', k2='v2', k3='v3')
 748         self.client.set_account_meta(metas)
 749         self.assertEqual(
 750             post.mock_calls[-1],
 751             call(update=True, metadata=metas))
 752
 753     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 754     def test_set_account_quota(self, post):
 755         qu = 1024
 756         self.client.set_account_quota(qu)
 757         self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
 758
 759     @patch('%s.account_post' % pithos_pkg, return_value=FR())
 760     def test_set_account_versioning(self, post):
 761         vrs = 'n3wV3r51on1ngTyp3'
 762         self.client.set_account_versioning(vrs)
 763         self.assertEqual(
 764             post.mock_calls[-1],
 765             call(update=True, versioning=vrs))
 766
 767     @patch('%s.container_delete' % pithos_pkg, return_value=FR())
 768     def test_del_container(self, delete):
 769         for kwarg in (
 770                 dict(delimiter=None, until=None),
 771                 dict(delimiter='X', until='50m3d473')):
 772             self.client.del_container(**kwarg)
 773             expected = dict(kwarg)
 774             expected['success'] = (204, 404, 409)
 775             self.assertEqual(delete.mock_calls[-1], call(**expected))
 776         for status_code in (404, 409):
 777             FR.status_code = status_code
 778             self.assertRaises(ClientError, self.client.del_container)
 779
 780     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 781     def test_get_container_versioning(self, GCI):
 782         key = 'x-container-policy-versioning'
 783         cont = 'c0n7-417'
 784         bu_cnt = self.client.container
 785         for container in (None, cont):
 786             r = self.client.get_container_versioning(container=container)
 787             self.assertEqual(r[key], container_info[key])
 788             self.assertEqual(GCI.mock_calls[-1], call())
 789             self.assertEqual(bu_cnt, self.client.container)
 790
 791     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 792     def test_get_container_quota(self, GCI):
 793         key = 'x-container-policy-quota'
 794         cont = 'c0n7-417'
 795         bu_cnt = self.client.container
 796         for container in (None, cont):
 797             r = self.client.get_container_quota(container=container)
 798             self.assertEqual(r[key], container_info[key])
 799             self.assertEqual(GCI.mock_calls[-1], call())
 800             self.assertEqual(bu_cnt, self.client.container)
 801
 802     def test_get_container_meta(self):
 803         somedate = '50m3d473'
 804         key = 'x-container-meta'
 805         metaval = '50m3m374v41'
 806         container_plus = dict(container_info)
 807         container_plus[key] = metaval
 808         for ret in ((container_info, {}), (container_plus, {key: metaval})):
 809             with patch.object(
 810                     PC,
 811                     'get_container_info',
 812                     return_value=ret[0]) as gci:
 813                 for until in (None, somedate):
 814                     r = self.client.get_container_meta(until=until)
 815                     self.assertEqual(r, ret[1])
 816                     self.assertEqual(gci.mock_calls[-1], call(until=until))
 817
 818     def test_get_container_object_meta(self):
 819         somedate = '50m3d473'
 820         key = 'x-container-object-meta'
 821         metaval = '50m3m374v41'
 822         container_plus = dict(container_info)
 823         container_plus[key] = metaval
 824         for ret in (
 825                 (container_info, {key: ''}),
 826                 (container_plus, {key: metaval})):
 827             with patch.object(
 828                     PC,
 829                     'get_container_info',
 830                     return_value=ret[0]) as gci:
 831                 for until in (None, somedate):
 832                     r = self.client.get_container_object_meta(until=until)
 833                     self.assertEqual(r, ret[1])
 834                     self.assertEqual(gci.mock_calls[-1], call(until=until))
 835
 836     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 837     def test_set_container_meta(self, post):
 838         metas = dict(k1='v1', k2='v2', k3='v3')
 839         self.client.set_container_meta(metas)
 840         self.assertEqual(
 841             post.mock_calls[-1],
 842             call(update=True, metadata=metas))
 843
 844     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 845     def test_del_container_meta(self, ap):
 846         self.client.del_container_meta('somekey')
 847         expected = [call(update=True, metadata={'somekey': ''})]
 848         self.assertEqual(ap.mock_calls, expected)
 849
 850     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 851     def test_set_container_quota(self, post):
 852         qu = 1024
 853         self.client.set_container_quota(qu)
 854         self.assertEqual(post.mock_calls[-1], call(update=True, quota=qu))
 855
 856     @patch('%s.container_post' % pithos_pkg, return_value=FR())
 857     def test_set_container_versioning(self, post):
 858         vrs = 'n3wV3r51on1ngTyp3'
 859         self.client.set_container_versioning(vrs)
 860         self.assertEqual(
 861             post.mock_calls[-1],
 862             call(update=True, versioning=vrs))
 863
 864     @patch('%s.object_delete' % pithos_pkg, return_value=FR())
 865     def test_del_object(self, delete):
 866         for kwarg in (
 867                 dict(delimiter=None, until=None),
 868                 dict(delimiter='X', until='50m3d473')):
 869             self.client.del_object(obj, **kwarg)
 870             self.assertEqual(delete.mock_calls[-1], call(obj, **kwarg))
 871
 872     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 873     def test_set_object_meta(self, post):
 874         metas = dict(k1='v1', k2='v2', k3='v3')
 875         self.assertRaises(
 876             AssertionError,
 877             self.client.set_object_meta,
 878             obj, 'Non dict arg')
 879         self.client.set_object_meta(obj, metas)
 880         self.assertEqual(
 881             post.mock_calls[-1],
 882             call(obj, update=True, metadata=metas))
 883
 884     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 885     def test_publish_object(self, post):
 886         oinfo = dict(object_info)
 887         val = 'pubL1c'
 888         oinfo['x-object-public'] = val
 889         with patch.object(PC, 'get_object_info', return_value=oinfo) as gof:
 890             r = self.client.publish_object(obj)
 891             self.assertEqual(
 892                 post.mock_calls[-1],
 893                 call(obj, public=True, update=True))
 894             self.assertEqual(gof.mock_calls[-1], call(obj))
 895             self.assertEqual(r, '%s%s' % (self.url[:-6], val))
 896
 897     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 898     def test_unpublish_object(self, post):
 899         self.client.unpublish_object(obj)
 900         self.assertEqual(
 901             post.mock_calls[-1],
 902             call(obj, public=False, update=True))
 903
 904     def test_get_object_sharing(self):
 905         info = dict(object_info)
 906         expected = dict(read='u1,g1,u2', write='u1')
 907         info['x-object-sharing'] = '; '.join(
 908             ['%s=%s' % (k, v) for k, v in expected.items()])
 909         with patch.object(PC, 'get_object_info', return_value=info) as GOF:
 910             r = self.client.get_object_sharing(obj)
 911             self.assertEqual(GOF.mock_calls[-1], call(obj))
 912             self.assert_dicts_are_equal(r, expected)
 913             info['x-object-sharing'] = '//'.join(
 914                 ['%s=%s' % (k, v) for k, v in expected.items()])
 915             self.assertRaises(
 916                 ValueError,
 917                 self.client.get_object_sharing,
 918                 obj)
 919             info['x-object-sharing'] = '; '.join(
 920                 ['%s:%s' % (k, v) for k, v in expected.items()])
 921             self.assertRaises(
 922                 ClientError,
 923                 self.client.get_object_sharing,
 924                 obj)
 925             info['x-object-sharing'] = 'read=%s' % expected['read']
 926             r = self.client.get_object_sharing(obj)
 927             expected.pop('write')
 928             self.assert_dicts_are_equal(r, expected)
 929
 930     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 931     def test_set_object_sharing(self, POST):
 932         read_perms = ['u1', 'g1', 'u2', 'g2']
 933         write_perms = ['u1', 'g1']
 934         for kwargs in (
 935                 dict(read_permition=read_perms, write_permition=write_perms),
 936                 dict(read_permition=read_perms),
 937                 dict(write_permition=write_perms),
 938                 dict()):
 939             self.client.set_object_sharing(obj, **kwargs)
 940             kwargs['read'] = kwargs.pop('read_permition', '')
 941             kwargs['write'] = kwargs.pop('write_permition', '')
 942             self.assertEqual(
 943                 POST.mock_calls[-1],
 944                 call(obj, update=True, permissions=kwargs))
 945
 946     @patch('%s.set_object_sharing' % pithos_pkg)
 947     def test_del_object_sharing(self, SOS):
 948         self.client.del_object_sharing(obj)
 949         self.assertEqual(SOS.mock_calls[-1], call(obj))
 950
 951     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
 952     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 953     def test_append_object(self, post, GCI):
 954         num_of_blocks = 4
 955         tmpFile = self._create_temp_file(num_of_blocks)
 956         tmpFile.seek(0, 2)
 957         file_size = tmpFile.tell()
 958         for turn in range(2):
 959             tmpFile.seek(0, 0)
 960
 961             try:
 962                 from progress.bar import ShadyBar
 963                 apn_bar = ShadyBar('Mock append')
 964             except ImportError:
 965                 apn_bar = None
 966
 967             if apn_bar:
 968
 969                 def append_gen(n):
 970                     for i in apn_bar.iter(range(n)):
 971                         yield
 972                     yield
 973
 974             else:
 975                 append_gen = None
 976
 977             self.client.append_object(
 978                 obj, tmpFile,
 979                 upload_cb=append_gen if turn else None)
 980             self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls))
 981             (args, kwargs) = post.mock_calls[-1][1:3]
 982             self.assertEqual(args, (obj,))
 983             self.assertEqual(kwargs['content_length'], len(kwargs['data']))
 984             fsize = num_of_blocks * int(kwargs['content_length'])
 985             self.assertEqual(fsize, file_size)
 986             self.assertEqual(kwargs['content_range'], 'bytes */*')
 987             exp = 'application/octet-stream'
 988             self.assertEqual(kwargs['content_type'], exp)
 989             self.assertEqual(kwargs['update'], True)
 990
 991     @patch('%s.object_post' % pithos_pkg, return_value=FR())
 992     def test_truncate_object(self, post):
 993         upto_bytes = 377
 994         self.client.truncate_object(obj, upto_bytes)
 995         self.assertEqual(post.mock_calls[-1], call(
 996             obj,
 997             update=True,
 998             object_bytes=upto_bytes,
 999             content_range='bytes 0-%s/*' % upto_bytes,
1000             content_type='application/octet-stream',
1001             source_object='/%s/%s' % (self.client.container, obj)))
1002
1003     @patch('%s.get_container_info' % pithos_pkg, return_value=container_info)
1004     @patch('%s.object_post' % pithos_pkg, return_value=FR())
1005     def test_overwrite_object(self, post, GCI):
1006         num_of_blocks = 4
1007         tmpFile = self._create_temp_file(num_of_blocks)
1008         tmpFile.seek(0, 2)
1009         file_size = tmpFile.tell()
1010         info = dict(object_info)
1011         info['content-length'] = file_size
1012         block_size = container_info['x-container-block-size']
1013         with patch.object(PC, 'get_object_info', return_value=info) as GOI:
1014             for start, end in (
1015                     (0, file_size + 1),
1016                     (file_size + 1, file_size + 2)):
1017                 tmpFile.seek(0, 0)
1018                 self.assertRaises(
1019                     ClientError,
1020                     self.client.overwrite_object,
1021                     obj, start, end, tmpFile)
1022             for start, end in ((0, 144), (144, 233), (233, file_size)):
1023                 tmpFile.seek(0, 0)
1024                 owr_gen = None
1025                 exp_size = end - start + 1
1026                 if not start or exp_size > block_size:
1027                     try:
1028                         from progress.bar import ShadyBar
1029                         owr_bar = ShadyBar('Mock append')
1030                     except ImportError:
1031                         owr_bar = None
1032
1033                     if owr_bar:
1034
1035                         def owr_gen(n):
1036                             for i in owr_bar.iter(range(n)):
1037                                 yield
1038                             yield
1039
1040                     if exp_size > block_size:
1041                         exp_size = exp_size % block_size or block_size
1042
1043                 self.client.overwrite_object(obj, start, end, tmpFile, owr_gen)
1044                 self.assertEqual(GOI.mock_calls[-1], call(obj))
1045                 self.assertEqual(GCI.mock_calls[-1], call())
1046                 (args, kwargs) = post.mock_calls[-1][1:3]
1047                 self.assertEqual(args, (obj,))
1048                 self.assertEqual(len(kwargs['data']), exp_size)
1049                 self.assertEqual(kwargs['content_length'], exp_size)
1050                 self.assertEqual(kwargs['update'], True)
1051                 exp = 'application/octet-stream'
1052                 self.assertEqual(kwargs['content_type'], exp)
1053
1054     @patch('%s.set_param' % client_pkg)
1055     @patch('%s.get' % pithos_pkg, return_value=FR())
1056     def test_get_sharing_accounts(self, get, SP):
1057         FR.json = sharers
1058         for kws in (
1059                 dict(),
1060                 dict(limit='50m3-11m17'),
1061                 dict(marker='X'),
1062                 dict(limit='50m3-11m17', marker='X')):
1063             r = self.client.get_sharing_accounts(**kws)
1064             self.assertEqual(SP.mock_calls[-3], call('format', 'json'))
1065             limit, marker = kws.get('limit', None), kws.get('marker', None)
1066             self.assertEqual(SP.mock_calls[-2], call(
1067                 'limit', limit,
1068                 iff=limit is not None))
1069             self.assertEqual(SP.mock_calls[-1], call(
1070                 'marker', marker,
1071                 iff=marker is not None))
1072             self.assertEqual(get.mock_calls[-1], call('', success=(200, 204)))
1073             for i in range(len(r)):
1074                 self.assert_dicts_are_equal(r[i], sharers[i])
1075
1076     @patch('%s.object_get' % pithos_pkg, return_value=FR())
1077     def test_get_object_versionlist(self, get):
1078         info = dict(object_info)
1079         info['versions'] = ['v1', 'v2']
1080         FR.json = info
1081         r = self.client.get_object_versionlist(obj)
1082         self.assertEqual(
1083             get.mock_calls[-1],
1084             call(obj, format='json', version='list'))
1085         self.assertEqual(r, info['versions'])
1086
1087 if __name__ == '__main__':
1088     from sys import argv
1089     from kamaki.clients.test import runTestCase
1090     runTestCase(Pithos, 'Pithos+ Client', argv[1:])