code.grnet.gr Git - kamaki/blob - kamaki/clients/test/pithos.py

   1 # Copyright 2013 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from unittest import TestCase
  35 from mock import patch, call, Mock
  36
  37 from kamaki.clients import ClientError
  38 from kamaki.clients.pithos import PithosClient as PC
  39 from kamaki.clients.astakos import AstakosClient
  40 from kamaki.clients.connection.kamakicon import KamakiHTTPConnection as C
  41
  42 user_id = 'ac0un7-1d-5tr1ng'
  43
  44 account_info = {
  45     'content-language': 'en-us',
  46     'content-type': 'text/html; charset=utf-8',
  47     'date': 'Wed, 06 Mar 2013 13:25:51 GMT',
  48     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  49     'server': 'gunicorn/0.14.5',
  50     'vary': 'Accept-Language',
  51     'x-account-bytes-used': '751615526',
  52     'x-account-container-count': 7,
  53     'x-account-policy-quota': 53687091200,
  54     'x-account-policy-versioning': 'auto'}
  55 container_info = {
  56     'content-language': 'en-us',
  57     'content-type': 'text/html; charset=utf-8',
  58     'date': 'Wed, 06 Mar 2013 15:11:05 GMT',
  59     'last-modified': 'Wed, 27 Feb 2013 15:56:13 GMT',
  60     'server': 'gunicorn/0.14.5',
  61     'vary': 'Accept-Language',
  62     'x-container-block-hash': 'sha256',
  63     'x-container-block-size': 4194304,
  64     'x-container-bytes-used': 309528938,
  65     'x-container-object-count': 14,
  66     'x-container-object-meta': '',
  67     'x-container-policy-quota': 53687091200,
  68     'x-container-policy-versioning': 'auto'}
  69 object_info = {
  70     'content-language': 'en-us',
  71     'content-length': 254965,
  72     'content-type': 'application/octet-stream',
  73     'date': 'Thu, 07 Mar 2013 13:27:43 GMT',
  74     'etag': '',
  75     'last-modified': 'Mon, 04 Mar 2013 18:22:31 GMT',
  76     'server': 'gunicorn/0.14.5',
  77     'vary': 'Accept-Language',
  78     'x-object-hash': 'obj3c7h45h1s0bj3c7h45h411r34dY',
  79     'x-object-uuid': 'd0c747ca-34bd-49e0-8e98-1d07d8b0cbc7',
  80     'x-object-version': '525996',
  81     'x-object-version-timestamp': 'Mon, 04 Mar 2013 18:22:31 GMT',
  82     'x-object-meta-k1': 'v1',
  83     'x-object-meta-k2': 'v2'}
  84 container_list = [
  85     dict(
  86         count=2,
  87         last_modified="2013-02-27T11:56:09.893033+00:00",
  88         bytes=677076979,
  89         name="pithos",
  90         x_container_policy=dict(quota="21474836480", versioning="auto")),
  91     dict(
  92         count=0,
  93         last_modified="2012-10-23T12:25:17.229187+00:00",
  94         bytes=0,
  95         name="trash",
  96         x_container_policy=dict(quota="21474836480", versioning="auto"))]
  97
  98
  99 class Pithos(TestCase):
 100
 101     class FR(object):
 102         """FR stands for Fake Response"""
 103         json = dict()
 104         headers = dict()
 105         content = json
 106         status = None
 107         status_code = 200
 108
 109         def release(self):
 110             pass
 111
 112     files = []
 113
 114     def assert_dicts_are_equal(self, d1, d2):
 115         for k, v in d1.items():
 116             self.assertTrue(k in d2)
 117             if isinstance(v, dict):
 118                 self.assert_dicts_are_equal(v, d2[k])
 119             else:
 120                 self.assertEqual(unicode(v), unicode(d2[k]))
 121
 122     def setUp(self):
 123         self.url = 'https://www.example.com/pithos'
 124         self.token = 'p17h0570k3n'
 125         self.client = PC(self.url, self.token)
 126         self.client.account = user_id
 127         self.client.container = 'c0nt@1n3r_i'
 128
 129     def tearDown(self):
 130         self.FR.headers = dict()
 131         self.FR.status_code = 200
 132         self.FR.json = dict()
 133         for f in self.files:
 134             f.close()
 135
 136     def test_get_account_info(self):
 137         self.FR.headers = account_info
 138         self.FR.status_code = 204
 139         with patch.object(C, 'perform_request', return_value=self.FR()):
 140             r = self.client.get_account_info()
 141             self.assertEqual(self.client.http_client.url, self.url)
 142             self.assertEqual(self.client.http_client.path, '/%s' % user_id)
 143             self.assert_dicts_are_equal(r, account_info)
 144             PC.set_param = Mock()
 145             untils = ['date 1', 'date 2', 'date 3']
 146             for unt in untils:
 147                 r = self.client.get_account_info(until=unt)
 148                 self.assert_dicts_are_equal(r, account_info)
 149             for i in range(len(untils)):
 150                 self.assertEqual(
 151                     PC.set_param.mock_calls[i],
 152                     call('until', untils[i], iff=untils[i]))
 153             self.FR.status_code = 401
 154             self.assertRaises(ClientError, self.client.get_account_info)
 155
 156     def test_replace_account_meta(self):
 157         self.FR.status_code = 202
 158         metas = dict(k1='v1', k2='v2', k3='v3')
 159         PC.set_header = Mock()
 160         with patch.object(C, 'perform_request', return_value=self.FR()):
 161             self.client.replace_account_meta(metas)
 162             self.assertEqual(self.client.http_client.url, self.url)
 163             self.assertEqual(self.client.http_client.path, '/%s' % user_id)
 164             prfx = 'X-Account-Meta-'
 165             expected = [call('%s%s' % (prfx, k), v) for k, v in metas.items()]
 166             self.assertEqual(PC.set_header.mock_calls, expected)
 167
 168     def test_del_account_meta(self):
 169         keys = ['k1', 'k2', 'k3']
 170         with patch.object(PC, 'account_post', return_value=self.FR()) as ap:
 171             expected = []
 172             for key in keys:
 173                 self.client.del_account_meta(key)
 174                 expected.append(call(update=True, metadata={key: ''}))
 175             self.assertEqual(ap.mock_calls, expected)
 176
 177     def test_create_container(self):
 178         self.FR.status_code = 201
 179         with patch.object(PC, 'put', return_value=self.FR()) as put:
 180             cont = 's0m3c0n731n3r'
 181             self.client.create_container(cont)
 182             expected = [call('/%s/%s' % (user_id, cont), success=(201, 202))]
 183             self.assertEqual(put.mock_calls, expected)
 184             self.FR.status_code = 202
 185             self.assertRaises(ClientError, self.client.create_container, cont)
 186
 187     def test_get_container_info(self):
 188         self.FR.headers = container_info
 189         with patch.object(PC, 'container_head', return_value=self.FR()) as ch:
 190             r = self.client.get_container_info()
 191             self.assert_dicts_are_equal(r, container_info)
 192             u = 'some date'
 193             r = self.client.get_container_info(until=u)
 194             self.assertEqual(ch.mock_calls, [call(until=None), call(until=u)])
 195
 196     def test_delete_container(self):
 197         self.FR.status_code = 204
 198         with patch.object(PC, 'delete', return_value=self.FR()) as delete:
 199             cont = 's0m3c0n731n3r'
 200             self.client.delete_container(cont)
 201             self.FR.status_code = 404
 202             self.assertRaises(ClientError, self.client.delete_container, cont)
 203             self.FR.status_code = 409
 204             self.assertRaises(ClientError, self.client.delete_container, cont)
 205             acall = call('/%s/%s' % (user_id, cont), success=(204, 404, 409))
 206             self.assertEqual(delete.mock_calls, [acall] * 3)
 207
 208     def test_list_containers(self):
 209         self.FR.json = container_list
 210         with patch.object(PC, 'account_get', return_value=self.FR()):
 211             r = self.client.list_containers()
 212             for i in range(len(r)):
 213                 self.assert_dicts_are_equal(r[i], container_list[i])
 214
 215     def test_upload_object(self):
 216         PC.get_container_info = Mock(return_value=container_info)
 217         PC.container_post = Mock(return_value=self.FR())
 218         PC.object_put = Mock(return_value=self.FR())
 219         from tempfile import NamedTemporaryFile
 220         from os import urandom
 221         self.files.append(NamedTemporaryFile())
 222         tmpFile = self.files[-1]
 223         num_of_blocks = 8
 224         file_size = num_of_blocks * 4 * 1024 * 1024
 225         print('\n\tCreate tmp file')
 226         tmpFile.write(urandom(file_size))
 227         tmpFile.flush()
 228         tmpFile.seek(0)
 229         print('\t\tDone')
 230         obj = 'objectName'
 231
 232         # No special args
 233         self.client.upload_object(obj, tmpFile)
 234         self.assertEqual(PC.get_container_info.mock_calls, [call()])
 235         [call1, call2] = PC.object_put.mock_calls
 236
 237         (args1, kwargs1) = call1[1:3]
 238         (args2, kwargs2) = call2[1:3]
 239         self.assertEqual(args1, (obj,))
 240         expected1 = dict(
 241             hashmap=True,
 242             success=(201, 409),
 243             format='json',
 244             json=dict(
 245                 hashes=['s0m3h@5h'] * num_of_blocks,
 246                 bytes=file_size),
 247             etag=None,
 248             content_encoding=None,
 249             content_type='application/octet-stream',
 250             content_disposition=None,
 251             public=None,
 252             permissions=None)
 253         for k, v in expected1.items():
 254             if k == 'json':
 255                 self.assertEqual(len(v['hashes']), len(kwargs1[k]['hashes']))
 256                 self.assertEqual(v['bytes'], kwargs1[k]['bytes'])
 257             else:
 258                 self.assertEqual(v, kwargs1[k])
 259
 260         (args2, kwargs2) = call2[1:3]
 261         self.assertEqual(args2, (obj,))
 262         expected2 = dict(
 263             json=dict(
 264                 hashes=['s0m3h@5h'] * num_of_blocks,
 265                 bytes=file_size),
 266             content_type='application/octet-stream',
 267             hashmap=True,
 268             success=201,
 269             format='json')
 270         for k, v in expected2.items():
 271             if k == 'json':
 272                 self.assertEqual(len(v['hashes']), len(kwargs2[k]['hashes']))
 273                 self.assertEqual(v['bytes'], kwargs2[k]['bytes'])
 274             else:
 275                 self.assertEqual(v, kwargs2[k])
 276
 277         OP = PC.object_put
 278         mock_offset = 2
 279
 280         #  With progress bars
 281         try:
 282             from progress.bar import ShadyBar
 283             blck_bar = ShadyBar('Mock blck calc.')
 284             upld_bar = ShadyBar('Mock uplds')
 285         except ImportError:
 286             blck_bar = None
 287             upld_bar = None
 288
 289         if blck_bar and upld_bar:
 290
 291             def blck_gen(n):
 292                 for i in blck_bar.iter(range(n)):
 293                     yield
 294                 yield
 295
 296             def upld_gen(n):
 297                 for i in upld_bar.iter(range(n)):
 298                     yield
 299                 yield
 300
 301             tmpFile.seek(0)
 302             self.client.upload_object(
 303                 obj, tmpFile,
 304                 hash_cb=blck_gen, upload_cb=upld_gen)
 305
 306             for i, c in enumerate(OP.mock_calls[-mock_offset:]):
 307                 self.assertEqual(OP.mock_calls[i], c)
 308
 309         #  With content-type
 310         tmpFile.seek(0)
 311         ctype = 'video/mpeg'
 312         sharing = dict(read=['u1', 'g1', 'u2'], write=['u1'])
 313         self.client.upload_object(obj, tmpFile,
 314             content_type=ctype, sharing=sharing)
 315         self.assertEqual(OP.mock_calls[-1][2]['content_type'], ctype)
 316         self.assert_dicts_are_equal(
 317             OP.mock_calls[-2][2]['permissions'],
 318             sharing)
 319
 320         # With other args
 321         tmpFile.seek(0)
 322         kwargs = dict(
 323             etag='s0m3E74g',
 324             content_type=ctype,
 325             content_disposition=ctype + 'd15p051710n',
 326             public=True,
 327             content_encoding='802.11')
 328         self.client.upload_object(obj, tmpFile, **kwargs)
 329         for arg, val in kwargs.items():
 330             self.assertEqual(OP.mock_calls[-2][2][arg], val)
 331
 332     def test_create_object(self):
 333         PC.set_header = Mock()
 334         obj = 'r4nd0m0bj3c7'
 335         cont = self.client.container
 336         ctype = 'c0n73n7/typ3'
 337         exp_shd = [
 338             call('Content-Type', 'application/octet-stream'),
 339             call('Content-length', '0'),
 340             call('Content-Type', ctype), call('Content-length', '42')]
 341         exp_put = [call('/%s/%s/%s' % (user_id, cont, obj), success=201)] * 2
 342         with patch.object(PC, 'put', return_value=self.FR()) as put:
 343             self.client.create_object(obj)
 344             self.client.create_object(obj,
 345                 content_type=ctype, content_length=42)
 346             self.assertEqual(PC.set_header.mock_calls, exp_shd)
 347             self.assertEqual(put.mock_calls, exp_put)
 348
 349     def test_create_directory(self):
 350         PC.set_header = Mock()
 351         obj = 'r4nd0m0bj3c7'
 352         cont = self.client.container
 353         exp_shd = [
 354             call('Content-Type', 'application/directory'),
 355             call('Content-length', '0')]
 356         exp_put = [call('/%s/%s/%s' % (user_id, cont, obj), success=201)]
 357         with patch.object(PC, 'put', return_value=self.FR()) as put:
 358             self.client.create_directory(obj)
 359             self.assertEqual(PC.set_header.mock_calls, exp_shd)
 360             self.assertEqual(put.mock_calls, exp_put)
 361
 362     def test_get_object_info(self):
 363         self.FR.headers = object_info
 364         obj = 'r4nd0m0bj3c7'
 365         version = 'v3r510n'
 366         with patch.object(PC, 'object_head', return_value=self.FR()) as head:
 367             r = self.client.get_object_info(obj)
 368             self.assertEqual(r, object_info)
 369             r = self.client.get_object_info(obj, version=version)
 370             self.assertEqual(head.mock_calls, [
 371                 call(obj, version=None),
 372                 call(obj, version=version)])
 373         with patch.object(
 374                 PC,
 375                 'object_head',
 376                 side_effect=ClientError('Obj not found', 404)):
 377             self.assertRaises(
 378                 ClientError,
 379                 self.client.get_object_info,
 380                 obj, version=version)
 381
 382     def test_get_object_meta(self):
 383         obj = 'r4nd0m0bj3c7'
 384         expected = dict()
 385         for k, v in object_info.items():
 386             expected[k] = v
 387         with patch.object(
 388                 PC,
 389                 'get_object_info',
 390                 return_value=object_info):
 391             r = self.client.get_object_meta(obj)
 392             self.assert_dicts_are_equal(r, expected)
 393
 394     def test_del_object_meta(self):
 395         obj = 'r4nd0m0bj3c7'
 396         metakey = '50m3m3t4k3y'
 397         with patch.object(PC, 'object_post', return_value=self.FR()) as post:
 398             self.client.del_object_meta(obj, metakey)
 399             self.assertEqual(
 400                 post.mock_calls,
 401                 [call(obj, update=True, metadata={metakey: ''})])
 402
 403     def test_replace_object_meta(self):
 404         PC.set_header = Mock()
 405         metas = dict(k1='new1', k2='new2', k3='new3')
 406         cont = self.client.container
 407         with patch.object(PC, 'post', return_value=self.FR()) as post:
 408             self.client.replace_object_meta(metas)
 409             self.assertEqual(post.mock_calls, [
 410                 call('/%s/%s' % (user_id, cont),
 411                 success=202)])
 412             prfx = 'X-Object-Meta-'
 413             expected = [call('%s%s' % (prfx, k), v) for k, v in metas.items()]
 414             self.assertEqual(PC.set_header.mock_calls, expected)
 415
 416     def test_copy_object(self):
 417         src_cont = 'src-c0nt41n3r'
 418         src_obj = 'src-0bj'
 419         dst_cont = 'dst-c0nt41n3r'
 420         dst_obj = 'dst-0bj'
 421         expected = call(
 422             src_obj,
 423             content_length=0,
 424             source_account=None,
 425             success=201,
 426             copy_from='/%s/%s' % (src_cont, src_obj),
 427             delimiter=None,
 428             content_type=None,
 429             source_version=None,
 430             public=False)
 431         with patch.object(PC, 'object_put', return_value=self.FR()) as put:
 432             self.client.copy_object(src_cont, src_obj, dst_cont)
 433             self.assertEqual(put.mock_calls[-1], expected)
 434             self.client.copy_object(src_cont, src_obj, dst_cont, dst_obj)
 435             self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 436             kwargs = dict(
 437                 source_version='src-v3r510n',
 438                 source_account='src-4cc0un7',
 439                 public=True,
 440                 content_type='c0n73n7Typ3',
 441                 delimiter='5')
 442             self.client.copy_object(src_cont, src_obj, dst_cont, **kwargs)
 443             for k, v in kwargs.items():
 444                 self.assertEqual(v, put.mock_calls[-1][2][k])
 445
 446     def test_move_object(self):
 447         src_cont = 'src-c0nt41n3r'
 448         src_obj = 'src-0bj'
 449         dst_cont = 'dst-c0nt41n3r'
 450         dst_obj = 'dst-0bj'
 451         expected = call(
 452             src_obj,
 453             content_length=0,
 454             source_account=None,
 455             success=201,
 456             move_from='/%s/%s' % (src_cont, src_obj),
 457             delimiter=None,
 458             content_type=None,
 459             source_version=None,
 460             public=False)
 461         with patch.object(PC, 'object_put', return_value=self.FR()) as put:
 462             self.client.move_object(src_cont, src_obj, dst_cont)
 463             self.assertEqual(put.mock_calls[-1], expected)
 464             self.client.move_object(src_cont, src_obj, dst_cont, dst_obj)
 465             self.assertEqual(put.mock_calls[-1][1], (dst_obj,))
 466             kwargs = dict(
 467                 source_version='src-v3r510n',
 468                 source_account='src-4cc0un7',
 469                 public=True,
 470                 content_type='c0n73n7Typ3',
 471                 delimiter='5')
 472             self.client.move_object(src_cont, src_obj, dst_cont, **kwargs)
 473             for k, v in kwargs.items():
 474                 self.assertEqual(v, put.mock_calls[-1][2][k])