code.grnet.gr Git - pithos/blob - tools/test

   1 #!/usr/bin/env python
   2 #coding=utf8
   3
   4 # Copyright 2011 GRNET S.A. All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or
   7 # without modification, are permitted provided that the following
   8 # conditions are met:
   9 #
  10 #   1. Redistributions of source code must retain the above
  11 #      copyright notice, this list of conditions and the following
  12 #      disclaimer.
  13 #
  14 #   2. Redistributions in binary form must reproduce the above
  15 #      copyright notice, this list of conditions and the following
  16 #      disclaimer in the documentation and/or other materials
  17 #      provided with the distribution.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  20 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  23 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  26 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  27 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  29 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 # POSSIBILITY OF SUCH DAMAGE.
  31 #
  32 # The views and conclusions contained in the software and
  33 # documentation are those of the authors and should not be
  34 # interpreted as representing official policies, either expressed
  35 # or implied, of GRNET S.A.
  36
  37 from lib.client import Pithos_Client, Fault
  38 from lib.util import get_user, get_auth, get_server, get_api
  39 from xml.dom import minidom
  40 from StringIO import StringIO
  41 import json
  42 import unittest
  43 import time as _time
  44 import types
  45 import hashlib
  46 import mimetypes
  47 import random
  48 import datetime
  49 import string
  50 import re
  51
  52 DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
  53                 "%A, %d-%b-%y %H:%M:%S GMT",
  54                 "%a, %d %b %Y %H:%M:%S GMT"]
  55
  56 OTHER_ACCOUNTS = {
  57     '0001': 'verigak',
  58     '0002': 'chazapis',
  59     '0003': 'gtsouk',
  60     '0004': 'papagian',
  61     '0005': 'louridas',
  62     '0006': 'chstath',
  63     '0007': 'pkanavos',
  64     '0008': 'mvasilak',
  65     '0009': 'διογένης'}
  66
  67 class BaseTestCase(unittest.TestCase):
  68     #TODO unauthorized request
  69     def setUp(self):
  70         self.client = Pithos_Client(get_server(), get_auth(), get_user(),
  71                                     get_api())
  72         self.invalid_client = Pithos_Client(get_server(), get_auth(), 'invalid',
  73                                             get_api())
  74         #self.headers = {
  75         #    'account': ('x-account-container-count',
  76         #                'x-account-bytes-used',
  77         #                'last-modified',
  78         #                'content-length',
  79         #                'date',
  80         #                'content_type',
  81         #                'server',),
  82         #    'object': ('etag',
  83         #               'content-length',
  84         #               'content_type',
  85         #               'content-encoding',
  86         #               'last-modified',
  87         #               'date',
  88         #               'x-object-manifest',
  89         #               'content-range',
  90         #               'x-object-modified-by',
  91         #               'x-object-version',
  92         #               'x-object-version-timestamp',
  93         #               'server',),
  94         #    'container': ('x-container-object-count',
  95         #                  'x-container-bytes-used',
  96         #                  'content_type',
  97         #                  'last-modified',
  98         #                  'content-length',
  99         #                  'date',
 100         #                  'x-container-block-size',
 101         #                  'x-container-block-hash',
 102         #                  'x-container-policy-quota',
 103         #                  'x-container-policy-versioning',
 104         #                  'server',
 105         #                  'x-container-object-meta',
 106         #                  'x-container-policy-versioning',
 107         #                  'server',)}
 108         #
 109         #self.contentTypes = {'xml':'application/xml',
 110         #                     'json':'application/json',
 111         #                     '':'text/plain'}
 112         self.extended = {
 113             'container':(
 114                 'name',
 115                 'count',
 116                 'bytes',
 117                 'last_modified',
 118                 'x_container_policy_quota',
 119                 'x_container_policy_versioning',),
 120             'object':(
 121                 'name',
 122                 'hash',
 123                 'bytes',
 124                 'content_type',
 125                 'content_encoding',
 126                 'last_modified',)}
 127         self.return_codes = (400, 401, 404, 503,)
 128
 129     def tearDown(self):
 130         for c in self.client.list_containers():
 131             while True:
 132                 #list objects returns at most 10000 objects
 133                 #so repeat until there are no more objects
 134                 objects = self.client.list_objects(c)
 135                 if not objects:
 136                     break
 137                 for o in objects:
 138                     self.client.delete_object(c, o)
 139             self.client.delete_container(c)
 140
 141     def assert_status(self, status, codes):
 142         l = [elem for elem in self.return_codes]
 143         if type(codes) == types.ListType:
 144             l.extend(codes)
 145         else:
 146             l.append(codes)
 147         self.assertTrue(status in l)
 148
 149     #def assert_headers(self, headers, type, **exp_meta):
 150     #    prefix = 'x-%s-meta-' %type
 151     #    system_headers = [h for h in headers if not h.startswith(prefix)]
 152     #    for k,v in headers.items():
 153     #        if k in system_headers:
 154     #            self.assertTrue(k in headers[type])
 155     #        elif exp_meta:
 156     #            k = k.split(prefix)[-1]
 157     #            self.assertEqual(v, exp_meta[k])
 158
 159     def assert_extended(self, data, format, type, size=10000):
 160         if format == 'xml':
 161             self._assert_xml(data, type, size)
 162         elif format == 'json':
 163             self._assert_json(data, type, size)
 164
 165     def _assert_json(self, data, type, size):
 166         convert = lambda s: s.lower()
 167         info = [convert(elem) for elem in self.extended[type]]
 168         self.assertTrue(len(data) <= size)
 169         for item in info:
 170             for i in data:
 171                 if 'subdir' in i.keys():
 172                     continue
 173                 self.assertTrue(item in i.keys())
 174
 175     def _assert_xml(self, data, type, size):
 176         convert = lambda s: s.lower()
 177         info = [convert(elem) for elem in self.extended[type]]
 178         try:
 179             info.remove('content_encoding')
 180         except ValueError:
 181             pass
 182         xml = data
 183         entities = xml.getElementsByTagName(type)
 184         self.assertTrue(len(entities) <= size)
 185         for e in entities:
 186             for item in info:
 187                 self.assertTrue(e.getElementsByTagName(item))
 188
 189     def assert_raises_fault(self, status, callableObj, *args, **kwargs):
 190         """
 191         asserts that a Fault with a specific status is raised
 192         when callableObj is called with the specific arguments
 193         """
 194         try:
 195             r = callableObj(*args, **kwargs)
 196             self.fail('Should never reach here')
 197         except Fault, f:
 198             self.failUnless(f.status == status)
 199
 200     def assert_not_raises_fault(self, status, callableObj, *args, **kwargs):
 201         """
 202         asserts that a Fault with a specific status is not raised
 203         when callableObj is called with the specific arguments
 204         """
 205         try:
 206             r = callableObj(*args, **kwargs)
 207         except Fault, f:
 208             self.failIfEqual(f.status, status)
 209
 210     def assert_container_exists(self, container):
 211         """
 212         asserts the existence of a container
 213         """
 214         try:
 215             self.client.retrieve_container_metadata(container)
 216         except Fault, f:
 217             self.failIf(f.status == 404)
 218
 219     def assert_container_not_exists(self, container):
 220         """
 221         asserts there is no such a container
 222         """
 223         self.assert_raises_fault(404, self.client.retrieve_container_metadata,
 224                                  container)
 225
 226     def assert_object_exists(self, container, object):
 227         """
 228         asserts the existence of an object
 229         """
 230         try:
 231             self.client.retrieve_object_metadata(container, object)
 232         except Fault, f:
 233             self.failIf(f.status == 404)
 234
 235     def assert_object_not_exists(self, container, object):
 236         """
 237         asserts there is no such an object
 238         """
 239         self.assert_raises_fault(404, self.client.retrieve_object_metadata,
 240                                  container, object)
 241
 242     def upload_random_data(self, container, name, length=1024, type=None,
 243                            enc=None, **meta):
 244         data = get_random_data(length)
 245         return self.upload_data(container, name, data, type, enc, **meta)
 246
 247     def upload_data(self, container, name, data, type=None, enc=None, etag=None,
 248                     **meta):
 249         obj = {}
 250         obj['name'] = name
 251         try:
 252             obj['data'] = data
 253             obj['hash'] = compute_md5_hash(obj['data'])
 254
 255             args = {}
 256             args['etag'] = etag if etag else obj['hash']
 257
 258             try:
 259                 guess = mimetypes.guess_type(name)
 260                 type = type if type else guess[0]
 261                 enc = enc if enc else guess[1]
 262             except:
 263                 pass
 264             args['content_type'] = type if type else 'plain/text'
 265             args['content_encoding'] = enc if enc else None
 266
 267             obj['meta'] = args
 268
 269             path = '/%s/%s' % (container, name)
 270             self.client.create_object(container, name, StringIO(obj['data']),
 271                                       meta, **args)
 272
 273             return obj
 274         except IOError:
 275             return
 276
 277 class AccountHead(BaseTestCase):
 278     def setUp(self):
 279         BaseTestCase.setUp(self)
 280         self.account = 'test'
 281         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 282         for item in self.containers:
 283             self.client.create_container(item)
 284
 285                 #keep track of initial account groups
 286         self.initial_groups = self.client.retrieve_account_groups()
 287
 288         #keep track of initial account meta
 289         self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
 290
 291         meta = {'foo':'bar'}
 292         self.client.update_account_metadata(**meta)
 293         self.updated_meta = self.initial_meta.update(meta)
 294
 295     def tearDown(self):
 296         #delete additionally created meta
 297         l = []
 298         for m in self.client.retrieve_account_metadata(restricted=True):
 299             if m not in self.initial_meta:
 300                 l.append(m)
 301         self.client.delete_account_metadata(l)
 302
 303         #delete additionally created groups
 304         l = []
 305         for g in self.client.retrieve_account_groups():
 306             if g not in self.initial_groups:
 307                 l.append(g)
 308         self.client.unset_account_groups(l)
 309
 310         BaseTestCase.tearDown(self)
 311
 312     def test_get_account_meta(self):
 313         meta = self.client.retrieve_account_metadata()
 314
 315         containers = self.client.list_containers()
 316         l = str(len(containers))
 317         self.assertEqual(meta['x-account-container-count'], l)
 318         size = 0
 319         for c in containers:
 320             m = self.client.retrieve_container_metadata(c)
 321             size = size + int(m['x-container-bytes-used'])
 322         self.assertEqual(meta['x-account-bytes-used'], str(size))
 323
 324     def test_get_account_401(self):
 325         self.assert_raises_fault(401,
 326                                  self.invalid_client.retrieve_account_metadata)
 327
 328     def test_get_account_meta_until(self):
 329         t = datetime.datetime.utcnow()
 330         past = t - datetime.timedelta(minutes=-15)
 331         past = int(_time.mktime(past.timetuple()))
 332
 333         meta = {'premium':True}
 334         self.client.update_account_metadata(**meta)
 335         meta = self.client.retrieve_account_metadata(restricted=True,
 336                                                      until=past)
 337         self.assertTrue('premium' not in meta)
 338
 339         meta = self.client.retrieve_account_metadata(restricted=True)
 340         self.assertTrue('premium' in meta)
 341
 342     def test_get_account_meta_until_invalid_date(self):
 343         meta = {'premium':True}
 344         self.client.update_account_metadata(**meta)
 345         meta = self.client.retrieve_account_metadata(restricted=True,
 346                                                      until='kshfksfh')
 347         self.assertTrue('premium' in meta)
 348
 349 class AccountGet(BaseTestCase):
 350     def setUp(self):
 351         BaseTestCase.setUp(self)
 352         self.account = 'test'
 353         #create some containers
 354         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 355         for item in self.containers:
 356             self.client.create_container(item)
 357
 358     def test_list(self):
 359         #list containers
 360         containers = self.client.list_containers()
 361         self.assertEquals(self.containers, containers)
 362
 363     def test_list_401(self):
 364         self.assert_raises_fault(401, self.invalid_client.list_containers)
 365
 366     def test_list_with_limit(self):
 367         limit = 2
 368         containers = self.client.list_containers(limit=limit)
 369         self.assertEquals(len(containers), limit)
 370         self.assertEquals(self.containers[:2], containers)
 371
 372     def test_list_with_marker(self):
 373         l = 2
 374         m = 'bananas'
 375         containers = self.client.list_containers(limit=l, marker=m)
 376         i = self.containers.index(m) + 1
 377         self.assertEquals(self.containers[i:(i+l)], containers)
 378
 379         m = 'oranges'
 380         containers = self.client.list_containers(limit=l, marker=m)
 381         i = self.containers.index(m) + 1
 382         self.assertEquals(self.containers[i:(i+l)], containers)
 383
 384     def test_list_json_with_marker(self):
 385         l = 2
 386         m = 'bananas'
 387         containers = self.client.list_containers(limit=l, marker=m, format='json')
 388         self.assert_extended(containers, 'json', 'container', l)
 389         self.assertEqual(containers[0]['name'], 'kiwis')
 390         self.assertEqual(containers[1]['name'], 'oranges')
 391
 392     def test_list_xml_with_marker(self):
 393         l = 2
 394         m = 'oranges'
 395         xml = self.client.list_containers(limit=l, marker=m, format='xml')
 396         self.assert_extended(xml, 'xml', 'container', l)
 397         nodes = xml.getElementsByTagName('name')
 398         self.assertEqual(len(nodes), 1)
 399         self.assertEqual(nodes[0].childNodes[0].data, 'pears')
 400
 401     def test_if_modified_since(self):
 402         t = datetime.datetime.utcnow()
 403         t2 = t - datetime.timedelta(minutes=10)
 404
 405         #add a new container
 406         self.client.create_container('dummy')
 407
 408         for f in DATE_FORMATS:
 409             past = t2.strftime(f)
 410             try:
 411                 c = self.client.list_containers(if_modified_since=past)
 412                 self.assertEqual(len(c), len(self.containers) + 1)
 413             except Fault, f:
 414                 self.failIf(f.status == 304) #fail if not modified
 415
 416     def test_if_modified_since_invalid_date(self):
 417         c = self.client.list_containers(if_modified_since='')
 418         self.assertEqual(len(c), len(self.containers))
 419
 420     def test_if_not_modified_since(self):
 421         now = datetime.datetime.utcnow()
 422         since = now + datetime.timedelta(1)
 423
 424         for f in DATE_FORMATS:
 425             args = {'if_modified_since':'%s' %since.strftime(f)}
 426
 427             #assert not modified
 428             self.assert_raises_fault(304, self.client.list_containers, **args)
 429
 430     def test_if_unmodified_since(self):
 431         now = datetime.datetime.utcnow()
 432         since = now + datetime.timedelta(1)
 433
 434         for f in DATE_FORMATS:
 435             c = self.client.list_containers(if_unmodified_since=since.strftime(f))
 436
 437             #assert success
 438             self.assertEqual(self.containers, c)
 439
 440     def test_if_unmodified_since_precondition_failed(self):
 441         t = datetime.datetime.utcnow()
 442         t2 = t - datetime.timedelta(minutes=10)
 443
 444         #add a new container
 445         self.client.create_container('dummy')
 446
 447         for f in DATE_FORMATS:
 448             past = t2.strftime(f)
 449
 450             args = {'if_unmodified_since':'%s' %past}
 451
 452             #assert precondition failed
 453             self.assert_raises_fault(412, self.client.list_containers, **args)
 454
 455 class AccountPost(BaseTestCase):
 456     def setUp(self):
 457         BaseTestCase.setUp(self)
 458         self.account = 'test'
 459         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 460         for item in self.containers:
 461             self.client.create_container(item)
 462
 463         #keep track of initial account groups
 464         self.initial_groups = self.client.retrieve_account_groups()
 465
 466         #keep track of initial account meta
 467         self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
 468
 469         meta = {'foo':'bar'}
 470         self.client.update_account_metadata(**meta)
 471         self.updated_meta = self.initial_meta.update(meta)
 472
 473     def tearDown(self):
 474         #delete additionally created meta
 475         l = []
 476         for m in self.client.retrieve_account_metadata(restricted=True):
 477             if m not in self.initial_meta:
 478                 l.append(m)
 479         self.client.delete_account_metadata(l)
 480
 481         #delete additionally created groups
 482         l = []
 483         for g in self.client.retrieve_account_groups():
 484             if g not in self.initial_groups:
 485                 l.append(g)
 486         self.client.unset_account_groups(l)
 487
 488         BaseTestCase.tearDown(self)
 489
 490     def test_update_meta(self):
 491         with AssertMappingInvariant(self.client.retrieve_account_groups):
 492             meta = {'test':'test', 'tost':'tost'}
 493             self.client.update_account_metadata(**meta)
 494
 495             meta.update(self.initial_meta)
 496             self.assertEqual(meta,
 497                              self.client.retrieve_account_metadata(
 498                                 restricted=True))
 499
 500     def test_invalid_account_update_meta(self):
 501         meta = {'test':'test', 'tost':'tost'}
 502         self.assert_raises_fault(401,
 503                                  self.invalid_client.update_account_metadata,
 504                                  **meta)
 505
 506     def test_reset_meta(self):
 507         with AssertMappingInvariant(self.client.retrieve_account_groups):
 508             meta = {'test':'test', 'tost':'tost'}
 509             self.client.update_account_metadata(**meta)
 510
 511             meta = {'test':'test33'}
 512             self.client.reset_account_metadata(**meta)
 513
 514             self.assertEqual(meta, self.client.retrieve_account_metadata(restricted=True))
 515
 516     def test_delete_meta(self):
 517         with AssertMappingInvariant(self.client.retrieve_account_groups):
 518             meta = {'test':'test', 'tost':'tost'}
 519             self.client.update_account_metadata(**meta)
 520
 521             self.client.delete_account_metadata(meta.keys())
 522
 523             account_meta = self.client.retrieve_account_metadata(restricted=True)
 524             for m in meta:
 525                 self.assertTrue(m not in account_meta.keys())
 526
 527     def test_set_account_groups(self):
 528         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 529             groups = {'pithosdev':'verigak,gtsouk,chazapis'}
 530             self.client.set_account_groups(**groups)
 531
 532             self.assertEqual(groups, self.client.retrieve_account_groups())
 533
 534             more_groups = {'clientsdev':'pkanavos,mvasilak'}
 535             self.client.set_account_groups(**more_groups)
 536
 537             groups.update(more_groups)
 538             self.assertEqual(set(groups['pithosdev']),
 539                              set(self.client.retrieve_account_groups()['pithosdev']))
 540
 541     def test_reset_account_groups(self):
 542         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 543             groups = {'pithosdev':'verigak,gtsouk,chazapis',
 544                       'clientsdev':'pkanavos,mvasilak'}
 545             self.client.set_account_groups(**groups)
 546
 547             self.assertEqual(groups, self.client.retrieve_account_groups())
 548
 549             groups = {'pithosdev':'verigak,gtsouk,chazapis,papagian'}
 550             self.client.reset_account_groups(**groups)
 551
 552             self.assertEqual(groups['pithosdev'].split(','),
 553                              self.client.retrieve_account_groups()['pithosdev'].split(','))
 554
 555     def test_delete_account_groups(self):
 556         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 557             groups = {'pithosdev':'verigak,gtsouk,chazapis',
 558                       'clientsdev':'pkanavos,mvasilak'}
 559             self.client.set_account_groups(**groups)
 560
 561             self.client.unset_account_groups(groups.keys())
 562
 563             self.assertEqual({}, self.client.retrieve_account_groups())
 564
 565 class ContainerHead(BaseTestCase):
 566     def setUp(self):
 567         BaseTestCase.setUp(self)
 568         self.account = 'test'
 569         self.container = 'apples'
 570         self.client.create_container(self.container)
 571
 572     def test_get_meta(self):
 573         meta = {'trash':'true'}
 574         t1 = datetime.datetime.utcnow()
 575         o = self.upload_random_data(self.container, o_names[0], **meta)
 576         if o:
 577             headers = self.client.retrieve_container_metadata(self.container)
 578             self.assertEqual(headers['x-container-object-count'], '1')
 579             self.assertEqual(headers['x-container-bytes-used'], str(len(o['data'])))
 580             t2 = datetime.datetime.strptime(headers['last-modified'], DATE_FORMATS[2])
 581             delta = (t2 - t1)
 582             threashold = datetime.timedelta(seconds=1)
 583             self.assertTrue(delta < threashold)
 584             self.assertTrue(headers['x-container-object-meta'])
 585             self.assertTrue('Trash' in headers['x-container-object-meta'])
 586
 587 class ContainerGet(BaseTestCase):
 588     def setUp(self):
 589         BaseTestCase.setUp(self)
 590         self.account = 'test'
 591         self.container = ['pears', 'apples']
 592         for c in self.container:
 593             self.client.create_container(c)
 594         self.obj = []
 595         for o in o_names[:8]:
 596             self.obj.append(self.upload_random_data(self.container[0], o))
 597         for o in o_names[8:]:
 598             self.obj.append(self.upload_random_data(self.container[1], o))
 599
 600     def test_list_objects(self):
 601         objects = self.client.list_objects(self.container[0])
 602         l = [elem['name'] for elem in self.obj[:8]]
 603         l.sort()
 604         self.assertEqual(objects, l)
 605
 606     def test_list_objects_containing_slash(self):
 607         self.client.create_container('test')
 608         self.upload_random_data('test', '/objectname')
 609
 610         objects = self.client.list_objects('test')
 611         self.assertEqual(objects, ['/objectname'])
 612
 613         objects = self.client.list_objects('test', format='json')
 614         self.assertEqual(objects[0]['name'], '/objectname')
 615
 616         objects = self.client.list_objects('test', format='xml')
 617         self.assert_extended(objects, 'xml', 'object')
 618         node_name = objects.getElementsByTagName('name')[0]
 619         self.assertEqual(node_name.firstChild.data, '/objectname')
 620
 621         #objects = self.client.list_objects('test', prefix='/')
 622         #self.assertEqual(objects, ['/objectname'])
 623         #
 624         #objects = self.client.list_objects('test', path='/')
 625         #self.assertEqual(objects, ['/objectname'])
 626         #
 627         #objects = self.client.list_objects('test', prefix='/', delimiter='n')
 628         #self.assertEqual(objects, ['/object'])
 629
 630     def test_list_objects_with_limit_marker(self):
 631         objects = self.client.list_objects(self.container[0], limit=2)
 632         l = [elem['name'] for elem in self.obj[:8]]
 633         l.sort()
 634         self.assertEqual(objects, l[:2])
 635
 636         markers = ['How To Win Friends And Influence People.pdf',
 637                    'moms_birthday.jpg']
 638         limit = 4
 639         for m in markers:
 640             objects = self.client.list_objects(self.container[0], limit=limit,
 641                                                marker=m)
 642             l = [elem['name'] for elem in self.obj[:8]]
 643             l.sort()
 644             start = l.index(m) + 1
 645             end = start + limit
 646             end = len(l) >= end and end or len(l)
 647             self.assertEqual(objects, l[start:end])
 648
 649     #takes too long
 650     def _test_list_limit_exceeds(self):
 651         self.client.create_container('pithos')
 652
 653         for i in range(10001):
 654             self.client.create_zero_length_object('pithos', i)
 655
 656         self.assertEqual(10000, len(self.client.list_objects('pithos')))
 657
 658     def test_list_empty_params(self):
 659         objects = self.client.get('/%s/%s' % (get_user(), self.container[0]))[2]
 660         if objects:
 661             objects = objects.strip().split('\n')
 662         self.assertEqual(objects,
 663                          self.client.list_objects(self.container[0]))
 664
 665     def test_list_pseudo_hierarchical_folders(self):
 666         objects = self.client.list_objects(self.container[1], prefix='photos',
 667                                            delimiter='/')
 668         self.assertEquals(['photos/animals/', 'photos/me.jpg',
 669                            'photos/plants/'], objects)
 670
 671         objects = self.client.list_objects(self.container[1],
 672                                            prefix='photos/animals',
 673                                            delimiter='/')
 674         l = ['photos/animals/cats/', 'photos/animals/dogs/']
 675         self.assertEquals(l, objects)
 676
 677         objects = self.client.list_objects(self.container[1], path='photos')
 678         self.assertEquals(['photos/me.jpg'], objects)
 679
 680     def test_extended_list_json(self):
 681         objects = self.client.list_objects(self.container[1], format='json',
 682                                            limit=2, prefix='photos/animals',
 683                                            delimiter='/')
 684         self.assertEqual(objects[0]['subdir'], 'photos/animals/cats/')
 685         self.assertEqual(objects[1]['subdir'], 'photos/animals/dogs/')
 686
 687     def test_extended_list_xml(self):
 688         xml = self.client.list_objects(self.container[1], format='xml', limit=4,
 689                                        prefix='photos', delimiter='/')
 690         self.assert_extended(xml, 'xml', 'object', size=4)
 691         dirs = xml.getElementsByTagName('subdir')
 692         self.assertEqual(len(dirs), 2)
 693         self.assertEqual(dirs[0].attributes['name'].value, 'photos/animals/')
 694         self.assertEqual(dirs[1].attributes['name'].value, 'photos/plants/')
 695
 696         objects = xml.getElementsByTagName('name')
 697         self.assertEqual(len(objects), 1)
 698         self.assertEqual(objects[0].childNodes[0].data, 'photos/me.jpg')
 699
 700     def test_list_meta_double_matching(self):
 701         meta = {'quality':'aaa', 'stock':'true'}
 702         self.client.update_object_metadata(self.container[0],
 703                                            self.obj[0]['name'], **meta)
 704         obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
 705         self.assertEqual(len(obj), 1)
 706         self.assertTrue(obj, self.obj[0]['name'])
 707
 708     def test_list_using_meta(self):
 709         meta = {'quality':'aaa'}
 710         for o in self.obj[:2]:
 711             self.client.update_object_metadata(self.container[0], o['name'],
 712                                                **meta)
 713         meta = {'stock':'true'}
 714         for o in self.obj[3:5]:
 715             self.client.update_object_metadata(self.container[0], o['name'],
 716                                                **meta)
 717
 718         obj = self.client.list_objects(self.container[0], meta='Quality')
 719         self.assertEqual(len(obj), 2)
 720         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 721
 722         # test case insensitive
 723         obj = self.client.list_objects(self.container[0], meta='quality')
 724         self.assertEqual(len(obj), 2)
 725         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 726
 727         # test multiple matches
 728         obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
 729         self.assertEqual(len(obj), 4)
 730         self.assertTrue(obj, [o['name'] for o in self.obj[:4]])
 731
 732         # test non 1-1 multiple match
 733         obj = self.client.list_objects(self.container[0], meta='Quality,aaaa')
 734         self.assertEqual(len(obj), 2)
 735         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 736
 737     def test_if_modified_since(self):
 738         t = datetime.datetime.utcnow()
 739         t2 = t - datetime.timedelta(minutes=10)
 740
 741         #add a new object
 742         self.upload_random_data(self.container[0], o_names[0])
 743
 744         for f in DATE_FORMATS:
 745             past = t2.strftime(f)
 746             try:
 747                 o = self.client.list_objects(self.container[0],
 748                                             if_modified_since=past)
 749                 self.assertEqual(o,
 750                                  self.client.list_objects(self.container[0]))
 751             except Fault, f:
 752                 self.failIf(f.status == 304) #fail if not modified
 753
 754     def test_if_modified_since_invalid_date(self):
 755         headers = {'if-modified-since':''}
 756         o = self.client.list_objects(self.container[0], if_modified_since='')
 757         self.assertEqual(o, self.client.list_objects(self.container[0]))
 758
 759     def test_if_not_modified_since(self):
 760         now = datetime.datetime.utcnow()
 761         since = now + datetime.timedelta(1)
 762
 763         for f in DATE_FORMATS:
 764             args = {'if_modified_since':'%s' %since.strftime(f)}
 765
 766             #assert not modified
 767             self.assert_raises_fault(304, self.client.list_objects,
 768                                      self.container[0], **args)
 769
 770     def test_if_unmodified_since(self):
 771         now = datetime.datetime.utcnow()
 772         since = now + datetime.timedelta(1)
 773
 774         for f in DATE_FORMATS:
 775             obj = self.client.list_objects(self.container[0],
 776                                            if_unmodified_since=since.strftime(f))
 777
 778             #assert unmodified
 779             self.assertEqual(obj, self.client.list_objects(self.container[0]))
 780
 781     def test_if_unmodified_since_precondition_failed(self):
 782         t = datetime.datetime.utcnow()
 783         t2 = t - datetime.timedelta(minutes=10)
 784
 785         #add a new container
 786         self.client.create_container('dummy')
 787
 788         for f in DATE_FORMATS:
 789             past = t2.strftime(f)
 790
 791             args = {'if_unmodified_since':'%s' %past}
 792
 793             #assert precondition failed
 794             self.assert_raises_fault(412, self.client.list_objects,
 795                                      self.container[0], **args)
 796
 797 class ContainerPut(BaseTestCase):
 798     def setUp(self):
 799         BaseTestCase.setUp(self)
 800         self.account = 'test'
 801         self.containers = ['c1', 'c2']
 802
 803     def test_create(self):
 804         self.client.create_container(self.containers[0])
 805         containers = self.client.list_containers()
 806         self.assertTrue(self.containers[0] in containers)
 807         self.assert_container_exists(self.containers[0])
 808
 809     def test_create_twice(self):
 810         self.client.create_container(self.containers[0])
 811         self.assertTrue(not self.client.create_container(self.containers[0]))
 812
 813 class ContainerPost(BaseTestCase):
 814     def setUp(self):
 815         BaseTestCase.setUp(self)
 816         self.account = 'test'
 817         self.container = 'apples'
 818         self.client.create_container(self.container)
 819
 820     def test_update_meta(self):
 821         meta = {'test':'test33',
 822                 'tost':'tost22'}
 823         self.client.update_container_metadata(self.container, **meta)
 824         headers = self.client.retrieve_container_metadata(self.container)
 825         for k,v in meta.items():
 826             k = 'x-container-meta-%s' % k
 827             self.assertTrue(headers[k])
 828             self.assertEqual(headers[k], v)
 829
 830 class ContainerDelete(BaseTestCase):
 831     def setUp(self):
 832         BaseTestCase.setUp(self)
 833         self.account = 'test'
 834         self.containers = ['c1', 'c2']
 835         for c in self.containers:
 836             self.client.create_container(c)
 837         self.upload_random_data(self.containers[1], o_names[0])
 838
 839     def test_delete(self):
 840         status = self.client.delete_container(self.containers[0])[0]
 841         self.assertEqual(status, 204)
 842
 843     def test_delete_non_empty(self):
 844         self.assert_raises_fault(409, self.client.delete_container,
 845                                  self.containers[1])
 846
 847     def test_delete_invalid(self):
 848         self.assert_raises_fault(404, self.client.delete_container, 'c3')
 849
 850 class ObjectHead(BaseTestCase):
 851     pass
 852
 853 class ObjectGet(BaseTestCase):
 854     def setUp(self):
 855         BaseTestCase.setUp(self)
 856         self.account = 'test'
 857         self.containers = ['c1', 'c2']
 858         #create some containers
 859         for c in self.containers:
 860             self.client.create_container(c)
 861
 862         #upload a file
 863         names = ('obj1', 'obj2')
 864         self.objects = []
 865         for n in names:
 866             self.objects.append(self.upload_random_data(self.containers[1], n))
 867
 868     def test_get(self):
 869         #perform get
 870         o = self.client.retrieve_object(self.containers[1],
 871                                         self.objects[0]['name'],
 872                                         self.objects[0]['meta'])
 873         self.assertEqual(o, self.objects[0]['data'])
 874
 875     def test_get_invalid(self):
 876         self.assert_raises_fault(404, self.client.retrieve_object,
 877                                  self.containers[0], self.objects[0]['name'])
 878
 879     def test_get_partial(self):
 880         #perform get with range
 881         status, headers, data = self.client.request_object(self.containers[1],
 882                                                             self.objects[0]['name'],
 883                                                             range='bytes=0-499')
 884
 885         #assert successful partial content
 886         self.assertEqual(status, 206)
 887
 888         #assert content-type
 889         self.assertEqual(headers['content-type'],
 890                          self.objects[0]['meta']['content_type'])
 891
 892         #assert content length
 893         self.assertEqual(int(headers['content-length']), 500)
 894
 895         #assert content
 896         self.assertEqual(self.objects[0]['data'][:500], data)
 897
 898     def test_get_final_500(self):
 899         #perform get with range
 900         headers = {'range':'bytes=-500'}
 901         status, headers, data = self.client.request_object(self.containers[1],
 902                                                             self.objects[0]['name'],
 903                                                             range='bytes=-500')
 904
 905         #assert successful partial content
 906         self.assertEqual(status, 206)
 907
 908         #assert content-type
 909         self.assertEqual(headers['content-type'],
 910                          self.objects[0]['meta']['content_type'])
 911
 912         #assert content length
 913         self.assertEqual(int(headers['content-length']), 500)
 914
 915         #assert content
 916         self.assertTrue(self.objects[0]['data'][-500:], data)
 917
 918     def test_get_rest(self):
 919         #perform get with range
 920         offset = len(self.objects[0]['data']) - 500
 921         status, headers, data = self.client.request_object(self.containers[1],
 922                                                 self.objects[0]['name'],
 923                                                 range='bytes=%s-' %offset)
 924
 925         #assert successful partial content
 926         self.assertEqual(status, 206)
 927
 928         #assert content-type
 929         self.assertEqual(headers['content-type'],
 930                          self.objects[0]['meta']['content_type'])
 931
 932         #assert content length
 933         self.assertEqual(int(headers['content-length']), 500)
 934
 935         #assert content
 936         self.assertTrue(self.objects[0]['data'][-500:], data)
 937
 938     def test_get_range_not_satisfiable(self):
 939         #perform get with range
 940         offset = len(self.objects[0]['data']) + 1
 941
 942         #assert range not satisfiable
 943         self.assert_raises_fault(416, self.client.retrieve_object,
 944                                  self.containers[1], self.objects[0]['name'],
 945                                  range='bytes=0-%s' %offset)
 946
 947     def test_multiple_range(self):
 948         #perform get with multiple range
 949         ranges = ['0-499', '-500', '1000-']
 950         bytes = 'bytes=%s' % ','.join(ranges)
 951         status, headers, data = self.client.request_object(self.containers[1],
 952                                                            self.objects[0]['name'],
 953                                                            range=bytes)
 954
 955         # assert partial content
 956         self.assertEqual(status, 206)
 957
 958         # assert Content-Type of the reply will be multipart/byteranges
 959         self.assertTrue(headers['content-type'])
 960         content_type_parts = headers['content-type'].split()
 961         self.assertEqual(content_type_parts[0], ('multipart/byteranges;'))
 962
 963         boundary = '--%s' %content_type_parts[1].split('=')[-1:][0]
 964         cparts = data.split(boundary)[1:-1]
 965
 966         # assert content parts are exactly 2
 967         self.assertEqual(len(cparts), len(ranges))
 968
 969         # for each content part assert headers
 970         i = 0
 971         for cpart in cparts:
 972             content = cpart.split('\r\n')
 973             headers = content[1:3]
 974             content_range = headers[0].split(': ')
 975             self.assertEqual(content_range[0], 'Content-Range')
 976
 977             r = ranges[i].split('-')
 978             if not r[0] and not r[1]:
 979                 pass
 980             elif not r[0]:
 981                 start = len(self.objects[0]['data']) - int(r[1])
 982                 end = len(self.objects[0]['data'])
 983             elif not r[1]:
 984                 start = int(r[0])
 985                 end = len(self.objects[0]['data'])
 986             else:
 987                 start = int(r[0])
 988                 end = int(r[1]) + 1
 989             fdata = self.objects[0]['data'][start:end]
 990             sdata = '\r\n'.join(content[4:-1])
 991             self.assertEqual(len(fdata), len(sdata))
 992             self.assertEquals(fdata, sdata)
 993             i+=1
 994
 995     def test_multiple_range_not_satisfiable(self):
 996         #perform get with multiple range
 997         out_of_range = len(self.objects[0]['data']) + 1
 998         ranges = ['0-499', '-500', '%d-' %out_of_range]
 999         bytes = 'bytes=%s' % ','.join(ranges)
1000
1001         # assert partial content
1002         self.assert_raises_fault(416, self.client.retrieve_object,
1003                                  self.containers[1],
1004                                  self.objects[0]['name'], range=bytes)
1005
1006     def test_get_with_if_match(self):
1007         #perform get with If-Match
1008         etag = self.objects[0]['hash']
1009         status, headers, data = self.client.request_object(self.containers[1],
1010                                                            self.objects[0]['name'],
1011                                                            if_match=etag)
1012         #assert get success
1013         self.assertEqual(status, 200)
1014
1015         #assert content-type
1016         self.assertEqual(headers['content-type'],
1017                          self.objects[0]['meta']['content_type'])
1018
1019         #assert response content
1020         self.assertEqual(self.objects[0]['data'], data)
1021
1022     def test_get_with_if_match_star(self):
1023         #perform get with If-Match *
1024         headers = {'if-match':'*'}
1025         status, headers, data = self.client.request_object(self.containers[1],
1026                                                 self.objects[0]['name'],
1027                                                 **headers)
1028         #assert get success
1029         self.assertEqual(status, 200)
1030
1031         #assert content-type
1032         self.assertEqual(headers['content-type'],
1033                          self.objects[0]['meta']['content_type'])
1034
1035         #assert response content
1036         self.assertEqual(self.objects[0]['data'], data)
1037
1038     def test_get_with_multiple_if_match(self):
1039         #perform get with If-Match
1040         etags = [i['hash'] for i in self.objects if i]
1041         etags = ','.join('"%s"' % etag for etag in etags)
1042         status, headers, data = self.client.request_object(self.containers[1],
1043                                                            self.objects[0]['name'],
1044                                                            if_match=etags)
1045         #assert get success
1046         self.assertEqual(status, 200)
1047
1048         #assert content-type
1049         self.assertEqual(headers['content-type'],
1050                          self.objects[0]['meta']['content_type'])
1051
1052         #assert content-type
1053         self.assertEqual(headers['content-type'],
1054                          self.objects[0]['meta']['content_type'])
1055
1056         #assert response content
1057         self.assertEqual(self.objects[0]['data'], data)
1058
1059     def test_if_match_precondition_failed(self):
1060         #assert precondition failed
1061         self.assert_raises_fault(412, self.client.retrieve_object,
1062                                  self.containers[1],
1063                                  self.objects[0]['name'], if_match='123')
1064
1065     def test_if_none_match(self):
1066         #perform get with If-None-Match
1067         status, headers, data = self.client.request_object(self.containers[1],
1068                                                            self.objects[0]['name'],
1069                                                            if_none_match='123')
1070
1071         #assert get success
1072         self.assertEqual(status, 200)
1073
1074         #assert content-type
1075         self.assertEqual(headers['content_type'],
1076                          self.objects[0]['meta']['content_type'])
1077
1078     def test_if_none_match(self):
1079         #perform get with If-None-Match * and assert not modified
1080         self.assert_raises_fault(304, self.client.retrieve_object,
1081                                  self.containers[1],
1082                                  self.objects[0]['name'],
1083                                  if_none_match='*')
1084
1085     def test_if_none_match_not_modified(self):
1086         #perform get with If-None-Match and assert not modified
1087         self.assert_raises_fault(304, self.client.retrieve_object,
1088                                  self.containers[1],
1089                                  self.objects[0]['name'],
1090                                  if_none_match=self.objects[0]['hash'])
1091
1092         meta = self.client.retrieve_object_metadata(self.containers[1],
1093                                                     self.objects[0]['name'])
1094         self.assertEqual(meta['etag'], self.objects[0]['hash'])
1095
1096     def test_if_modified_since(self):
1097         t = datetime.datetime.utcnow()
1098         t2 = t - datetime.timedelta(minutes=10)
1099
1100         #modify the object
1101         self.upload_data(self.containers[1],
1102                            self.objects[0]['name'],
1103                            self.objects[0]['data'][:200])
1104
1105         for f in DATE_FORMATS:
1106             past = t2.strftime(f)
1107
1108             headers = {'if-modified-since':'%s' %past}
1109             try:
1110                 o = self.client.retrieve_object(self.containers[1],
1111                                                 self.objects[0]['name'],
1112                                                 if_modified_since=past)
1113                 self.assertEqual(o,
1114                                  self.client.retrieve_object(self.containers[1],
1115                                                              self.objects[0]['name']))
1116             except Fault, f:
1117                 self.failIf(f.status == 304)
1118
1119     def test_if_modified_since_invalid_date(self):
1120         o = self.client.retrieve_object(self.containers[1],
1121                                         self.objects[0]['name'],
1122                                         if_modified_since='')
1123         self.assertEqual(o, self.client.retrieve_object(self.containers[1],
1124                                                         self.objects[0]['name']))
1125
1126     def test_if_not_modified_since(self):
1127         now = datetime.datetime.utcnow()
1128         since = now + datetime.timedelta(1)
1129
1130         for f in DATE_FORMATS:
1131             #assert not modified
1132             self.assert_raises_fault(304, self.client.retrieve_object,
1133                                      self.containers[1], self.objects[0]['name'],
1134                                      if_modified_since=since.strftime(f))
1135
1136     def test_if_unmodified_since(self):
1137         now = datetime.datetime.utcnow()
1138         since = now + datetime.timedelta(1)
1139
1140         for f in DATE_FORMATS:
1141             t = since.strftime(f)
1142             status, headers, data = self.client.request_object(self.containers[1],
1143                                                                self.objects[0]['name'],
1144                                                                if_unmodified_since=t)
1145             #assert success
1146             self.assertEqual(status, 200)
1147             self.assertEqual(self.objects[0]['data'], data)
1148
1149             #assert content-type
1150             self.assertEqual(headers['content-type'],
1151                              self.objects[0]['meta']['content_type'])
1152
1153     def test_if_unmodified_since_precondition_failed(self):
1154         t = datetime.datetime.utcnow()
1155         t2 = t - datetime.timedelta(minutes=10)
1156
1157         #modify the object
1158         self.upload_data(self.containers[1],
1159                            self.objects[0]['name'],
1160                            self.objects[0]['data'][:200])
1161
1162         for f in DATE_FORMATS:
1163             past = t2.strftime(f)
1164             #assert precondition failed
1165             self.assert_raises_fault(412, self.client.retrieve_object,
1166                                      self.containers[1], self.objects[0]['name'],
1167                                      if_unmodified_since=past)
1168
1169     def test_hashes(self):
1170         l = 8388609
1171         fname = 'largefile'
1172         o = self.upload_random_data(self.containers[1], fname, l)
1173         if o:
1174             data = self.client.retrieve_object(self.containers[1], fname,
1175                                                format='json')
1176             body = json.loads(data)
1177             hashes = body['hashes']
1178             block_size = body['block_size']
1179             block_hash = body['block_hash']
1180             block_num = l/block_size == 0 and l/block_size or l/block_size + 1
1181             self.assertTrue(len(hashes), block_num)
1182             i = 0
1183             for h in hashes:
1184                 start = i * block_size
1185                 end = (i + 1) * block_size
1186                 hash = compute_block_hash(o['data'][start:end], block_hash)
1187                 self.assertEqual(h, hash)
1188                 i += 1
1189
1190 class ObjectPut(BaseTestCase):
1191     def setUp(self):
1192         BaseTestCase.setUp(self)
1193         self.account = 'test'
1194         self.container = 'c1'
1195         self.client.create_container(self.container)
1196
1197     def test_upload(self):
1198         name = o_names[0]
1199         meta = {'test':'test1'}
1200         o = self.upload_random_data(self.container, name, **meta)
1201
1202         headers = self.client.retrieve_object_metadata(self.container,
1203                                                        name,
1204                                                        restricted=True)
1205         self.assertTrue('test' in headers.keys())
1206         self.assertEqual(headers['test'], meta['test'])
1207
1208         #assert uploaded content
1209         status, h, data = self.client.request_object(self.container, name)
1210         self.assertEqual(len(o['data']), int(h['content-length']))
1211         self.assertEqual(o['data'], data)
1212
1213         #assert content-type
1214         self.assertEqual(h['content-type'], o['meta']['content_type'])
1215
1216     def _test_maximum_upload_size_exceeds(self):
1217         name = o_names[0]
1218         meta = {'test':'test1'}
1219         #upload 100MB
1220         length=1024*1024*100
1221         self.assert_raises_fault(400, self.upload_random_data, self.container,
1222                                  name, length, **meta)
1223
1224     def test_upload_with_name_containing_slash(self):
1225         name = '/%s' % o_names[0]
1226         meta = {'test':'test1'}
1227         o = self.upload_random_data(self.container, name, **meta)
1228
1229         self.assertEqual(o['data'],
1230                          self.client.retrieve_object(self.container, name))
1231
1232         self.assertTrue(name in self.client.list_objects(self.container))
1233
1234     def test_create_directory_marker(self):
1235         self.client.create_directory_marker(self.container, 'foo')
1236         meta = self.client.retrieve_object_metadata(self.container, 'foo')
1237         self.assertEqual(meta['content-length'], '0')
1238         self.assertEqual(meta['content-type'], 'application/directory')
1239
1240     def test_upload_unprocessable_entity(self):
1241         meta={'etag':'123', 'test':'test1'}
1242
1243         #assert unprocessable entity
1244         self.assert_raises_fault(422, self.upload_random_data, self.container,
1245                                  o_names[0], **meta)
1246
1247     def test_chunked_transfer(self):
1248         data = get_random_data()
1249         objname = 'object'
1250         self.client.create_object_using_chunks(self.container, objname,
1251                                                StringIO(data))
1252
1253         uploaded_data = self.client.retrieve_object(self.container, objname)
1254         self.assertEqual(data, uploaded_data)
1255
1256     def test_manifestation(self):
1257         prefix = 'myobject/'
1258         data = ''
1259         for i in range(5):
1260             part = '%s%d' %(prefix, i)
1261             o = self.upload_random_data(self.container, part)
1262             data += o['data']
1263
1264         manifest = '%s/%s' %(self.container, prefix)
1265         self.client.create_manifestation(self.container, 'large-object', manifest)
1266
1267         self.assert_object_exists(self.container, 'large-object')
1268         self.assertEqual(data, self.client.retrieve_object(self.container,
1269                                                            'large-object'))
1270
1271         #wrong manifestation
1272         self.client.create_manifestation(self.container, 'large-object',
1273                                          '%s/invalid' % self.container)
1274         self.assertEqual('', self.client.retrieve_object(self.container,
1275                                                          'large-object'))
1276
1277 class ObjectCopy(BaseTestCase):
1278     def setUp(self):
1279         BaseTestCase.setUp(self)
1280         self.account = 'test'
1281         self.containers = ['c1', 'c2']
1282         for c in self.containers:
1283             self.client.create_container(c)
1284         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1285
1286     def tearDown(self):
1287         pass
1288
1289     def test_copy(self):
1290         with AssertMappingInvariant(self.client.retrieve_object_metadata,
1291                              self.containers[0], self.obj['name']):
1292             #perform copy
1293             meta = {'test':'testcopy'}
1294             status = self.client.copy_object(self.containers[0],
1295                                               self.obj['name'],
1296                                               self.containers[0],
1297                                               'testcopy',
1298                                               meta)[0]
1299
1300             #assert copy success
1301             self.assertEqual(status, 201)
1302
1303             #assert access the new object
1304             headers = self.client.retrieve_object_metadata(self.containers[0],
1305                                                            'testcopy')
1306             self.assertTrue('x-object-meta-test' in headers.keys())
1307             self.assertTrue(headers['x-object-meta-test'], 'testcopy')
1308
1309             #assert etag is the same
1310             self.assertEqual(headers['etag'], self.obj['hash'])
1311
1312             #assert src object still exists
1313             self.assert_object_exists(self.containers[0], self.obj['name'])
1314
1315     def test_copy_from_different_container(self):
1316         with AssertMappingInvariant(self.client.retrieve_object_metadata,
1317                              self.containers[0], self.obj['name']):
1318             meta = {'test':'testcopy'}
1319             status = self.client.copy_object(self.containers[0],
1320                                              self.obj['name'],
1321                                              self.containers[1],
1322                                              'testcopy',
1323                                              meta)[0]
1324             self.assertEqual(status, 201)
1325
1326             # assert updated metadata
1327             meta = self.client.retrieve_object_metadata(self.containers[1],
1328                                                            'testcopy',
1329                                                            restricted=True)
1330             self.assertTrue('test' in meta.keys())
1331             self.assertTrue(meta['test'], 'testcopy')
1332
1333             #assert src object still exists
1334             self.assert_object_exists(self.containers[0], self.obj['name'])
1335
1336     def test_copy_invalid(self):
1337         #copy from invalid object
1338         meta = {'test':'testcopy'}
1339         self.assert_raises_fault(404, self.client.copy_object, self.containers[0],
1340                                  'test.py', self.containers[1], 'testcopy', meta)
1341
1342         #copy from invalid container
1343         meta = {'test':'testcopy'}
1344         self.assert_raises_fault(404, self.client.copy_object, self.containers[1],
1345                                  self.obj['name'], self.containers[1],
1346                                  'testcopy', meta)
1347
1348 class ObjectMove(BaseTestCase):
1349     def setUp(self):
1350         BaseTestCase.setUp(self)
1351         self.account = 'test'
1352         self.containers = ['c1', 'c2']
1353         for c in self.containers:
1354             self.client.create_container(c)
1355         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1356
1357     def test_move(self):
1358         #perform move
1359         meta = {'test':'testcopy'}
1360         src_path = '/'.join(('/', self.containers[0], self.obj['name']))
1361         status = self.client.move_object(self.containers[0], self.obj['name'],
1362                                          self.containers[0], 'testcopy',
1363                                          meta)[0]
1364
1365         #assert successful move
1366         self.assertEqual(status, 201)
1367
1368         #assert updated metadata
1369         meta = self.client.retrieve_object_metadata(self.containers[0],
1370                                                     'testcopy',
1371                                                     restricted=True)
1372         self.assertTrue('test' in meta.keys())
1373         self.assertTrue(meta['test'], 'testcopy')
1374
1375         #assert src object no more exists
1376         self.assert_object_not_exists(self.containers[0], self.obj['name'])
1377
1378 class ObjectPost(BaseTestCase):
1379     def setUp(self):
1380         BaseTestCase.setUp(self)
1381         self.account = 'test'
1382         self.containers = ['c1', 'c2']
1383         for c in self.containers:
1384             self.client.create_container(c)
1385         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1386
1387     def test_update_meta(self):
1388         #perform update metadata
1389         more = {'foo':'foo', 'bar':'bar'}
1390         status = self.client.update_object_metadata(self.containers[0],
1391                                                     self.obj['name'],
1392                                                     **more)[0]
1393         #assert request accepted
1394         self.assertEqual(status, 202)
1395
1396         #assert old metadata are still there
1397         headers = self.client.retrieve_object_metadata(self.containers[0],
1398                                                        self.obj['name'],
1399                                                        restricted=True)
1400         #assert new metadata have been updated
1401         for k,v in more.items():
1402             self.assertTrue(k in headers.keys())
1403             self.assertTrue(headers[k], v)
1404
1405     def test_update_object(self,
1406                            first_byte_pos=0,
1407                            last_byte_pos=499,
1408                            instance_length = True,
1409                            content_length = 500):
1410         l = len(self.obj['data'])
1411         length = l if instance_length else '*'
1412         range = 'bytes %d-%d/%s' %(first_byte_pos,
1413                                        last_byte_pos,
1414                                        length)
1415         partial = last_byte_pos - first_byte_pos + 1
1416         data = get_random_data(partial)
1417         args = {'content_type':'application/octet-stream',
1418                 'content_range':'%s' %range}
1419         if content_length:
1420             args['content_length'] = content_length
1421
1422         status = self.client.update_object(self.containers[0], self.obj['name'],
1423                                   StringIO(data), **args)[0]
1424
1425         if partial < 0 or (instance_length and l <= last_byte_pos):
1426             self.assertEqual(status, 202)
1427         else:
1428             self.assertEqual(status, 204)
1429             #check modified object
1430             content = self.client.retrieve_object(self.containers[0],
1431                                               self.obj['name'])
1432             self.assertEqual(content[0:partial], data)
1433             self.assertEqual(content[partial:l], self.obj['data'][partial:l])
1434
1435     def test_update_object_no_content_length(self):
1436         self.test_update_object(content_length = None)
1437
1438     def test_update_object_invalid_content_length(self):
1439         with AssertContentInvariant(self.client.retrieve_object,
1440                                     self.containers[0], self.obj['name']):
1441             self.assert_raises_fault(400, self.test_update_object,
1442                                      content_length = 1000)
1443
1444     def test_update_object_invalid_range(self):
1445         with AssertContentInvariant(self.client.retrieve_object,
1446                                     self.containers[0], self.obj['name']):
1447             self.assert_raises_fault(416, self.test_update_object, 499, 0, True)
1448
1449     def test_update_object_invalid_range_and_length(self):
1450         with AssertContentInvariant(self.client.retrieve_object,
1451                                     self.containers[0], self.obj['name']):
1452             self.assert_raises_fault(416, self.test_update_object, 499, 0, True,
1453                                      -1)
1454
1455     def test_update_object_invalid_range_with_no_content_length(self):
1456         with AssertContentInvariant(self.client.retrieve_object,
1457                                     self.containers[0], self.obj['name']):
1458             self.assert_raises_fault(416, self.test_update_object, 499, 0, True,
1459                                      content_length = None)
1460
1461     def test_update_object_out_of_limits(self):
1462         with AssertContentInvariant(self.client.retrieve_object,
1463                                     self.containers[0], self.obj['name']):
1464             l = len(self.obj['data'])
1465             self.assert_raises_fault(416, self.test_update_object, 0, l+1, True)
1466
1467     def test_append(self):
1468         data = get_random_data(500)
1469         headers = {}
1470         self.client.update_object(self.containers[0], self.obj['name'],
1471                                   StringIO(data), content_length=500,
1472                                   content_type='application/octet-stream')
1473
1474         content = self.client.retrieve_object(self.containers[0],
1475                                               self.obj['name'])
1476         self.assertEqual(len(content), len(self.obj['data']) + 500)
1477         self.assertEqual(content[:-500], self.obj['data'])
1478
1479     def test_update_with_chunked_transfer(self):
1480         data = get_random_data(500)
1481         dl = len(data)
1482         fl = len(self.obj['data'])
1483
1484         self.client.update_object_using_chunks(self.containers[0],
1485                                                self.obj['name'], StringIO(data),
1486                                                offset=0,
1487                                                content_type='application/octet-stream')
1488
1489         #check modified object
1490         content = self.client.retrieve_object(self.containers[0],
1491                                               self.obj['name'])
1492         self.assertEqual(content[0:dl], data)
1493         self.assertEqual(content[dl:fl], self.obj['data'][dl:fl])
1494
1495     def test_update_from_other_object(self):
1496         data = self.upload_random_data(self.containers[0], o_names[1])['data']
1497         source_object = '/%s/%s' % (self.containers[0], o_names[0])
1498         self.client.update_from_other_source(self.containers[0], o_names[1],
1499                                              source_object)
1500
1501         self.assertEqual(
1502             self.client.retrieve_object(self.containers[0], o_names[0]),
1503             self.client.retrieve_object(self.containers[0], o_names[1]))
1504
1505
1506 class ObjectDelete(BaseTestCase):
1507     def setUp(self):
1508         BaseTestCase.setUp(self)
1509         self.account = 'test'
1510         self.containers = ['c1', 'c2']
1511         for c in self.containers:
1512             self.client.create_container(c)
1513         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1514
1515     def test_delete(self):
1516         #perform delete object
1517         self.client.delete_object(self.containers[0], self.obj['name'])[0]
1518
1519     def test_delete_invalid(self):
1520         #assert item not found
1521         self.assert_raises_fault(404, self.client.delete_object, self.containers[1],
1522                                  self.obj['name'])
1523
1524 class ListSharing(BaseTestCase):
1525     def setUp(self):
1526         BaseTestCase.setUp(self)
1527         self.client.create_container('c')
1528         for i in range(2):
1529             self.upload_random_data('c', 'o%s' %i)
1530         accounts = OTHER_ACCOUNTS.copy()
1531         self.o1_sharing_with = accounts.popitem()
1532         self.o1_sharing = [self.o1_sharing_with[1]]
1533         self.client.share_object('c', 'o1', self.o1_sharing, read=True)
1534
1535         l = []
1536         for i in range(2):
1537             l.append(accounts.popitem())
1538         #self.client.set_account_groups({'pithos-dev':'chazapis,verigak,papagian'})
1539         #self.o2_sharing = 'write=%s' %
1540         #self.client.share_object('c', 'o2', self.o2_sharing)
1541
1542     def test_listing(self):
1543         self.other = Pithos_Client(get_server(),
1544                               self.o1_sharing_with[0],
1545                               self.o1_sharing_with[1],
1546                               get_api())
1547         self.assertTrue('test' in self.other.list_shared_by_others())
1548
1549 class TestGreek(BaseTestCase):
1550     def setUp(self):
1551         BaseTestCase.setUp(self)
1552         #keep track of initial account groups
1553         self.initial_groups = self.client.retrieve_account_groups()
1554
1555         #keep track of initial account meta
1556         self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
1557
1558     def tearDown(self):
1559         #delete additionally created meta
1560         l = []
1561         for m in self.client.retrieve_account_metadata(restricted=True):
1562             if m not in self.initial_meta:
1563                 l.append(m)
1564         self.client.delete_account_metadata(l)
1565
1566         #delete additionally created groups
1567         l = []
1568         for g in self.client.retrieve_account_groups():
1569             if g not in self.initial_groups:
1570                 l.append(g)
1571         self.client.unset_account_groups(l)
1572
1573         BaseTestCase.tearDown(self)
1574
1575     def test_create_container(self):
1576         self.client.create_container('φάκελος')
1577         self.assert_container_exists('φάκελος')
1578
1579         self.assertTrue('φάκελος' in self.client.list_containers())
1580
1581     def test_create_object(self):
1582         self.client.create_container('φάκελος')
1583         self.upload_random_data('φάκελος', 'αντικείμενο')
1584
1585         self.assert_object_exists('φάκελος', 'αντικείμενο')
1586         self.assertTrue('αντικείμενο' in self.client.list_objects('φάκελος'))
1587
1588     def test_copy_object(self):
1589         src_container = 'φάκελος'
1590         src_object = 'αντικείμενο'
1591         dest_container = 'αντίγραφα'
1592         dest_object = 'ασφαλές-αντίγραφο'
1593
1594         self.client.create_container(src_container)
1595         self.upload_random_data(src_container, src_object)
1596
1597         self.client.create_container(dest_container)
1598         self.client.copy_object(src_container, src_object, dest_container,
1599                                 dest_object)
1600
1601         self.assert_object_exists(src_container, src_object)
1602         self.assert_object_exists(dest_container, dest_object)
1603         self.assertTrue(dest_object in self.client.list_objects(dest_container))
1604
1605     def test_move_object(self):
1606         src_container = 'φάκελος'
1607         src_object = 'αντικείμενο'
1608         dest_container = 'αντίγραφα'
1609         dest_object = 'ασφαλές-αντίγραφο'
1610
1611         self.client.create_container(src_container)
1612         self.upload_random_data(src_container, src_object)
1613
1614         self.client.create_container(dest_container)
1615         self.client.move_object(src_container, src_object, dest_container,
1616                                 dest_object)
1617
1618         self.assert_object_not_exists(src_container, src_object)
1619         self.assert_object_exists(dest_container, dest_object)
1620         self.assertTrue(dest_object in self.client.list_objects(dest_container))
1621
1622     def test_delete_object(self):
1623         self.client.create_container('φάκελος')
1624         self.upload_random_data('φάκελος', 'αντικείμενο')
1625         self.assert_object_exists('φάκελος', 'αντικείμενο')
1626
1627         self.client.delete_object('φάκελος', 'αντικείμενο')
1628         self.assert_object_not_exists('φάκελος', 'αντικείμενο')
1629         self.assertTrue('αντικείμενο' not in self.client.list_objects('φάκελος'))
1630
1631     def test_delete_container(self):
1632         self.client.create_container('φάκελος')
1633         self.assert_container_exists('φάκελος')
1634
1635         self.client.delete_container('φάκελος')
1636         self.assert_container_not_exists('φάκελος')
1637         self.assertTrue('φάκελος' not in self.client.list_containers())
1638
1639     def test_account_meta(self):
1640         meta = {'ποιότητα':'ΑΑΑ'}
1641         self.client.update_account_metadata(**meta)
1642         meta = self.client.retrieve_account_metadata(restricted=True)
1643         self.assertTrue('ποιότητα' in meta.keys())
1644         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1645
1646     def test_container_meta(self):
1647         meta = {'ποιότητα':'ΑΑΑ'}
1648         self.client.create_container('φάκελος', **meta)
1649
1650         meta = self.client.retrieve_container_metadata('φάκελος', restricted=True)
1651         self.assertTrue('ποιότητα' in meta.keys())
1652         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1653
1654     def test_object_meta(self):
1655         self.client.create_container('φάκελος')
1656         meta = {'ποιότητα':'ΑΑΑ'}
1657         self.upload_random_data('φάκελος', 'αντικείμενο', **meta)
1658
1659         meta = self.client.retrieve_object_metadata('φάκελος', 'αντικείμενο',
1660                                                     restricted=True)
1661         self.assertTrue('ποιότητα' in meta.keys())
1662         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1663
1664     def test_list_meta_filtering(self):
1665         self.client.create_container('φάκελος')
1666         meta = {'ποιότητα':'ΑΑΑ'}
1667         self.upload_random_data('φάκελος', 'ο1', **meta)
1668         self.upload_random_data('φάκελος', 'ο2')
1669         self.upload_random_data('φάκελος', 'ο3')
1670
1671         meta = {'ποσότητα':'μεγάλη'}
1672         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
1673         objects = self.client.list_objects('φάκελος', meta='ποιότητα, ποσότητα')
1674         self.assertTrue('ο1' in objects)
1675         self.assertTrue('ο2' in objects)
1676         self.assertTrue('ο3' not in objects)
1677
1678     def test_groups(self):
1679         #create a group
1680         groups = {'σεφς':'chazapis,διογένης'}
1681         self.client.set_account_groups(**groups)
1682         groups.update(self.initial_groups)
1683         self.assertEqual(groups['σεφς'],
1684                          self.client.retrieve_account_groups()['σεφς'])
1685
1686         #check read access
1687         self.client.create_container('φάκελος')
1688         o = self.upload_random_data('φάκελος', 'ο1')
1689         self.client.share_object('φάκελος', 'ο1', ['test:σεφς'])
1690         chef = Pithos_Client(get_server(),
1691                             '0009',
1692                             'διογένης',
1693                             get_api())
1694         self.assert_not_raises_fault(401, chef.retrieve_object_metadata,
1695                                      'φάκελος', 'ο1', account=get_user())
1696
1697         #check write access
1698         self.client.share_object('φάκελος', 'ο1', ['διογένης'], read=False)
1699         new_data = get_random_data()
1700         self.assert_not_raises_fault(401, chef.update_object,
1701                                      'φάκελος', 'ο1', StringIO(new_data),
1702                                      account=get_user())
1703
1704         server_data = self.client.retrieve_object('φάκελος', 'ο1')
1705         self.assertEqual(server_data[:len(o['data'])], o['data'])
1706         self.assertEqual(server_data[len(o['data']):], new_data)
1707
1708     def test_manifestation(self):
1709         self.client.create_container('κουβάς')
1710         prefix = 'μέρη/'
1711         data = ''
1712         for i in range(5):
1713             part = '%s%d' %(prefix, i)
1714             o = self.upload_random_data('κουβάς', part)
1715             data += o['data']
1716
1717         self.client.create_container('φάκελος')
1718         manifest = '%s/%s' %('κουβάς', prefix)
1719         self.client.create_manifestation('φάκελος', 'άπαντα', manifest)
1720
1721         self.assert_object_exists('φάκελος', 'άπαντα')
1722         self.assertEqual(data, self.client.retrieve_object('φάκελος',
1723                                                            'άπαντα'))
1724
1725         #wrong manifestation
1726         self.client.create_manifestation('φάκελος', 'άπαντα', 'κουβάς/άκυρο')
1727         self.assertEqual('', self.client.retrieve_object('φάκελος', 'άπαντα'))
1728
1729     def test_update_from_another_object(self):
1730         self.client.create_container('κουβάς')
1731         src_data = self.upload_random_data('κουβάς', 'πηγή')['data']
1732         initial_data = self.upload_random_data('κουβάς', 'νέο')['data']
1733         source_object = '/%s/%s' % ('κουβάς', 'πηγή')
1734         self.client.update_from_other_source('κουβάς', 'νέο', source_object)
1735
1736         self.assertEqual(
1737             self.client.retrieve_object('κουβάς', 'νέο'),
1738             '%s%s' % (initial_data, self.client.retrieve_object('κουβάς', 'πηγή')))
1739
1740 class TestPermissions(BaseTestCase):
1741     def setUp(self):
1742         BaseTestCase.setUp(self)
1743         #keep track of initial account groups
1744         self.initial_groups = self.client.retrieve_account_groups()
1745         #keep track of initial account meta
1746         self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
1747
1748         #create a group
1749         self.authorized = ['chazapis', 'verigak', 'gtsouk', 'papagian']
1750         groups = {'pithosdev':','.join(self.authorized)}
1751         self.client.set_account_groups(**groups)
1752
1753     def tearDown(self):
1754         #delete additionally created meta
1755         l = []
1756         for m in self.client.retrieve_account_metadata(restricted=True):
1757             if m not in self.initial_meta:
1758                 l.append(m)
1759         self.client.delete_account_metadata(l)
1760
1761         #delete additionally created groups
1762         l = []
1763         for g in self.client.retrieve_account_groups():
1764             if g not in self.initial_groups:
1765                 l.append(g)
1766         self.client.unset_account_groups(l)
1767
1768         BaseTestCase.tearDown(self)
1769
1770     def test_read_access(self):
1771         self.client.create_container('c')
1772         o = self.upload_random_data('c', 'o')
1773         self.client.share_object('c', 'o', ['test:pithosdev'])
1774         for token, account in OTHER_ACCOUNTS.items():
1775             cl = Pithos_Client(get_server(), token, account, get_api())
1776             if account in self.authorized:
1777                 self.assert_not_raises_fault(401, cl.retrieve_object_metadata,
1778                                              'c', 'o', account=get_user())
1779             else:
1780                 self.assert_raises_fault(401, cl.retrieve_object_metadata,
1781                                          'c', 'o', account=get_user())
1782
1783
1784     def test_write_access(self):
1785         self.client.create_container('c')
1786         o = self.upload_random_data('c', 'o')
1787         self.client.share_object('c', 'o', ['chazapis'], read=False)
1788         for token, account in OTHER_ACCOUNTS.items():
1789             cl = Pithos_Client(get_server(), token, account, get_api())
1790             new_data = get_random_data()
1791             if account == 'chazapis':
1792                 self.assert_not_raises_fault(401, cl.update_object,
1793                                              'c', 'o', StringIO(new_data),
1794                                              account=get_user())
1795                 server_data = self.client.retrieve_object('c', 'o')
1796                 self.assertEqual(o['data'], server_data[:len(o['data'])])
1797                 self.assertEqual(new_data, server_data[len(o['data']):])
1798             else:
1799                 self.assert_raises_fault(401, cl.update_object,
1800                                              'c', 'o', StringIO(new_data),
1801                                              account=get_user())
1802
1803 class AssertMappingInvariant(object):
1804     def __init__(self, callable, *args, **kwargs):
1805         self.callable = callable
1806         self.args = args
1807         self.kwargs = kwargs
1808
1809     def __enter__(self):
1810         self.map = self.callable(*self.args, **self.kwargs)
1811         return self.map
1812
1813     def __exit__(self, type, value, tb):
1814         map = self.callable(*self.args, **self.kwargs)
1815         for k in self.map.keys():
1816             if is_date(self.map[k]):
1817                 continue
1818             assert map[k] == self.map[k]
1819
1820 class AssertContentInvariant(object):
1821     def __init__(self, callable, *args, **kwargs):
1822         self.callable = callable
1823         self.args = args
1824         self.kwargs = kwargs
1825
1826     def __enter__(self):
1827         self.content = self.callable(*self.args, **self.kwargs)[2]
1828         return self.content
1829
1830     def __exit__(self, type, value, tb):
1831         content = self.callable(*self.args, **self.kwargs)[2]
1832         assert self.content == content
1833
1834 def get_content_splitted(response):
1835     if response:
1836         return response.content.split('\n')
1837
1838 def compute_md5_hash(data):
1839     md5 = hashlib.md5()
1840     offset = 0
1841     md5.update(data)
1842     return md5.hexdigest().lower()
1843
1844 def compute_block_hash(data, algorithm):
1845     h = hashlib.new(algorithm)
1846     h.update(data.rstrip('\x00'))
1847     return h.hexdigest()
1848
1849 def get_random_data(length=500):
1850     char_set = string.ascii_uppercase + string.digits
1851     return ''.join(random.choice(char_set) for x in range(length))
1852
1853 def is_date(date):
1854     MONTHS = 'jan feb mar apr may jun jul aug sep oct nov dec'.split()
1855     __D = r'(?P<day>\d{2})'
1856     __D2 = r'(?P<day>[ \d]\d)'
1857     __M = r'(?P<mon>\w{3})'
1858     __Y = r'(?P<year>\d{4})'
1859     __Y2 = r'(?P<year>\d{2})'
1860     __T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
1861     RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
1862     RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
1863     ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
1864     for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
1865         m = regex.match(date)
1866         if m is not None:
1867             return True
1868     return False
1869
1870 o_names = ['kate.jpg',
1871            'kate_beckinsale.jpg',
1872            'How To Win Friends And Influence People.pdf',
1873            'moms_birthday.jpg',
1874            'poodle_strut.mov',
1875            'Disturbed - Down With The Sickness.mp3',
1876            'army_of_darkness.avi',
1877            'the_mad.avi',
1878            'photos/animals/dogs/poodle.jpg',
1879            'photos/animals/dogs/terrier.jpg',
1880            'photos/animals/cats/persian.jpg',
1881            'photos/animals/cats/siamese.jpg',
1882            'photos/plants/fern.jpg',
1883            'photos/plants/rose.jpg',
1884            'photos/me.jpg']
1885
1886 if __name__ == "__main__":
1887     unittest.main()