code.grnet.gr Git - pithos/blob - snf-pithos-tools/pithos/tools/test.py

   1 #!/usr/bin/env python
   2 #coding=utf8
   3
   4 # Copyright 2011-2012 GRNET S.A. All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or
   7 # without modification, are permitted provided that the following
   8 # conditions are met:
   9 #
  10 #   1. Redistributions of source code must retain the above
  11 #      copyright notice, this list of conditions and the following
  12 #      disclaimer.
  13 #
  14 #   2. Redistributions in binary form must reproduce the above
  15 #      copyright notice, this list of conditions and the following
  16 #      disclaimer in the documentation and/or other materials
  17 #      provided with the distribution.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  20 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  23 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  26 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  27 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  29 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 # POSSIBILITY OF SUCH DAMAGE.
  31 #
  32 # The views and conclusions contained in the software and
  33 # documentation are those of the authors and should not be
  34 # interpreted as representing official policies, either expressed
  35 # or implied, of GRNET S.A.
  36
  37 from pithos.tools.lib.client import Pithos_Client, Fault
  38 from pithos.tools.lib.util import get_user, get_auth, get_url
  39
  40 from xml.dom import minidom
  41 from StringIO import StringIO
  42 from hashlib import new as newhasher
  43 from binascii import hexlify
  44 from httplib import HTTPConnection, HTTPSConnection
  45 from urlparse import urlparse
  46
  47 import json
  48 import unittest
  49 import time as _time
  50 import types
  51 import hashlib
  52 import mimetypes
  53 import random
  54 import datetime
  55 import string
  56 import re
  57
  58 DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
  59                 "%A, %d-%b-%y %H:%M:%S GMT",
  60                 "%a, %d %b %Y %H:%M:%S GMT"]
  61
  62 OTHER_ACCOUNTS = {
  63     '0001': 'verigak',
  64     '0002': 'chazapis',
  65     '0003': 'gtsouk',
  66     '0004': 'papagian',
  67     '0005': 'louridas',
  68     '0006': 'chstath',
  69     '0007': 'pkanavos',
  70     '0008': 'mvasilak',
  71     '0009': 'διογένης'}
  72
  73 class BaseTestCase(unittest.TestCase):
  74     #TODO unauthorized request
  75     def setUp(self):
  76         self.client = Pithos_Client(get_url(), get_auth(), get_user())
  77         self._clean_account()
  78         self.invalid_client = Pithos_Client(get_url(), get_auth(), 'invalid')
  79
  80         #keep track of initial account groups
  81         self.initial_groups = self.client.retrieve_account_groups()
  82
  83         #keep track of initial account meta
  84         self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
  85
  86         self.extended = {
  87             'container':(
  88                 'name',
  89                 'count',
  90                 'bytes',
  91                 'last_modified',
  92                 'x_container_policy'),
  93             'object':(
  94                 'name',
  95                 'hash',
  96                 'bytes',
  97                 'content_type',
  98                 'content_encoding',
  99                 'last_modified',)}
 100         self.return_codes = (400, 401, 403, 404, 503,)
 101
 102     def tearDown(self):
 103         #delete additionally created meta
 104         l = []
 105         for m in self.client.retrieve_account_metadata(restricted=True):
 106             if m not in self.initial_meta:
 107                 l.append(m)
 108         self.client.delete_account_metadata(l)
 109
 110         #delete additionally created groups
 111         l = []
 112         for g in self.client.retrieve_account_groups():
 113             if g not in self.initial_groups:
 114                 l.append(g)
 115         self.client.unset_account_groups(l)
 116         self._clean_account()
 117
 118     def _clean_account(self):
 119         for c in self.client.list_containers():
 120             while True:
 121                 #list objects returns at most 10000 objects
 122                 #so repeat until there are no more objects
 123                 objects = self.client.list_objects(c)
 124                 if not objects:
 125                     break
 126                 for o in objects:
 127                     self.client.delete_object(c, o)
 128             self.client.delete_container(c)
 129
 130     def assert_status(self, status, codes):
 131         l = [elem for elem in self.return_codes]
 132         if type(codes) == types.ListType:
 133             l.extend(codes)
 134         else:
 135             l.append(codes)
 136         self.assertTrue(status in l)
 137
 138     def assert_extended(self, data, format, type, size=10000):
 139         if format == 'xml':
 140             self._assert_xml(data, type, size)
 141         elif format == 'json':
 142             self._assert_json(data, type, size)
 143
 144     def _assert_json(self, data, type, size):
 145         convert = lambda s: s.lower()
 146         info = [convert(elem) for elem in self.extended[type]]
 147         self.assertTrue(len(data) <= size)
 148         for item in info:
 149             for i in data:
 150                 if 'subdir' in i.keys():
 151                     continue
 152                 self.assertTrue(item in i.keys())
 153
 154     def _assert_xml(self, data, type, size):
 155         convert = lambda s: s.lower()
 156         info = [convert(elem) for elem in self.extended[type]]
 157         try:
 158             info.remove('content_encoding')
 159         except ValueError:
 160             pass
 161         xml = data
 162         entities = xml.getElementsByTagName(type)
 163         self.assertTrue(len(entities) <= size)
 164         for e in entities:
 165             for item in info:
 166                 self.assertTrue(e.getElementsByTagName(item))
 167
 168     def assert_raises_fault(self, status, callableObj, *args, **kwargs):
 169         """
 170         asserts that a Fault with a specific status is raised
 171         when callableObj is called with the specific arguments
 172         """
 173         try:
 174             r = callableObj(*args, **kwargs)
 175             self.fail('Should never reach here')
 176         except Fault, f:
 177             if type(status) == types.ListType:
 178                 self.failUnless(f.status in status)
 179             else:
 180                 self.failUnless(f.status == status)
 181
 182     def assert_not_raises_fault(self, status, callableObj, *args, **kwargs):
 183         """
 184         asserts that a Fault with a specific status is not raised
 185         when callableObj is called with the specific arguments
 186         """
 187         try:
 188             r = callableObj(*args, **kwargs)
 189         except Fault, f:
 190             self.failIfEqual(f.status, status)
 191
 192     def assert_container_exists(self, container):
 193         """
 194         asserts the existence of a container
 195         """
 196         try:
 197             self.client.retrieve_container_metadata(container)
 198         except Fault, f:
 199             self.failIf(f.status == 404)
 200
 201     def assert_container_not_exists(self, container):
 202         """
 203         asserts there is no such a container
 204         """
 205         self.assert_raises_fault(404, self.client.retrieve_container_metadata,
 206                                  container)
 207
 208     def assert_object_exists(self, container, object):
 209         """
 210         asserts the existence of an object
 211         """
 212         try:
 213             self.client.retrieve_object_metadata(container, object)
 214         except Fault, f:
 215             self.failIf(f.status == 404)
 216
 217     def assert_object_not_exists(self, container, object):
 218         """
 219         asserts there is no such an object
 220         """
 221         self.assert_raises_fault(404, self.client.retrieve_object_metadata,
 222                                  container, object)
 223
 224     def assert_versionlist_structure(self, versionlist):
 225         self.assertTrue(type(versionlist) == types.ListType)
 226         for elem in versionlist:
 227             self.assertTrue(type(elem) == types.ListType)
 228             self.assertEqual(len(elem), 2)
 229
 230     def upload_random_data(self, container, name, length=1024, type=None,
 231                            enc=None, **meta):
 232         data = get_random_data(length)
 233         return self.upload_data(container, name, data, type, enc, **meta)
 234
 235     def upload_data(self, container, name, data, type=None, enc=None, etag=None,
 236                     **meta):
 237         obj = {}
 238         obj['name'] = name
 239         try:
 240             obj['data'] = data
 241             obj['hash'] = compute_md5_hash(obj['data'])
 242
 243             args = {}
 244             args['etag'] = etag if etag else obj['hash']
 245
 246             try:
 247                 guess = mimetypes.guess_type(name)
 248                 type = type if type else guess[0]
 249                 enc = enc if enc else guess[1]
 250             except:
 251                 pass
 252             args['content_type'] = type if type else 'plain/text'
 253             args['content_encoding'] = enc if enc else None
 254
 255             obj['meta'] = args
 256
 257             path = '/%s/%s' % (container, name)
 258             self.client.create_object(container, name, f=StringIO(obj['data']),
 259                                       meta=meta, **args)
 260
 261             return obj
 262         except IOError:
 263             return
 264
 265 class AccountHead(BaseTestCase):
 266     def setUp(self):
 267         BaseTestCase.setUp(self)
 268         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 269         for item in self.containers:
 270             self.client.create_container(item)
 271
 272         meta = {'foo':'bar'}
 273         self.client.update_account_metadata(**meta)
 274         #self.updated_meta = self.initial_meta.update(meta)
 275
 276     def test_get_account_meta(self):
 277         meta = self.client.retrieve_account_metadata()
 278
 279         containers = self.client.list_containers()
 280         l = str(len(containers))
 281         self.assertEqual(meta['x-account-container-count'], l)
 282         size = 0
 283         for c in containers:
 284             m = self.client.retrieve_container_metadata(c)
 285             size = size + int(m['x-container-bytes-used'])
 286         self.assertEqual(meta['x-account-bytes-used'], str(size))
 287
 288     def test_get_account_403(self):
 289         self.assert_raises_fault(403,
 290                                  self.invalid_client.retrieve_account_metadata)
 291
 292     def test_get_account_meta_until(self):
 293         t = datetime.datetime.utcnow()
 294         past = t - datetime.timedelta(minutes=-15)
 295         past = int(_time.mktime(past.timetuple()))
 296
 297         meta = {'premium':True}
 298         self.client.update_account_metadata(**meta)
 299         meta = self.client.retrieve_account_metadata(restricted=True,
 300                                                      until=past)
 301         self.assertTrue('premium' not in meta)
 302
 303         meta = self.client.retrieve_account_metadata(restricted=True)
 304         self.assertTrue('premium' in meta)
 305
 306     def test_get_account_meta_until_invalid_date(self):
 307         meta = {'premium':True}
 308         self.client.update_account_metadata(**meta)
 309         meta = self.client.retrieve_account_metadata(restricted=True,
 310                                                      until='kshfksfh')
 311         self.assertTrue('premium' in meta)
 312
 313 class AccountGet(BaseTestCase):
 314     def setUp(self):
 315         BaseTestCase.setUp(self)
 316         #create some containers
 317         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 318         for item in self.containers:
 319             self.client.create_container(item)
 320
 321     def test_list(self):
 322         #list containers
 323         containers = self.client.list_containers()
 324         self.assertEquals(self.containers, containers)
 325
 326     def test_list_403(self):
 327         self.assert_raises_fault(403, self.invalid_client.list_containers)
 328
 329     def test_list_with_limit(self):
 330         limit = 2
 331         containers = self.client.list_containers(limit=limit)
 332         self.assertEquals(len(containers), limit)
 333         self.assertEquals(self.containers[:2], containers)
 334
 335     def test_list_with_marker(self):
 336         l = 2
 337         m = 'bananas'
 338         containers = self.client.list_containers(limit=l, marker=m)
 339         i = self.containers.index(m) + 1
 340         self.assertEquals(self.containers[i:(i+l)], containers)
 341
 342         m = 'oranges'
 343         containers = self.client.list_containers(limit=l, marker=m)
 344         i = self.containers.index(m) + 1
 345         self.assertEquals(self.containers[i:(i+l)], containers)
 346
 347     def test_list_json_with_marker(self):
 348         l = 2
 349         m = 'bananas'
 350         containers = self.client.list_containers(limit=l, marker=m, format='json')
 351         self.assert_extended(containers, 'json', 'container', l)
 352         self.assertEqual(containers[0]['name'], 'kiwis')
 353         self.assertEqual(containers[1]['name'], 'oranges')
 354
 355     def test_list_xml_with_marker(self):
 356         l = 2
 357         m = 'oranges'
 358         xml = self.client.list_containers(limit=l, marker=m, format='xml')
 359         self.assert_extended(xml, 'xml', 'container', l)
 360         nodes = xml.getElementsByTagName('name')
 361         self.assertEqual(len(nodes), 1)
 362         self.assertEqual(nodes[0].childNodes[0].data, 'pears')
 363
 364     def test_if_modified_since(self):
 365         t = datetime.datetime.utcnow()
 366         t2 = t - datetime.timedelta(minutes=10)
 367
 368         #add a new container
 369         self.client.create_container('dummy')
 370
 371         for f in DATE_FORMATS:
 372             past = t2.strftime(f)
 373             try:
 374                 c = self.client.list_containers(if_modified_since=past)
 375                 self.assertEqual(len(c), len(self.containers) + 1)
 376             except Fault, f:
 377                 self.failIf(f.status == 304) #fail if not modified
 378
 379
 380     def test_if_modified_since_invalid_date(self):
 381         c = self.client.list_containers(if_modified_since='')
 382         self.assertEqual(len(c), len(self.containers))
 383
 384     def test_if_not_modified_since(self):
 385         now = datetime.datetime.utcnow()
 386         since = now + datetime.timedelta(1)
 387
 388         for f in DATE_FORMATS:
 389             args = {'if_modified_since':'%s' %since.strftime(f)}
 390
 391             #assert not modified
 392             self.assert_raises_fault(304, self.client.list_containers, **args)
 393
 394     def test_if_unmodified_since(self):
 395         now = datetime.datetime.utcnow()
 396         since = now + datetime.timedelta(1)
 397
 398         for f in DATE_FORMATS:
 399             c = self.client.list_containers(if_unmodified_since=since.strftime(f))
 400
 401             #assert success
 402             self.assertEqual(self.containers, c)
 403
 404     def test_if_unmodified_since_precondition_failed(self):
 405         t = datetime.datetime.utcnow()
 406         t2 = t - datetime.timedelta(minutes=10)
 407
 408         #add a new container
 409         self.client.create_container('dummy')
 410
 411         for f in DATE_FORMATS:
 412             past = t2.strftime(f)
 413
 414             args = {'if_unmodified_since':'%s' %past}
 415
 416             #assert precondition failed
 417             self.assert_raises_fault(412, self.client.list_containers, **args)
 418
 419 class AccountPost(BaseTestCase):
 420     def setUp(self):
 421         BaseTestCase.setUp(self)
 422         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 423         for item in self.containers:
 424             self.client.create_container(item)
 425
 426         meta = {'foo':'bar'}
 427         self.client.update_account_metadata(**meta)
 428         self.updated_meta = self.initial_meta.update(meta)
 429
 430     def test_update_meta(self):
 431         with AssertMappingInvariant(self.client.retrieve_account_groups):
 432             meta = {'test':'test', 'tost':'tost'}
 433             self.client.update_account_metadata(**meta)
 434
 435             meta.update(self.initial_meta)
 436             self.assertEqual(meta,
 437                              self.client.retrieve_account_metadata(
 438                                 restricted=True))
 439
 440     def test_invalid_account_update_meta(self):
 441         meta = {'test':'test', 'tost':'tost'}
 442         self.assert_raises_fault(403,
 443                                  self.invalid_client.update_account_metadata,
 444                                  **meta)
 445
 446     def test_reset_meta(self):
 447         with AssertMappingInvariant(self.client.retrieve_account_groups):
 448             meta = {'test':'test', 'tost':'tost'}
 449             self.client.update_account_metadata(**meta)
 450
 451             meta = {'test':'test33'}
 452             self.client.reset_account_metadata(**meta)
 453
 454             self.assertEqual(meta, self.client.retrieve_account_metadata(restricted=True))
 455
 456     def test_delete_meta(self):
 457         with AssertMappingInvariant(self.client.retrieve_account_groups):
 458             meta = {'test':'test', 'tost':'tost'}
 459             self.client.update_account_metadata(**meta)
 460
 461             self.client.delete_account_metadata(meta.keys())
 462
 463             account_meta = self.client.retrieve_account_metadata(restricted=True)
 464             for m in meta:
 465                 self.assertTrue(m not in account_meta.keys())
 466
 467     def test_set_account_groups(self):
 468         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 469             groups = {'pithosdev':'verigak,gtsouk,chazapis'}
 470             self.client.set_account_groups(**groups)
 471
 472             self.assertEqual(set(groups['pithosdev']),
 473                              set(self.client.retrieve_account_groups()['pithosdev']))
 474
 475             more_groups = {'clientsdev':'pkanavos,mvasilak'}
 476             self.client.set_account_groups(**more_groups)
 477
 478             groups.update(more_groups)
 479             self.assertEqual(set(groups['clientsdev']),
 480                              set(self.client.retrieve_account_groups()['clientsdev']))
 481
 482     def test_reset_account_groups(self):
 483         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 484             groups = {'pithosdev':'verigak,gtsouk,chazapis',
 485                       'clientsdev':'pkanavos,mvasilak'}
 486             self.client.set_account_groups(**groups)
 487
 488             self.assertEqual(set(groups['pithosdev'].split(',')),
 489                              set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
 490             self.assertEqual(set(groups['clientsdev'].split(',')),
 491                              set(self.client.retrieve_account_groups()['clientsdev'].split(',')))
 492
 493             groups = {'pithosdev':'verigak,gtsouk,chazapis,papagian'}
 494             self.client.reset_account_groups(**groups)
 495
 496             self.assertEqual(set(groups['pithosdev'].split(',')),
 497                              set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
 498
 499     def test_delete_account_groups(self):
 500         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 501             groups = {'pithosdev':'verigak,gtsouk,chazapis',
 502                       'clientsdev':'pkanavos,mvasilak'}
 503             self.client.set_account_groups(**groups)
 504
 505             self.client.unset_account_groups(groups.keys())
 506
 507             self.assertEqual({}, self.client.retrieve_account_groups())
 508
 509 class ContainerHead(BaseTestCase):
 510     def setUp(self):
 511         BaseTestCase.setUp(self)
 512         self.container = 'apples'
 513         self.client.create_container(self.container)
 514
 515     def test_get_meta(self):
 516         meta = {'trash':'true'}
 517         t1 = datetime.datetime.utcnow()
 518         o = self.upload_random_data(self.container, o_names[0], **meta)
 519         if o:
 520             headers = self.client.retrieve_container_metadata(self.container)
 521             self.assertEqual(headers['x-container-object-count'], '1')
 522             self.assertEqual(headers['x-container-bytes-used'], str(len(o['data'])))
 523             t2 = datetime.datetime.strptime(headers['last-modified'], DATE_FORMATS[2])
 524             delta = (t2 - t1)
 525             threashold = datetime.timedelta(seconds=1)
 526             self.assertTrue(delta < threashold)
 527             self.assertTrue(headers['x-container-object-meta'])
 528             self.assertTrue('Trash' in headers['x-container-object-meta'])
 529
 530 class ContainerGet(BaseTestCase):
 531     def setUp(self):
 532         BaseTestCase.setUp(self)
 533         self.container = ['pears', 'apples']
 534         for c in self.container:
 535             self.client.create_container(c)
 536         self.obj = []
 537         for o in o_names[:8]:
 538             self.obj.append(self.upload_random_data(self.container[0], o))
 539         for o in o_names[8:]:
 540             self.obj.append(self.upload_random_data(self.container[1], o))
 541
 542     def test_list_objects(self):
 543         objects = self.client.list_objects(self.container[0])
 544         l = [elem['name'] for elem in self.obj[:8]]
 545         l.sort()
 546         self.assertEqual(objects, l)
 547
 548     def test_list_objects_containing_slash(self):
 549         self.client.create_container('test')
 550         self.upload_random_data('test', '/objectname')
 551
 552         objects = self.client.list_objects('test')
 553         self.assertEqual(objects, ['/objectname'])
 554
 555         objects = self.client.list_objects('test', format='json')
 556         self.assertEqual(objects[0]['name'], '/objectname')
 557
 558         objects = self.client.list_objects('test', format='xml')
 559         self.assert_extended(objects, 'xml', 'object')
 560         node_name = objects.getElementsByTagName('name')[0]
 561         self.assertEqual(node_name.firstChild.data, '/objectname')
 562
 563     def test_list_objects_with_limit_marker(self):
 564         objects = self.client.list_objects(self.container[0], limit=2)
 565         l = [elem['name'] for elem in self.obj[:8]]
 566         l.sort()
 567         self.assertEqual(objects, l[:2])
 568
 569         markers = ['How To Win Friends And Influence People.pdf',
 570                    'moms_birthday.jpg']
 571         limit = 4
 572         for m in markers:
 573             objects = self.client.list_objects(self.container[0], limit=limit,
 574                                                marker=m)
 575             l = [elem['name'] for elem in self.obj[:8]]
 576             l.sort()
 577             start = l.index(m) + 1
 578             end = start + limit
 579             end = end if len(l) >= end else len(l)
 580             self.assertEqual(objects, l[start:end])
 581
 582     #takes too long
 583     def _test_list_limit_exceeds(self):
 584         self.client.create_container('pithos')
 585
 586         for i in range(10001):
 587             self.client.create_zero_length_object('pithos', i)
 588
 589         self.assertEqual(10000, len(self.client.list_objects('pithos')))
 590
 591     def test_list_empty_params(self):
 592         objects = self.client.get('/%s/%s' % (get_user(), self.container[0]))[2]
 593         if objects:
 594             objects = objects.strip().split('\n')
 595         self.assertEqual(objects,
 596                          self.client.list_objects(self.container[0]))
 597
 598     def test_list_pseudo_hierarchical_folders(self):
 599         objects = self.client.list_objects(self.container[1], prefix='photos',
 600                                            delimiter='/')
 601         self.assertEquals(['photos/animals/', 'photos/me.jpg',
 602                            'photos/plants/'], objects)
 603
 604         objects = self.client.list_objects(self.container[1],
 605                                            prefix='photos/animals',
 606                                            delimiter='/')
 607         l = ['photos/animals/cats/', 'photos/animals/dogs/']
 608         self.assertEquals(l, objects)
 609
 610         objects = self.client.list_objects(self.container[1], path='photos')
 611         self.assertEquals(['photos/me.jpg'], objects)
 612
 613     def test_extended_list_json(self):
 614         objects = self.client.list_objects(self.container[1], format='json',
 615                                            limit=2, prefix='photos/animals',
 616                                            delimiter='/')
 617         self.assertEqual(objects[0]['subdir'], 'photos/animals/cats/')
 618         self.assertEqual(objects[1]['subdir'], 'photos/animals/dogs/')
 619
 620     def test_extended_list_xml(self):
 621         xml = self.client.list_objects(self.container[1], format='xml', limit=4,
 622                                        prefix='photos', delimiter='/')
 623         self.assert_extended(xml, 'xml', 'object', size=4)
 624         dirs = xml.getElementsByTagName('subdir')
 625         self.assertEqual(len(dirs), 2)
 626         self.assertEqual(dirs[0].attributes['name'].value, 'photos/animals/')
 627         self.assertEqual(dirs[1].attributes['name'].value, 'photos/plants/')
 628
 629         objects = xml.getElementsByTagName('name')
 630         self.assertEqual(len(objects), 1)
 631         self.assertEqual(objects[0].childNodes[0].data, 'photos/me.jpg')
 632
 633     def test_list_meta_double_matching(self):
 634         meta = {'quality':'aaa', 'stock':'true'}
 635         self.client.update_object_metadata(self.container[0],
 636                                            self.obj[0]['name'], **meta)
 637         obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
 638         self.assertEqual(len(obj), 1)
 639         self.assertTrue(obj, self.obj[0]['name'])
 640
 641     def test_list_using_meta(self):
 642         meta = {'quality':'aaa'}
 643         for o in self.obj[:2]:
 644             self.client.update_object_metadata(self.container[0], o['name'],
 645                                                **meta)
 646         meta = {'stock':'true'}
 647         for o in self.obj[3:5]:
 648             self.client.update_object_metadata(self.container[0], o['name'],
 649                                                **meta)
 650
 651         obj = self.client.list_objects(self.container[0], meta='Quality')
 652         self.assertEqual(len(obj), 2)
 653         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 654
 655         # test case insensitive
 656         obj = self.client.list_objects(self.container[0], meta='quality')
 657         self.assertEqual(len(obj), 2)
 658         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 659
 660         # test multiple matches
 661         obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
 662         self.assertEqual(len(obj), 4)
 663         self.assertTrue(obj, [o['name'] for o in self.obj[:4]])
 664
 665         # test non 1-1 multiple match
 666         obj = self.client.list_objects(self.container[0], meta='Quality,aaaa')
 667         self.assertEqual(len(obj), 2)
 668         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 669
 670     def test_if_modified_since(self):
 671         t = datetime.datetime.utcnow()
 672         t2 = t - datetime.timedelta(minutes=10)
 673
 674         #add a new object
 675         self.upload_random_data(self.container[0], o_names[0])
 676
 677         for f in DATE_FORMATS:
 678             past = t2.strftime(f)
 679             try:
 680                 o = self.client.list_objects(self.container[0],
 681                                             if_modified_since=past)
 682                 self.assertEqual(o,
 683                                  self.client.list_objects(self.container[0]))
 684             except Fault, f:
 685                 self.failIf(f.status == 304) #fail if not modified
 686
 687     def test_if_modified_since_invalid_date(self):
 688         headers = {'if-modified-since':''}
 689         o = self.client.list_objects(self.container[0], if_modified_since='')
 690         self.assertEqual(o, self.client.list_objects(self.container[0]))
 691
 692     def test_if_not_modified_since(self):
 693         now = datetime.datetime.utcnow()
 694         since = now + datetime.timedelta(1)
 695
 696         for f in DATE_FORMATS:
 697             args = {'if_modified_since':'%s' %since.strftime(f)}
 698
 699             #assert not modified
 700             self.assert_raises_fault(304, self.client.list_objects,
 701                                      self.container[0], **args)
 702
 703     def test_if_unmodified_since(self):
 704         now = datetime.datetime.utcnow()
 705         since = now + datetime.timedelta(1)
 706
 707         for f in DATE_FORMATS:
 708             obj = self.client.list_objects(self.container[0],
 709                                            if_unmodified_since=since.strftime(f))
 710
 711             #assert unmodified
 712             self.assertEqual(obj, self.client.list_objects(self.container[0]))
 713
 714     def test_if_unmodified_since_precondition_failed(self):
 715         t = datetime.datetime.utcnow()
 716         t2 = t - datetime.timedelta(minutes=10)
 717
 718         #add a new container
 719         self.client.create_container('dummy')
 720
 721         for f in DATE_FORMATS:
 722             past = t2.strftime(f)
 723
 724             args = {'if_unmodified_since':'%s' %past}
 725
 726             #assert precondition failed
 727             self.assert_raises_fault(412, self.client.list_objects,
 728                                      self.container[0], **args)
 729
 730 class ContainerPut(BaseTestCase):
 731     def setUp(self):
 732         BaseTestCase.setUp(self)
 733         self.containers = ['c1', 'c2']
 734
 735     def test_create(self):
 736         self.client.create_container(self.containers[0])
 737         containers = self.client.list_containers()
 738         self.assertTrue(self.containers[0] in containers)
 739         self.assert_container_exists(self.containers[0])
 740
 741     def test_create_twice(self):
 742         self.client.create_container(self.containers[0])
 743         self.assertTrue(not self.client.create_container(self.containers[0]))
 744
 745     def test_quota(self):
 746         self.client.create_container(self.containers[0])
 747
 748         policy = {'quota':100}
 749         self.client.set_container_policies('c1', **policy)
 750
 751         meta = self.client.retrieve_container_metadata('c1')
 752         self.assertTrue('x-container-policy-quota' in meta)
 753         self.assertEqual(meta['x-container-policy-quota'], '100')
 754
 755         args = ['c1', 'o1']
 756         kwargs = {'length':101}
 757         self.assert_raises_fault(413, self.upload_random_data, *args, **kwargs)
 758
 759         #reset quota
 760         policy = {'quota':0}
 761         self.client.set_container_policies('c1', **policy)
 762
 763 class ContainerPost(BaseTestCase):
 764     def setUp(self):
 765         BaseTestCase.setUp(self)
 766         self.container = 'apples'
 767         self.client.create_container(self.container)
 768
 769     def test_update_meta(self):
 770         meta = {'test':'test33',
 771                 'tost':'tost22'}
 772         self.client.update_container_metadata(self.container, **meta)
 773         headers = self.client.retrieve_container_metadata(self.container)
 774         for k,v in meta.items():
 775             k = 'x-container-meta-%s' % k
 776             self.assertTrue(headers[k])
 777             self.assertEqual(headers[k], v)
 778
 779 class ContainerDelete(BaseTestCase):
 780     def setUp(self):
 781         BaseTestCase.setUp(self)
 782         self.containers = ['c1', 'c2']
 783         for c in self.containers:
 784             self.client.create_container(c)
 785
 786     def test_delete(self):
 787         status = self.client.delete_container(self.containers[0])[0]
 788         self.assertEqual(status, 204)
 789
 790     def test_delete_non_empty(self):
 791         self.upload_random_data(self.containers[1], o_names[0])
 792         self.assert_raises_fault(409, self.client.delete_container,
 793                                  self.containers[1])
 794
 795     def test_delete_invalid(self):
 796         self.assert_raises_fault(404, self.client.delete_container, 'c3')
 797
 798 class ObjectGet(BaseTestCase):
 799     def setUp(self):
 800         BaseTestCase.setUp(self)
 801         self.containers = ['c1', 'c2']
 802         #create some containers
 803         for c in self.containers:
 804             self.client.create_container(c)
 805
 806         #upload a file
 807         names = ('obj1', 'obj2')
 808         self.objects = []
 809         for n in names:
 810             self.objects.append(self.upload_random_data(self.containers[1], n))
 811
 812     def test_versions(self):
 813         c = self.containers[1]
 814         o = self.objects[0]
 815         b = self.client.retrieve_object_versionlist(c, o['name'])['versions']
 816         self.assert_versionlist_structure(b)
 817
 818         #update meta
 819         meta = {'quality':'AAA', 'stock':True}
 820         self.client.update_object_metadata(c, o['name'], **meta)
 821
 822         a = self.client.retrieve_object_versionlist(c, o['name'])['versions']
 823         self.assert_versionlist_structure(a)
 824         self.assertEqual(len(b)+1, len(a))
 825         self.assertEqual(b, a[:-1])
 826
 827         #get exact previous version metadata
 828         v = a[-2][0]
 829         v_meta = self.client.retrieve_object_metadata(c, o['name'],
 830                                                       restricted=True,
 831                                                       version=v)
 832         for k in meta.keys():
 833             self.assertTrue(k not in v_meta)
 834
 835         #update obejct
 836         data = get_random_data()
 837         self.client.update_object(c, o['name'], StringIO(data))
 838
 839         aa = self.client.retrieve_object_versionlist(c, o['name'])['versions']
 840         self.assert_versionlist_structure(aa)
 841         self.assertEqual(len(a)+1, len(aa))
 842         self.assertEqual(a, aa[:-1])
 843
 844         #get exact previous version
 845         v = aa[-3][0]
 846         v_data = self.client.retrieve_object_version(c, o['name'], version=v)
 847         self.assertEqual(o['data'], v_data)
 848         self.assertEqual(self.client.retrieve_object(c, o['name']),
 849                          '%s%s' %(v_data, data))
 850
 851     def test_get(self):
 852         #perform get
 853         o = self.client.retrieve_object(self.containers[1],
 854                                         self.objects[0]['name'],
 855                                         self.objects[0]['meta'])
 856         self.assertEqual(o, self.objects[0]['data'])
 857
 858     def test_objects_with_trailing_spaces(self):
 859         self.client.create_container('test')
 860         #create 'a' object
 861         self.upload_random_data('test', 'a')
 862         #look for 'a ' object
 863         self.assert_raises_fault(404, self.client.retrieve_object,
 864                                  'test', 'a ')
 865
 866         #delete 'a' object
 867         self.client.delete_object('test', 'a')
 868         self.assert_raises_fault(404, self.client.retrieve_object,
 869                                  'test', 'a')
 870
 871         #create 'a ' object
 872         self.upload_random_data('test', 'a ')
 873         #look for 'a' object
 874         self.assert_raises_fault(404, self.client.retrieve_object,
 875                                  'test', 'a')
 876
 877     def test_get_invalid(self):
 878         self.assert_raises_fault(404, self.client.retrieve_object,
 879                                  self.containers[0], self.objects[0]['name'])
 880
 881     def test_get_partial(self):
 882         #perform get with range
 883         status, headers, data = self.client.request_object(self.containers[1],
 884                                                             self.objects[0]['name'],
 885                                                             range='bytes=0-499')
 886
 887         #assert successful partial content
 888         self.assertEqual(status, 206)
 889
 890         #assert content-type
 891         self.assertEqual(headers['content-type'],
 892                          self.objects[0]['meta']['content_type'])
 893
 894         #assert content length
 895         self.assertEqual(int(headers['content-length']), 500)
 896
 897         #assert content
 898         self.assertEqual(self.objects[0]['data'][:500], data)
 899
 900     def test_get_final_500(self):
 901         #perform get with range
 902         headers = {'range':'bytes=-500'}
 903         status, headers, data = self.client.request_object(self.containers[1],
 904                                                             self.objects[0]['name'],
 905                                                             range='bytes=-500')
 906
 907         #assert successful partial content
 908         self.assertEqual(status, 206)
 909
 910         #assert content-type
 911         self.assertEqual(headers['content-type'],
 912                          self.objects[0]['meta']['content_type'])
 913
 914         #assert content length
 915         self.assertEqual(int(headers['content-length']), 500)
 916
 917         #assert content
 918         self.assertTrue(self.objects[0]['data'][-500:], data)
 919
 920     def test_get_rest(self):
 921         #perform get with range
 922         offset = len(self.objects[0]['data']) - 500
 923         status, headers, data = self.client.request_object(self.containers[1],
 924                                                 self.objects[0]['name'],
 925                                                 range='bytes=%s-' %offset)
 926
 927         #assert successful partial content
 928         self.assertEqual(status, 206)
 929
 930         #assert content-type
 931         self.assertEqual(headers['content-type'],
 932                          self.objects[0]['meta']['content_type'])
 933
 934         #assert content length
 935         self.assertEqual(int(headers['content-length']), 500)
 936
 937         #assert content
 938         self.assertTrue(self.objects[0]['data'][-500:], data)
 939
 940     def test_get_range_not_satisfiable(self):
 941         #perform get with range
 942         offset = len(self.objects[0]['data']) + 1
 943
 944         #assert range not satisfiable
 945         self.assert_raises_fault(416, self.client.retrieve_object,
 946                                  self.containers[1], self.objects[0]['name'],
 947                                  range='bytes=0-%s' %offset)
 948
 949     def test_multiple_range(self):
 950         #perform get with multiple range
 951         ranges = ['0-499', '-500', '1000-']
 952         bytes = 'bytes=%s' % ','.join(ranges)
 953         status, headers, data = self.client.request_object(self.containers[1],
 954                                                            self.objects[0]['name'],
 955                                                            range=bytes)
 956
 957         # assert partial content
 958         self.assertEqual(status, 206)
 959
 960         # assert Content-Type of the reply will be multipart/byteranges
 961         self.assertTrue(headers['content-type'])
 962         content_type_parts = headers['content-type'].split()
 963         self.assertEqual(content_type_parts[0], ('multipart/byteranges;'))
 964
 965         boundary = '--%s' %content_type_parts[1].split('=')[-1:][0]
 966         cparts = data.split(boundary)[1:-1]
 967
 968         # assert content parts are exactly 2
 969         self.assertEqual(len(cparts), len(ranges))
 970
 971         # for each content part assert headers
 972         i = 0
 973         for cpart in cparts:
 974             content = cpart.split('\r\n')
 975             headers = content[1:3]
 976             content_range = headers[0].split(': ')
 977             self.assertEqual(content_range[0], 'Content-Range')
 978
 979             r = ranges[i].split('-')
 980             if not r[0] and not r[1]:
 981                 pass
 982             elif not r[0]:
 983                 start = len(self.objects[0]['data']) - int(r[1])
 984                 end = len(self.objects[0]['data'])
 985             elif not r[1]:
 986                 start = int(r[0])
 987                 end = len(self.objects[0]['data'])
 988             else:
 989                 start = int(r[0])
 990                 end = int(r[1]) + 1
 991             fdata = self.objects[0]['data'][start:end]
 992             sdata = '\r\n'.join(content[4:-1])
 993             self.assertEqual(len(fdata), len(sdata))
 994             self.assertEquals(fdata, sdata)
 995             i+=1
 996
 997     def test_multiple_range_not_satisfiable(self):
 998         #perform get with multiple range
 999         out_of_range = len(self.objects[0]['data']) + 1
1000         ranges = ['0-499', '-500', '%d-' %out_of_range]
1001         bytes = 'bytes=%s' % ','.join(ranges)
1002
1003         # assert partial content
1004         self.assert_raises_fault(416, self.client.retrieve_object,
1005                                  self.containers[1],
1006                                  self.objects[0]['name'], range=bytes)
1007
1008     def test_get_with_if_match(self):
1009         #perform get with If-Match
1010         etag = self.objects[0]['hash']
1011         status, headers, data = self.client.request_object(self.containers[1],
1012                                                            self.objects[0]['name'],
1013                                                            if_match=etag)
1014         #assert get success
1015         self.assertEqual(status, 200)
1016
1017         #assert content-type
1018         self.assertEqual(headers['content-type'],
1019                          self.objects[0]['meta']['content_type'])
1020
1021         #assert response content
1022         self.assertEqual(self.objects[0]['data'], data)
1023
1024     def test_get_with_if_match_star(self):
1025         #perform get with If-Match *
1026         headers = {'if-match':'*'}
1027         status, headers, data = self.client.request_object(self.containers[1],
1028                                                 self.objects[0]['name'],
1029                                                 **headers)
1030         #assert get success
1031         self.assertEqual(status, 200)
1032
1033         #assert content-type
1034         self.assertEqual(headers['content-type'],
1035                          self.objects[0]['meta']['content_type'])
1036
1037         #assert response content
1038         self.assertEqual(self.objects[0]['data'], data)
1039
1040     def test_get_with_multiple_if_match(self):
1041         #perform get with If-Match
1042         etags = [i['hash'] for i in self.objects if i]
1043         etags = ','.join('"%s"' % etag for etag in etags)
1044         status, headers, data = self.client.request_object(self.containers[1],
1045                                                            self.objects[0]['name'],
1046                                                            if_match=etags)
1047         #assert get success
1048         self.assertEqual(status, 200)
1049
1050         #assert content-type
1051         self.assertEqual(headers['content-type'],
1052                          self.objects[0]['meta']['content_type'])
1053
1054         #assert content-type
1055         self.assertEqual(headers['content-type'],
1056                          self.objects[0]['meta']['content_type'])
1057
1058         #assert response content
1059         self.assertEqual(self.objects[0]['data'], data)
1060
1061     def test_if_match_precondition_failed(self):
1062         #assert precondition failed
1063         self.assert_raises_fault(412, self.client.retrieve_object,
1064                                  self.containers[1],
1065                                  self.objects[0]['name'], if_match='123')
1066
1067     def test_if_none_match(self):
1068         #perform get with If-None-Match
1069         status, headers, data = self.client.request_object(self.containers[1],
1070                                                            self.objects[0]['name'],
1071                                                            if_none_match='123')
1072
1073         #assert get success
1074         self.assertEqual(status, 200)
1075
1076         #assert content-type
1077         self.assertEqual(headers['content_type'],
1078                          self.objects[0]['meta']['content_type'])
1079
1080     def test_if_none_match(self):
1081         #perform get with If-None-Match * and assert not modified
1082         self.assert_raises_fault(304, self.client.retrieve_object,
1083                                  self.containers[1],
1084                                  self.objects[0]['name'],
1085                                  if_none_match='*')
1086
1087     def test_if_none_match_not_modified(self):
1088         #perform get with If-None-Match and assert not modified
1089         self.assert_raises_fault(304, self.client.retrieve_object,
1090                                  self.containers[1],
1091                                  self.objects[0]['name'],
1092                                  if_none_match=self.objects[0]['hash'])
1093
1094         meta = self.client.retrieve_object_metadata(self.containers[1],
1095                                                     self.objects[0]['name'])
1096         self.assertEqual(meta['etag'], self.objects[0]['hash'])
1097
1098     def test_if_modified_since(self):
1099         t = datetime.datetime.utcnow()
1100         t2 = t - datetime.timedelta(minutes=10)
1101
1102         #modify the object
1103         self.upload_data(self.containers[1],
1104                            self.objects[0]['name'],
1105                            self.objects[0]['data'][:200])
1106
1107         for f in DATE_FORMATS:
1108             past = t2.strftime(f)
1109
1110             headers = {'if-modified-since':'%s' %past}
1111             try:
1112                 o = self.client.retrieve_object(self.containers[1],
1113                                                 self.objects[0]['name'],
1114                                                 if_modified_since=past)
1115                 self.assertEqual(o,
1116                                  self.client.retrieve_object(self.containers[1],
1117                                                              self.objects[0]['name']))
1118             except Fault, f:
1119                 self.failIf(f.status == 304)
1120
1121     def test_if_modified_since_invalid_date(self):
1122         o = self.client.retrieve_object(self.containers[1],
1123                                         self.objects[0]['name'],
1124                                         if_modified_since='')
1125         self.assertEqual(o, self.client.retrieve_object(self.containers[1],
1126                                                         self.objects[0]['name']))
1127
1128     def test_if_not_modified_since(self):
1129         now = datetime.datetime.utcnow()
1130         since = now + datetime.timedelta(1)
1131
1132         for f in DATE_FORMATS:
1133             #assert not modified
1134             self.assert_raises_fault(304, self.client.retrieve_object,
1135                                      self.containers[1], self.objects[0]['name'],
1136                                      if_modified_since=since.strftime(f))
1137
1138     def test_if_unmodified_since(self):
1139         now = datetime.datetime.utcnow()
1140         since = now + datetime.timedelta(1)
1141
1142         for f in DATE_FORMATS:
1143             t = since.strftime(f)
1144             status, headers, data = self.client.request_object(self.containers[1],
1145                                                                self.objects[0]['name'],
1146                                                                if_unmodified_since=t)
1147             #assert success
1148             self.assertEqual(status, 200)
1149             self.assertEqual(self.objects[0]['data'], data)
1150
1151             #assert content-type
1152             self.assertEqual(headers['content-type'],
1153                              self.objects[0]['meta']['content_type'])
1154
1155     def test_if_unmodified_since_precondition_failed(self):
1156         t = datetime.datetime.utcnow()
1157         t2 = t - datetime.timedelta(minutes=10)
1158
1159         #modify the object
1160         self.upload_data(self.containers[1],
1161                            self.objects[0]['name'],
1162                            self.objects[0]['data'][:200])
1163
1164         for f in DATE_FORMATS:
1165             past = t2.strftime(f)
1166             #assert precondition failed
1167             self.assert_raises_fault(412, self.client.retrieve_object,
1168                                      self.containers[1], self.objects[0]['name'],
1169                                      if_unmodified_since=past)
1170
1171     def test_hashes(self):
1172         l = 8388609
1173         fname = 'largefile'
1174         o = self.upload_random_data(self.containers[1], fname, l)
1175         if o:
1176             body = self.client.retrieve_object(self.containers[1], fname,
1177                                                format='json')
1178             hashes = body['hashes']
1179             block_size = body['block_size']
1180             block_hash = body['block_hash']
1181             block_num = l/block_size if l/block_size == 0 else l/block_size + 1
1182             self.assertTrue(len(hashes), block_num)
1183             i = 0
1184             for h in hashes:
1185                 start = i * block_size
1186                 end = (i + 1) * block_size
1187                 hash = compute_block_hash(o['data'][start:end], block_hash)
1188                 self.assertEqual(h, hash)
1189                 i += 1
1190
1191 class ObjectPut(BaseTestCase):
1192     def setUp(self):
1193         BaseTestCase.setUp(self)
1194         self.container = 'c1'
1195         self.client.create_container(self.container)
1196
1197     def test_upload(self):
1198         name = o_names[0]
1199         meta = {'test':'test1'}
1200         o = self.upload_random_data(self.container, name, **meta)
1201
1202         headers = self.client.retrieve_object_metadata(self.container,
1203                                                        name,
1204                                                        restricted=True)
1205         self.assertTrue('test' in headers.keys())
1206         self.assertEqual(headers['test'], meta['test'])
1207
1208         #assert uploaded content
1209         status, h, data = self.client.request_object(self.container, name)
1210         self.assertEqual(len(o['data']), int(h['content-length']))
1211         self.assertEqual(o['data'], data)
1212
1213         #assert content-type
1214         self.assertEqual(h['content-type'], o['meta']['content_type'])
1215
1216     def _test_maximum_upload_size_exceeds(self):
1217         name = o_names[0]
1218         meta = {'test':'test1'}
1219         #upload 100MB
1220         length=1024*1024*100
1221         self.assert_raises_fault(400, self.upload_random_data, self.container,
1222                                  name, length, **meta)
1223
1224     def test_upload_with_name_containing_slash(self):
1225         name = '/%s' % o_names[0]
1226         meta = {'test':'test1'}
1227         o = self.upload_random_data(self.container, name, **meta)
1228
1229         self.assertEqual(o['data'],
1230                          self.client.retrieve_object(self.container, name))
1231
1232         self.assertTrue(name in self.client.list_objects(self.container))
1233
1234     def test_create_directory_marker(self):
1235         self.client.create_directory_marker(self.container, 'foo')
1236         meta = self.client.retrieve_object_metadata(self.container, 'foo')
1237         self.assertEqual(meta['content-length'], '0')
1238         self.assertEqual(meta['content-type'], 'application/directory')
1239
1240     def test_upload_unprocessable_entity(self):
1241         meta={'etag':'123', 'test':'test1'}
1242
1243         #assert unprocessable entity
1244         self.assert_raises_fault(422, self.upload_random_data, self.container,
1245                                  o_names[0], **meta)
1246
1247     def test_chunked_transfer(self):
1248         data = get_random_data()
1249         objname = 'object'
1250         self.client.create_object_using_chunks(self.container, objname,
1251                                                StringIO(data))
1252
1253         uploaded_data = self.client.retrieve_object(self.container, objname)
1254         self.assertEqual(data, uploaded_data)
1255
1256     def test_manifestation(self):
1257         prefix = 'myobject/'
1258         data = ''
1259         for i in range(5):
1260             part = '%s%d' %(prefix, i)
1261             o = self.upload_random_data(self.container, part)
1262             data += o['data']
1263
1264         manifest = '%s/%s' %(self.container, prefix)
1265         self.client.create_manifestation(self.container, 'large-object', manifest)
1266
1267         self.assert_object_exists(self.container, 'large-object')
1268         self.assertEqual(data, self.client.retrieve_object(self.container,
1269                                                            'large-object'))
1270
1271         r = self.client.retrieve_object_hashmap(self.container,'large-object')
1272         hashes = r['hashes']
1273         block_size = int(r['block_size'])
1274         block_hash = r['block_hash']
1275         l = len(data)
1276         block_num = l/block_size if l/block_size != 0 else l/block_size + 1
1277         self.assertEqual(block_num, len(hashes))
1278
1279         #wrong manifestation
1280         self.client.create_manifestation(self.container, 'large-object',
1281                                          '%s/invalid' % self.container)
1282         self.assertEqual('', self.client.retrieve_object(self.container,
1283                                                          'large-object'))
1284
1285     def test_create_zero_length_object(self):
1286         c = self.container
1287         o = 'object'
1288         zero = self.client.create_zero_length_object(c, o)
1289         zero_meta = self.client.retrieve_object_metadata(c, o)
1290         zero_hash = self.client.retrieve_object_hashmap(c, o)["hashes"]
1291         zero_data = self.client.retrieve_object(c, o)
1292
1293         self.assertEqual(int(zero_meta['content-length']), 0)
1294         hasher = newhasher('sha256')
1295         hasher.update("")
1296         emptyhash = hasher.digest()
1297         self.assertEqual(zero_hash, [hexlify(emptyhash)])
1298         self.assertEqual(zero_data, '')
1299
1300     def test_create_object_by_hashmap(self):
1301         c = self.container
1302         o = 'object'
1303         self.upload_random_data(c, o)
1304         hashmap = self.client.retrieve_object(c, o, format='json')
1305         o2 = 'object-copy'
1306         self.client.create_object_by_hashmap(c, o2, hashmap)
1307         self.assertEqual(self.client.retrieve_object(c, o),
1308                          self.client.retrieve_object(c, o))
1309
1310 class ObjectCopy(BaseTestCase):
1311     def setUp(self):
1312         BaseTestCase.setUp(self)
1313         self.containers = ['c1', 'c2']
1314         for c in self.containers:
1315             self.client.create_container(c)
1316         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1317
1318     def test_copy(self):
1319         with AssertMappingInvariant(self.client.retrieve_object_metadata,
1320                              self.containers[0], self.obj['name']):
1321             #perform copy
1322             meta = {'test':'testcopy'}
1323             status = self.client.copy_object(self.containers[0],
1324                                               self.obj['name'],
1325                                               self.containers[0],
1326                                               'testcopy',
1327                                               meta)[0]
1328
1329             #assert copy success
1330             self.assertEqual(status, 201)
1331
1332             #assert access the new object
1333             headers = self.client.retrieve_object_metadata(self.containers[0],
1334                                                            'testcopy')
1335             self.assertTrue('x-object-meta-test' in headers.keys())
1336             self.assertTrue(headers['x-object-meta-test'], 'testcopy')
1337
1338             #assert etag is the same
1339             self.assertEqual(headers['etag'], self.obj['hash'])
1340
1341             #assert src object still exists
1342             self.assert_object_exists(self.containers[0], self.obj['name'])
1343
1344     def test_copy_from_different_container(self):
1345         with AssertMappingInvariant(self.client.retrieve_object_metadata,
1346                              self.containers[0], self.obj['name']):
1347             meta = {'test':'testcopy'}
1348             status = self.client.copy_object(self.containers[0],
1349                                              self.obj['name'],
1350                                              self.containers[1],
1351                                              'testcopy',
1352                                              meta)[0]
1353             self.assertEqual(status, 201)
1354
1355             # assert updated metadata
1356             meta = self.client.retrieve_object_metadata(self.containers[1],
1357                                                            'testcopy',
1358                                                            restricted=True)
1359             self.assertTrue('test' in meta.keys())
1360             self.assertTrue(meta['test'], 'testcopy')
1361
1362             #assert src object still exists
1363             self.assert_object_exists(self.containers[0], self.obj['name'])
1364
1365     def test_copy_invalid(self):
1366         #copy from invalid object
1367         meta = {'test':'testcopy'}
1368         self.assert_raises_fault(404, self.client.copy_object, self.containers[0],
1369                                  'test.py', self.containers[1], 'testcopy', meta)
1370
1371         #copy from invalid container
1372         meta = {'test':'testcopy'}
1373         self.assert_raises_fault(404, self.client.copy_object, self.containers[1],
1374                                  self.obj['name'], self.containers[1],
1375                                  'testcopy', meta)
1376
1377 class ObjectMove(BaseTestCase):
1378     def setUp(self):
1379         BaseTestCase.setUp(self)
1380         self.containers = ['c1', 'c2']
1381         for c in self.containers:
1382             self.client.create_container(c)
1383         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1384
1385     def test_move(self):
1386         meta = self.client.retrieve_object_metadata(self.containers[0],
1387                                                     self.obj['name'])
1388         self.assertTrue('x-object-uuid' in meta)
1389         uuid = meta['x-object-uuid']
1390
1391         #perform move
1392         meta = {'test':'testcopy'}
1393         src_path = '/'.join(('/', self.containers[0], self.obj['name']))
1394         status = self.client.move_object(self.containers[0], self.obj['name'],
1395                                          self.containers[0], 'testcopy',
1396                                          meta)[0]
1397
1398         #assert successful move
1399         self.assertEqual(status, 201)
1400
1401         #assert updated metadata
1402         meta = self.client.retrieve_object_metadata(self.containers[0],
1403                                                     'testcopy')
1404         self.assertTrue('x-object-meta-test' in meta.keys())
1405         self.assertTrue(meta['x-object-meta-test'], 'testcopy')
1406
1407         #assert same uuid
1408         self.assertTrue(meta['x-object-uuid'], uuid)
1409
1410         #assert src object no more exists
1411         self.assert_object_not_exists(self.containers[0], self.obj['name'])
1412
1413 class ObjectPost(BaseTestCase):
1414     def setUp(self):
1415         BaseTestCase.setUp(self)
1416         self.containers = ['c1', 'c2']
1417         for c in self.containers:
1418             self.client.create_container(c)
1419         self.obj = []
1420         for i in range(2):
1421             self.obj.append(self.upload_random_data(self.containers[0], o_names[i]))
1422
1423     def test_update_meta(self):
1424         with AssertUUidInvariant(self.client.retrieve_object_metadata,
1425                                  self.containers[0],
1426                                  self.obj[0]['name']):
1427             #perform update metadata
1428             more = {'foo': 'foo', 'bar': 'bar', 'f' * 114: 'b' * 256}
1429             status = self.client.update_object_metadata(self.containers[0],
1430                                                         self.obj[0]['name'],
1431                                                         **more)[0]
1432             #assert request accepted
1433             self.assertEqual(status, 202)
1434
1435             #assert old metadata are still there
1436             headers = self.client.retrieve_object_metadata(self.containers[0],
1437                                                            self.obj[0]['name'],
1438                                                            restricted=True)
1439             #assert new metadata have been updated
1440             for k,v in more.items():
1441                 self.assertTrue(k in headers.keys())
1442                 self.assertTrue(headers[k], v)
1443
1444             #out of limits
1445             more = {'f' * 114: 'b' * 257}
1446             self.assert_raises_fault(400, self.client.update_object_metadata,
1447                                                         self.containers[0],
1448                                                         self.obj[0]['name'],
1449                                                         **more)
1450
1451             #perform update metadata
1452             more = {'α': 'β' * 256}
1453             status = self.client.update_object_metadata(self.containers[0],
1454                                                         self.obj[0]['name'],
1455                                                         **more)[0]
1456             #assert request accepted
1457             self.assertEqual(status, 202)
1458
1459             #assert old metadata are still there
1460             headers = self.client.retrieve_object_metadata(self.containers[0],
1461                                                            self.obj[0]['name'],
1462                                                            restricted=True)
1463             #assert new metadata have been updated
1464             for k,v in more.items():
1465                 self.assertTrue(k in headers.keys())
1466                 self.assertTrue(headers[k], v)
1467
1468             #out of limits
1469             more = {'α': 'β' * 257}
1470             self.assert_raises_fault(400, self.client.update_object_metadata,
1471                                                         self.containers[0],
1472                                                         self.obj[0]['name'],
1473                                                         **more)
1474
1475     def test_update_object(self,
1476                            first_byte_pos=0,
1477                            last_byte_pos=499,
1478                            instance_length = True,
1479                            content_length = 500):
1480         with AssertUUidInvariant(self.client.retrieve_object_metadata,
1481                                  self.containers[0],
1482                                  self.obj[0]['name']):
1483             l = len(self.obj[0]['data'])
1484             range = 'bytes %d-%d/%s' %(first_byte_pos,
1485                                            last_byte_pos,
1486                                             l if instance_length else '*')
1487             partial = last_byte_pos - first_byte_pos + 1
1488             length = first_byte_pos + partial
1489             data = get_random_data(partial)
1490             args = {'content_type':'application/octet-stream',
1491                     'content_range':'%s' %range}
1492             if content_length:
1493                 args['content_length'] = content_length
1494
1495             r = self.client.update_object(self.containers[0], self.obj[0]['name'],
1496                                       StringIO(data), **args)
1497             status = r[0]
1498             etag = r[1]['etag']
1499             if partial < 0 or (instance_length and l <= last_byte_pos):
1500                 self.assertEqual(status, 202)
1501             else:
1502                 self.assertEqual(status, 204)
1503                 #check modified object
1504                 content = self.client.retrieve_object(self.containers[0],
1505                                                   self.obj[0]['name'])
1506                 self.assertEqual(content[:first_byte_pos], self.obj[0]['data'][:first_byte_pos])
1507                 self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1508                 self.assertEqual(content[last_byte_pos+1:], self.obj[0]['data'][last_byte_pos+1:])
1509                 self.assertEqual(etag, compute_md5_hash(content))
1510
1511     def test_update_object_lt_blocksize(self):
1512         self.test_update_object(10, 20, content_length=None)
1513
1514     def test_update_object_gt_blocksize(self):
1515         o = self.upload_random_data(self.containers[0], o_names[1],
1516                                 length=4*1024*1024+5)
1517         c = self.containers[0]
1518         o_name = o['name']
1519         o_data = o['data']
1520         first_byte_pos = 4*1024*1024+1
1521         last_byte_pos = 4*1024*1024+4
1522         l = last_byte_pos - first_byte_pos + 1
1523         data = get_random_data(l)
1524         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1525         self.client.update_object(c, o_name, StringIO(data), content_range=range)
1526         content = self.client.retrieve_object(c, o_name)
1527         self.assertEqual(content[:first_byte_pos], o_data[:first_byte_pos])
1528         self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1529         self.assertEqual(content[last_byte_pos+1:], o_data[last_byte_pos+1:])
1530
1531     def test_update_object_divided_by_blocksize(self):
1532         o = self.upload_random_data(self.containers[0], o_names[1],
1533                                 length=4*1024*1024+5)
1534         c = self.containers[0]
1535         o_name = o['name']
1536         o_data = o['data']
1537         first_byte_pos = 4*1024*1024
1538         last_byte_pos = 5*1024*1024
1539         l = last_byte_pos - first_byte_pos + 1
1540         data = get_random_data(l)
1541         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1542         self.client.update_object(c, o_name, StringIO(data), content_range=range)
1543         content = self.client.retrieve_object(c, o_name)
1544         self.assertEqual(content[:first_byte_pos], o_data[:first_byte_pos])
1545         self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1546         self.assertEqual(content[last_byte_pos+1:], o_data[last_byte_pos+1:])
1547
1548     def test_update_object_no_content_length(self):
1549         self.test_update_object(content_length = None)
1550
1551     def test_update_object_invalid_content_length(self):
1552         with AssertContentInvariant(self.client.retrieve_object,
1553                                     self.containers[0], self.obj[0]['name']):
1554             self.assert_raises_fault(400, self.test_update_object,
1555                                      content_length = 1000)
1556
1557     def _test_update_object_invalid_range(self):
1558         with AssertContentInvariant(self.client.retrieve_object,
1559                                     self.containers[0], self.obj[0]['name']):
1560             self.assert_raises_fault(416, self.test_update_object, 499, 0, True)
1561
1562     def _test_update_object_invalid_range_and_length(self):
1563         with AssertContentInvariant(self.client.retrieve_object,
1564                                     self.containers[0], self.obj[0]['name']):
1565             self.assert_raises_fault([400, 416], self.test_update_object, 499, 0, True,
1566                                      -1)
1567
1568     def test_update_object_invalid_range_with_no_content_length(self):
1569         with AssertContentInvariant(self.client.retrieve_object,
1570                                     self.containers[0], self.obj[0]['name']):
1571             self.assert_raises_fault(416, self.test_update_object, 499, 0, True,
1572                                      content_length = None)
1573
1574     def test_update_object_out_of_limits(self):
1575         with AssertContentInvariant(self.client.retrieve_object,
1576                                     self.containers[0], self.obj[0]['name']):
1577             l = len(self.obj[0]['data'])
1578             self.assert_raises_fault(416, self.test_update_object, 0, l+1, True)
1579
1580     def test_append(self):
1581         data = get_random_data(500)
1582         headers = {}
1583         self.client.update_object(self.containers[0], self.obj[0]['name'],
1584                                   StringIO(data), content_length=500,
1585                                   content_type='application/octet-stream')
1586
1587         content = self.client.retrieve_object(self.containers[0],
1588                                               self.obj[0]['name'])
1589         self.assertEqual(len(content), len(self.obj[0]['data']) + 500)
1590         self.assertEqual(content[:-500], self.obj[0]['data'])
1591
1592     def test_update_with_chunked_transfer(self):
1593         data = get_random_data(500)
1594         dl = len(data)
1595         fl = len(self.obj[0]['data'])
1596
1597         self.client.update_object_using_chunks(self.containers[0],
1598                                                self.obj[0]['name'],
1599                                                StringIO(data),
1600                                                offset=0,
1601                                                content_type='application/octet-stream')
1602
1603         #check modified object
1604         content = self.client.retrieve_object(self.containers[0],
1605                                               self.obj[0]['name'])
1606         self.assertEqual(content[0:dl], data)
1607         self.assertEqual(content[dl:fl], self.obj[0]['data'][dl:fl])
1608
1609     def test_update_from_other_object(self):
1610         c = self.containers[0]
1611         src = o_names[0]
1612         dest = 'object'
1613
1614         source_data = self.client.retrieve_object(c, src)
1615         source_meta = self.client.retrieve_object_metadata(c, src)
1616         source_hash = self.client.retrieve_object_hashmap(c, src)["hashes"]
1617
1618         #update zero length object
1619         self.client.create_zero_length_object(c, dest)
1620         source_object = '/%s/%s' % (c, src)
1621         self.client.update_from_other_source(c, dest, source_object)
1622         dest_data = self.client.retrieve_object(c, src)
1623         dest_meta = self.client.retrieve_object_metadata(c, dest)
1624         dest_hash = self.client.retrieve_object_hashmap(c, src)["hashes"]
1625         self.assertEqual(source_data, dest_data)
1626         self.assertEqual(source_hash, dest_hash)
1627
1628         #test append
1629         self.client.update_from_other_source(c, dest, source_object)
1630         content = self.client.retrieve_object(c, dest)
1631         self.assertEqual(source_data * 2, content)
1632
1633     def test_update_range_from_other_object(self):
1634         c = self.containers[0]
1635         dest = 'object'
1636
1637         #test update range
1638         src = self.obj[1]['name']
1639         src_data = self.client.retrieve_object(c, src)
1640
1641         #update zero length object
1642         prev_data = self.upload_random_data(c, dest, length=4*1024*1024+10)['data']
1643         source_object = '/%s/%s' % (c, src)
1644         first_byte_pos = 4*1024*1024+1
1645         last_byte_pos = 4*1024*1024+4
1646         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1647         self.client.update_from_other_source(c, dest, source_object,
1648                                              content_range=range)
1649         content = self.client.retrieve_object(c, dest)
1650         self.assertEqual(content[:first_byte_pos], prev_data[:first_byte_pos])
1651         self.assertEqual(content[first_byte_pos:last_byte_pos+1], src_data[:last_byte_pos - first_byte_pos + 1])
1652         self.assertEqual(content[last_byte_pos+1:], prev_data[last_byte_pos+1:])
1653
1654     def test_update_hashes_from_other_object(self):
1655         c = self.containers[0]
1656         dest = 'object'
1657
1658         #test update range
1659         src_data = self.upload_random_data(c, o_names[0], length=1024*1024+10)['data']
1660
1661         #update zero length object
1662         prev_data = self.upload_random_data(c, dest, length=5*1024*1024+10)['data']
1663         source_object = '/%s/%s' % (c, o_names[0])
1664         first_byte_pos = 4*1024*1024
1665         last_byte_pos = 5*1024*1024
1666         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1667         self.client.update_from_other_source(c, dest, source_object,
1668                                              content_range=range)
1669         content = self.client.retrieve_object(c, dest)
1670         self.assertEqual(content[:first_byte_pos], prev_data[:first_byte_pos])
1671         self.assertEqual(content[first_byte_pos:last_byte_pos+1], src_data[:last_byte_pos - first_byte_pos + 1])
1672         self.assertEqual(content[last_byte_pos+1:], prev_data[last_byte_pos+1:])
1673
1674
1675     def test_update_zero_length_object(self):
1676         c = self.containers[0]
1677         o = 'object'
1678         other = 'other'
1679         zero = self.client.create_zero_length_object(c, o)
1680
1681         data = get_random_data()
1682         self.client.update_object(c, o, StringIO(data))
1683         self.client.create_object(c, other, StringIO(data))
1684
1685         self.assertEqual(self.client.retrieve_object(c, o),
1686                          self.client.retrieve_object(c, other))
1687
1688         self.assertEqual(self.client.retrieve_object_hashmap(c, o)["hashes"],
1689                          self.client.retrieve_object_hashmap(c, other)["hashes"])
1690
1691 class ObjectDelete(BaseTestCase):
1692     def setUp(self):
1693         BaseTestCase.setUp(self)
1694         self.containers = ['c1', 'c2']
1695         for c in self.containers:
1696             self.client.create_container(c)
1697         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1698
1699     def test_delete(self):
1700         #perform delete object
1701         self.client.delete_object(self.containers[0], self.obj['name'])[0]
1702
1703     def test_delete_invalid(self):
1704         #assert item not found
1705         self.assert_raises_fault(404, self.client.delete_object, self.containers[1],
1706                                  self.obj['name'])
1707
1708 class ListSharing(BaseTestCase):
1709     def setUp(self):
1710         BaseTestCase.setUp(self)
1711         for i in range(2):
1712             self.client.create_container('c%s' %i)
1713         self.client.create_container('c')
1714         for i in range(2):
1715             self.upload_random_data('c1', 'o%s' %i)
1716         accounts = OTHER_ACCOUNTS.copy()
1717         self.o1_sharing_with = accounts.popitem()
1718         self.o1_sharing = [self.o1_sharing_with[1]]
1719         self.client.share_object('c1', 'o1', self.o1_sharing, read=True)
1720
1721         l = []
1722         for i in range(2):
1723             l.append(accounts.popitem())
1724
1725     def test_list_other_shared(self):
1726         self.other = Pithos_Client(get_url(),
1727                               self.o1_sharing_with[0],
1728                               self.o1_sharing_with[1])
1729         self.assertTrue(get_user() in self.other.list_shared_by_others())
1730
1731     def test_list_my_shared(self):
1732         my_shared_containers = self.client.list_containers(shared=True)
1733         self.assertTrue('c1' in my_shared_containers)
1734         self.assertTrue('c2' not in my_shared_containers)
1735
1736         my_shared_objects = self.client.list_objects('c1', shared=True)
1737         self.assertTrue('o1' in my_shared_objects)
1738         self.assertTrue('o2' not in my_shared_objects)
1739
1740 class List(BaseTestCase):
1741     def setUp(self):
1742         BaseTestCase.setUp(self)
1743         for i in range(1, 5):
1744             c = 'c%s' % i
1745             self.client.create_container(c)
1746             for j in range(1, 3):
1747                 o = 'o%s' % j
1748                 self.upload_random_data(c, o)
1749             if i < 3:
1750                 self.client.share_object(c, 'o1', ['papagian'], read=True)
1751             if i%2 != 0:
1752                 self.client.publish_object(c, 'o2')
1753
1754     def test_shared_public(self):
1755         func, kwargs = self.client.list_containers, {'shared':True}
1756         l = func(**kwargs)
1757         self.assertEqual(l, ['c1', 'c2'])
1758         self.assertEqual(l, [e['name'] for e in func(format='json', **kwargs)])
1759
1760         func, kwargs = self.client.list_containers, {'public':True}
1761         l = func(**kwargs)
1762         self.assertEqual(l, ['c1', 'c3'])
1763         self.assertEqual(l, [e['name'] for e in func(format='json', **kwargs)])
1764
1765         func, kwargs = self.client.list_containers, {'shared':True, 'public':True}
1766         l = func(**kwargs)
1767         self.assertEqual(l, ['c1', 'c2', 'c3'])
1768         self.assertEqual(l, [e['name'] for e in func(format='json', **kwargs)])
1769
1770
1771         func, args, kwargs = self.client.list_objects, ['c1'], {'shared':True}
1772         l = func(*args, **kwargs)
1773         self.assertEqual(l, ['o1'])
1774         self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1775
1776         func, args, kwargs = self.client.list_objects, ['c1'], {'public':True}
1777         l = func(*args, **kwargs)
1778         self.assertEqual(l, ['o2'])
1779         self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1780
1781         func, args, kwargs = self.client.list_objects, ['c1'], {'shared':True, 'public':True}
1782         l = func(*args, **kwargs)
1783         self.assertEqual(l, ['o1', 'o2'])
1784         self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1785
1786
1787         func, args, kwargs = self.client.list_objects, ['c2'], {'shared':True}
1788         l = func(*args, **kwargs)
1789         self.assertEqual(l, ['o1'])
1790         self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1791
1792         func, args, kwargs = self.client.list_objects, ['c2'], {'public':True}
1793         l = func(*args, **kwargs)
1794         self.assertEqual(l, '')
1795         self.assertEqual([], func(*args, format='json', **kwargs))
1796
1797         func, args, kwargs = self.client.list_objects, ['c2'], {'shared':True, 'public':True}
1798         l = func(*args, **kwargs)
1799         self.assertEqual(l, ['o1'])
1800         self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1801
1802
1803         func, args, kwargs = self.client.list_objects, ['c3'], {'shared':True}
1804         l = func(*args, **kwargs)
1805         self.assertEqual(l, '')
1806         self.assertEqual([], func(*args, format='json', **kwargs))
1807
1808         func, args, kwargs = self.client.list_objects, ['c3'], {'public':True}
1809         l = func(*args, **kwargs)
1810         self.assertEqual(l, ['o2'])
1811         self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1812
1813         func, args, kwargs = self.client.list_objects, ['c3'], {'shared':True, 'public':True}
1814         l = func(*args, **kwargs)
1815         self.assertEqual(l, ['o2'])
1816         self.assertEqual(l, [e['name'] for e in func(*args, format='json', **kwargs)])
1817
1818
1819         func, args, kwargs = self.client.list_objects, ['c4'], {'shared':True}
1820         l = func(*args, **kwargs)
1821         self.assertEqual(l, '')
1822         self.assertEqual([], func(*args, format='json', **kwargs))
1823
1824         func, args, kwargs = self.client.list_objects, ['c4'], {'public':True}
1825         l = func(*args, **kwargs)
1826         self.assertEqual(l, '')
1827         self.assertEqual([], func(*args, format='json', **kwargs))
1828
1829         func, args, kwargs = self.client.list_objects, ['c4'], {'shared':True, 'public':True}
1830         l = func(*args, **kwargs)
1831         self.assertEqual(l, '')
1832         self.assertEqual([], func(*args, format='json', **kwargs))
1833
1834 class TestGreek(BaseTestCase):
1835     def test_create_container(self):
1836         self.client.create_container('φάκελος')
1837         self.assert_container_exists('φάκελος')
1838
1839         self.assertTrue('φάκελος' in self.client.list_containers())
1840
1841     def test_create_object(self):
1842         self.client.create_container('φάκελος')
1843         self.upload_random_data('φάκελος', 'αντικείμενο')
1844
1845         self.assert_object_exists('φάκελος', 'αντικείμενο')
1846         self.assertTrue('αντικείμενο' in self.client.list_objects('φάκελος'))
1847
1848     def test_copy_object(self):
1849         src_container = 'φάκελος'
1850         src_object = 'αντικείμενο'
1851         dest_container = 'αντίγραφα'
1852         dest_object = 'ασφαλές-αντίγραφο'
1853
1854         self.client.create_container(src_container)
1855         self.upload_random_data(src_container, src_object)
1856
1857         self.client.create_container(dest_container)
1858         self.client.copy_object(src_container, src_object, dest_container,
1859                                 dest_object)
1860
1861         self.assert_object_exists(src_container, src_object)
1862         self.assert_object_exists(dest_container, dest_object)
1863         self.assertTrue(dest_object in self.client.list_objects(dest_container))
1864
1865     def test_move_object(self):
1866         src_container = 'φάκελος'
1867         src_object = 'αντικείμενο'
1868         dest_container = 'αντίγραφα'
1869         dest_object = 'ασφαλές-αντίγραφο'
1870
1871         self.client.create_container(src_container)
1872         self.upload_random_data(src_container, src_object)
1873
1874         self.client.create_container(dest_container)
1875         self.client.move_object(src_container, src_object, dest_container,
1876                                 dest_object)
1877
1878         self.assert_object_not_exists(src_container, src_object)
1879         self.assert_object_exists(dest_container, dest_object)
1880         self.assertTrue(dest_object in self.client.list_objects(dest_container))
1881
1882     def test_delete_object(self):
1883         self.client.create_container('φάκελος')
1884         self.upload_random_data('φάκελος', 'αντικείμενο')
1885         self.assert_object_exists('φάκελος', 'αντικείμενο')
1886
1887         self.client.delete_object('φάκελος', 'αντικείμενο')
1888         self.assert_object_not_exists('φάκελος', 'αντικείμενο')
1889         self.assertTrue('αντικείμενο' not in self.client.list_objects('φάκελος'))
1890
1891     def test_delete_container(self):
1892         self.client.create_container('φάκελος')
1893         self.assert_container_exists('φάκελος')
1894
1895         self.client.delete_container('φάκελος')
1896         self.assert_container_not_exists('φάκελος')
1897         self.assertTrue('φάκελος' not in self.client.list_containers())
1898
1899     def test_account_meta(self):
1900         meta = {'ποιότητα':'ΑΑΑ'}
1901         self.client.update_account_metadata(**meta)
1902         meta = self.client.retrieve_account_metadata(restricted=True)
1903         self.assertTrue('ποιότητα' in meta.keys())
1904         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1905
1906     def test_container_meta(self):
1907         meta = {'ποιότητα':'ΑΑΑ'}
1908         self.client.create_container('φάκελος', meta=meta)
1909
1910         meta = self.client.retrieve_container_metadata('φάκελος', restricted=True)
1911         self.assertTrue('ποιότητα' in meta.keys())
1912         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1913
1914     def test_object_meta(self):
1915         self.client.create_container('φάκελος')
1916         meta = {'ποιότητα':'ΑΑΑ'}
1917         self.upload_random_data('φάκελος', 'αντικείμενο', **meta)
1918
1919         meta = self.client.retrieve_object_metadata('φάκελος', 'αντικείμενο',
1920                                                     restricted=True)
1921         self.assertTrue('ποιότητα' in meta.keys())
1922         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1923
1924     def test_list_meta_filtering(self):
1925         self.client.create_container('φάκελος')
1926         meta = {'ποιότητα':'ΑΑΑ'}
1927         self.upload_random_data('φάκελος', 'ο1', **meta)
1928         self.upload_random_data('φάκελος', 'ο2')
1929         self.upload_random_data('φάκελος', 'ο3')
1930
1931         meta = {'ποσότητα':'μεγάλη'}
1932         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
1933         objects = self.client.list_objects('φάκελος', meta='ποιότητα, ποσότητα')
1934         self.assertEquals(objects, ['ο1', 'ο2'])
1935
1936         objects = self.client.list_objects('φάκελος', meta='!ποιότητα')
1937         self.assertEquals(objects, ['ο2', 'ο3'])
1938
1939         objects = self.client.list_objects('φάκελος', meta='!ποιότητα, !ποσότητα')
1940         self.assertEquals(objects, ['ο3'])
1941
1942         meta = {'ποιότητα':'ΑΒ'}
1943         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
1944         objects = self.client.list_objects('φάκελος', meta='ποιότητα=ΑΑΑ')
1945         self.assertEquals(objects, ['ο1'])
1946         objects = self.client.list_objects('φάκελος', meta='ποιότητα!=ΑΑΑ')
1947         self.assertEquals(objects, ['ο2'])
1948
1949         meta = {'έτος':'2011'}
1950         self.client.update_object_metadata('φάκελος', 'ο3', **meta)
1951         meta = {'έτος':'2012'}
1952         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
1953         objects = self.client.list_objects('φάκελος', meta='έτος<2012')
1954         self.assertEquals(objects, ['ο3'])
1955         objects = self.client.list_objects('φάκελος', meta='έτος<=2012')
1956         self.assertEquals(objects, ['ο2', 'ο3'])
1957         objects = self.client.list_objects('φάκελος', meta='έτος<2012,έτος!=2011')
1958         self.assertEquals(objects, '')
1959
1960     def test_groups(self):
1961         #create a group
1962         groups = {'σεφς':'chazapis,διογένης'}
1963         self.client.set_account_groups(**groups)
1964         groups.update(self.initial_groups)
1965         self.assertEqual(groups['σεφς'],
1966                          self.client.retrieve_account_groups()['σεφς'])
1967
1968         #check read access
1969         self.client.create_container('φάκελος')
1970         o = self.upload_random_data('φάκελος', 'ο1')
1971         self.client.share_object('φάκελος', 'ο1', ['%s:σεφς' % get_user()])
1972         chef = Pithos_Client(get_url(),
1973                             '0009',
1974                             'διογένης')
1975         self.assert_not_raises_fault(403, chef.retrieve_object_metadata,
1976                                      'φάκελος', 'ο1', account=get_user())
1977
1978         #check write access
1979         self.client.share_object('φάκελος', 'ο1', ['διογένης'], read=False)
1980         new_data = get_random_data()
1981         self.assert_not_raises_fault(403, chef.update_object,
1982                                      'φάκελος', 'ο1', StringIO(new_data),
1983                                      account=get_user())
1984
1985         server_data = self.client.retrieve_object('φάκελος', 'ο1')
1986         self.assertEqual(server_data[:len(o['data'])], o['data'])
1987         self.assertEqual(server_data[len(o['data']):], new_data)
1988
1989     def test_manifestation(self):
1990         self.client.create_container('κουβάς')
1991         prefix = 'μέρη/'
1992         data = ''
1993         for i in range(5):
1994             part = '%s%d' %(prefix, i)
1995             o = self.upload_random_data('κουβάς', part)
1996             data += o['data']
1997
1998         self.client.create_container('φάκελος')
1999         manifest = '%s/%s' %('κουβάς', prefix)
2000         self.client.create_manifestation('φάκελος', 'άπαντα', manifest)
2001
2002         self.assert_object_exists('φάκελος', 'άπαντα')
2003         self.assertEqual(data, self.client.retrieve_object('φάκελος',
2004                                                            'άπαντα'))
2005
2006         #wrong manifestation
2007         self.client.create_manifestation('φάκελος', 'άπαντα', 'κουβάς/άκυρο')
2008         self.assertEqual('', self.client.retrieve_object('φάκελος', 'άπαντα'))
2009
2010     def test_update_from_another_object(self):
2011         self.client.create_container('κουβάς')
2012         src_data = self.upload_random_data('κουβάς', 'πηγή')['data']
2013         initial_data = self.upload_random_data('κουβάς', 'νέο')['data']
2014         source_object = '/%s/%s' % ('κουβάς', 'πηγή')
2015         self.client.update_from_other_source('κουβάς', 'νέο', source_object)
2016
2017         self.assertEqual(
2018             self.client.retrieve_object('κουβάς', 'νέο'),
2019             '%s%s' % (initial_data, self.client.retrieve_object('κουβάς', 'πηγή')))
2020
2021 class TestPermissions(BaseTestCase):
2022     def setUp(self):
2023         BaseTestCase.setUp(self)
2024
2025         #create a group
2026         self.authorized = ['chazapis', 'verigak', 'gtsouk']
2027         groups = {'pithosdev':','.join(self.authorized)}
2028         self.client.set_account_groups(**groups)
2029
2030         self.container = 'c'
2031         self.object = 'o'
2032         self.client.create_container(self.container)
2033         self.upload_random_data(self.container, self.object)
2034         self.upload_random_data(self.container, self.object+'/')
2035         self.upload_random_data(self.container, self.object+'/a')
2036         self.upload_random_data(self.container, self.object+'a')
2037         self.upload_random_data(self.container, self.object+'a/')
2038         self.dir_content_types = ('application/directory', 'application/folder')
2039
2040     def assert_read(self, authorized=[], any=False, depth=0):
2041         for token, account in OTHER_ACCOUNTS.items():
2042             cl = Pithos_Client(get_url(), token, account)
2043             if account in authorized or any:
2044                 self.assert_not_raises_fault(403, cl.retrieve_object_metadata,
2045                                              self.container, self.object,
2046                                              account=get_user())
2047             else:
2048                 self.assert_raises_fault(403, cl.retrieve_object_metadata,
2049                                          self.container, self.object,
2050                                          account=get_user())
2051
2052         #check inheritance
2053         meta = self.client.retrieve_object_metadata(self.container, self.object)
2054         type = meta['content-type']
2055         derivatives = self.client.list_objects(self.container, prefix=self.object)
2056         #exclude the self.object
2057         del derivatives[derivatives.index(self.object)]
2058         for o in derivatives:
2059             for token, account in OTHER_ACCOUNTS.items():
2060                 cl = Pithos_Client(get_url(), token, account)
2061                 prefix = self.object if self.object.endswith('/') else self.object+'/'
2062                 if (account in authorized or any) and \
2063                 (type in self.dir_content_types) and \
2064                 o.startswith(prefix):
2065                     self.assert_not_raises_fault(403, cl.retrieve_object_metadata,
2066                                              self.container, o, account=get_user())
2067                 else:
2068                     self.assert_raises_fault(403, cl.retrieve_object_metadata,
2069                                          self.container, o, account=get_user())
2070
2071     def assert_write(self, authorized=[], any=False):
2072         o_data = self.client.retrieve_object(self.container, self.object)
2073         for token, account in OTHER_ACCOUNTS.items():
2074             cl = Pithos_Client(get_url(), token, account)
2075             new_data = get_random_data()
2076             if account in authorized or any:
2077                 # test write access
2078                 self.assert_not_raises_fault(403, cl.update_object,
2079                                              self.container, self.object, StringIO(new_data),
2080                                              account=get_user())
2081                 try:
2082                     # test read access
2083                     server_data = cl.retrieve_object(self.container, self.object, account=get_user())
2084                     self.assertEqual(o_data, server_data[:len(o_data)])
2085                     self.assertEqual(new_data, server_data[len(o_data):])
2086                     o_data = server_data
2087                 except Fault, f:
2088                     self.failIf(f.status == 403)
2089             else:
2090                 self.assert_raises_fault(403, cl.update_object,
2091                                              self.container, self.object, StringIO(new_data),
2092                                              account=get_user())
2093         #check inheritance
2094         meta = self.client.retrieve_object_metadata(self.container, self.object)
2095         type = meta['content-type']
2096         derivatives = self.client.list_objects(self.container, prefix=self.object)
2097         #exclude the object
2098         del derivatives[derivatives.index(self.object)]
2099         for o in derivatives:
2100             for token, account in OTHER_ACCOUNTS.items():
2101                 prefix = self.object if self.object.endswith('/') else self.object+'/'
2102                 cl = Pithos_Client(get_url(), token, account)
2103                 new_data = get_random_data()
2104                 if (account in authorized or any) and \
2105                 (type in self.dir_content_types) and \
2106                 o.startswith(prefix):
2107                     # test write access
2108                     self.assert_not_raises_fault(403, cl.update_object,
2109                                                  self.container, o,
2110                                                  StringIO(new_data),
2111                                                  account=get_user())
2112                     try:
2113                         server_data = cl.retrieve_object(self.container, o, account=get_user())
2114                         self.assertEqual(new_data, server_data[-len(new_data):])
2115                     except Fault, f:
2116                         self.failIf(f.status == 403)
2117                 else:
2118                     self.assert_raises_fault(403, cl.update_object,
2119                                                  self.container, o,
2120                                                  StringIO(new_data),
2121                                                  account=get_user())
2122
2123     def test_group_read(self):
2124         self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()])
2125         self.assert_read(authorized=self.authorized)
2126
2127     def test_read_many(self):
2128         self.client.share_object(self.container, self.object, self.authorized)
2129         self.assert_read(authorized=self.authorized)
2130
2131     def test_read_by_everyone(self):
2132         self.client.share_object(self.container, self.object, ['*'])
2133         self.assert_read(any=True)
2134
2135     def test_read_directory(self):
2136         for type in self.dir_content_types:
2137             #change content type
2138             self.client.move_object(self.container, self.object, self.container, self.object, content_type=type)
2139             self.client.share_object(self.container, self.object, ['*'])
2140             self.assert_read(any=True)
2141             self.client.share_object(self.container, self.object, self.authorized)
2142             self.assert_read(authorized=self.authorized)
2143             self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()])
2144             self.assert_read(authorized=self.authorized)
2145
2146     def test_group_write(self):
2147         self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()], read=False)
2148         self.assert_write(authorized=self.authorized)
2149
2150     def test_write_many(self):
2151         self.client.share_object(self.container, self.object, self.authorized, read=False)
2152         self.assert_write(authorized=self.authorized)
2153
2154     def test_write_by_everyone(self):
2155         self.client.share_object(self.container, self.object, ['*'], read=False)
2156         self.assert_write(any=True)
2157
2158     def test_write_directory(self):
2159         dir_content_types = ('application/directory', 'application/foler')
2160         for type in dir_content_types:
2161             #change content type
2162             self.client.move_object(self.container, self.object, self.container, self.object, content_type='application/folder')
2163             self.client.share_object(self.container, self.object, ['*'], read=False)
2164             self.assert_write(any=True)
2165             self.client.share_object(self.container, self.object, self.authorized, read=False)
2166             self.assert_write(authorized=self.authorized)
2167             self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()], read=False)
2168             self.assert_write(authorized=self.authorized)
2169
2170     def test_shared_listing(self):
2171         self.client.share_object(self.container, self.object, self.authorized)
2172
2173         my_shared_containers = self.client.list_containers(shared=True)
2174         self.assertEqual(['c'], my_shared_containers)
2175         my_shared_objects = self.client.list_objects('c', shared=True)
2176         self.assertEqual(['o'], my_shared_objects)
2177
2178         dir_content_types = ('application/directory', 'application/foler')
2179         for type in dir_content_types:
2180             #change content type
2181             self.client.move_object(self.container, self.object, self.container, self.object, content_type='application/folder')
2182             my_shared_objects = self.client.list_objects('c', shared=True)
2183             self.assertEqual(['o', 'o/', 'o/a'], my_shared_objects)
2184
2185         for token, account in OTHER_ACCOUNTS.items():
2186             if account in self.authorized:
2187                 self.other = Pithos_Client(get_url(), token, account)
2188                 self.assertTrue(get_user() in self.other.list_shared_by_others())
2189
2190 class TestPublish(BaseTestCase):
2191     def test_publish(self):
2192         self.client.create_container('c')
2193         o_data = self.upload_random_data('c', 'o')['data']
2194         self.client.publish_object('c', 'o')
2195         meta = self.client.retrieve_object_metadata('c', 'o')
2196         self.assertTrue('x-object-public' in meta)
2197         url = meta['x-object-public']
2198
2199         p = urlparse(get_url())
2200         if p.scheme == 'http':
2201             conn = HTTPConnection(p.netloc)
2202         elif p.scheme == 'https':
2203             conn = HTTPSConnection(p.netloc)
2204         else:
2205             raise Exception('Unknown URL scheme')
2206
2207         conn.request('GET', url)
2208         resp = conn.getresponse()
2209         length = resp.getheader('content-length', None)
2210         data = resp.read(length)
2211         self.assertEqual(o_data, data)
2212
2213 class TestPolicies(BaseTestCase):
2214     def test_none_versioning(self):
2215         self.client.create_container('c', policies={'versioning':'none'})
2216         o = self.upload_random_data('c', 'o')
2217         meta = self.client.retrieve_object_metadata('c', 'o')
2218         v = meta['x-object-version']
2219         more_data = get_random_data()
2220         self.client.update_object('c', 'o', StringIO(more_data))
2221         vlist = self.client.retrieve_object_versionlist('c', 'o')
2222         self.assert_raises_fault(404, self.client.retrieve_object_version,
2223                                  'c', 'o', v)
2224         data = self.client.retrieve_object('c', 'o')
2225         end = len(o['data'])
2226         self.assertEqual(data[:end], o['data'])
2227         self.assertEqual(data[end:], more_data)
2228
2229     def test_quota(self):
2230         self.client.create_container('c', policies={'quota':'1'})
2231         meta = self.client.retrieve_container_metadata('c')
2232         self.assertEqual(meta['x-container-policy-quota'], '1')
2233         self.assert_raises_fault(413, self.upload_random_data, 'c', 'o',
2234                                  length=1024*1024+1)
2235
2236     def test_quota_none(self):
2237         self.client.create_container('c', policies={'quota':'0'})
2238         meta = self.client.retrieve_container_metadata('c')
2239         self.assertEqual(meta['x-container-policy-quota'], '0')
2240         self.assert_not_raises_fault(413, self.upload_random_data, 'c', 'o',
2241                                  length=1024*1024+1)
2242
2243 class AssertUUidInvariant(object):
2244     def __init__(self, callable, *args, **kwargs):
2245         self.callable = callable
2246         self.args = args
2247         self.kwargs = kwargs
2248
2249     def __enter__(self):
2250         self.map = self.callable(*self.args, **self.kwargs)
2251         assert('x-object-uuid' in self.map)
2252         self.uuid = self.map['x-object-uuid']
2253         return self.map
2254
2255     def __exit__(self, type, value, tb):
2256         map = self.callable(*self.args, **self.kwargs)
2257         assert('x-object-uuid' in self.map)
2258         uuid = map['x-object-uuid']
2259         assert(uuid == self.uuid)
2260
2261 class AssertMappingInvariant(object):
2262     def __init__(self, callable, *args, **kwargs):
2263         self.callable = callable
2264         self.args = args
2265         self.kwargs = kwargs
2266
2267     def __enter__(self):
2268         self.map = self.callable(*self.args, **self.kwargs)
2269         return self.map
2270
2271     def __exit__(self, type, value, tb):
2272         map = self.callable(*self.args, **self.kwargs)
2273         for k, v in self.map.items():
2274             if is_date(v):
2275                 continue
2276             #print '#', k, v, map[k]
2277             assert(k in map)
2278             assert v == map[k]
2279
2280 class AssertContentInvariant(object):
2281     def __init__(self, callable, *args, **kwargs):
2282         self.callable = callable
2283         self.args = args
2284         self.kwargs = kwargs
2285
2286     def __enter__(self):
2287         self.content = self.callable(*self.args, **self.kwargs)[2]
2288         return self.content
2289
2290     def __exit__(self, type, value, tb):
2291         content = self.callable(*self.args, **self.kwargs)[2]
2292         assert self.content == content
2293
2294 def get_content_splitted(response):
2295     if response:
2296         return response.content.split('\n')
2297
2298 def compute_md5_hash(data):
2299     md5 = hashlib.md5()
2300     offset = 0
2301     md5.update(data)
2302     return md5.hexdigest().lower()
2303
2304 def compute_block_hash(data, algorithm):
2305     h = hashlib.new(algorithm)
2306     h.update(data.rstrip('\x00'))
2307     return h.hexdigest()
2308
2309 def get_random_data(length=500):
2310     char_set = string.ascii_uppercase + string.digits
2311     return ''.join(random.choice(char_set) for x in xrange(length))
2312
2313 def is_date(date):
2314     MONTHS = 'jan feb mar apr may jun jul aug sep oct nov dec'.split()
2315     __D = r'(?P<day>\d{2})'
2316     __D2 = r'(?P<day>[ \d]\d)'
2317     __M = r'(?P<mon>\w{3})'
2318     __Y = r'(?P<year>\d{4})'
2319     __Y2 = r'(?P<year>\d{2})'
2320     __T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
2321     RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
2322     RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
2323     ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
2324     for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
2325         m = regex.match(date)
2326         if m is not None:
2327             return True
2328     return False
2329
2330 o_names = ['kate.jpg',
2331            'kate_beckinsale.jpg',
2332            'How To Win Friends And Influence People.pdf',
2333            'moms_birthday.jpg',
2334            'poodle_strut.mov',
2335            'Disturbed - Down With The Sickness.mp3',
2336            'army_of_darkness.avi',
2337            'the_mad.avi',
2338            'photos/animals/dogs/poodle.jpg',
2339            'photos/animals/dogs/terrier.jpg',
2340            'photos/animals/cats/persian.jpg',
2341            'photos/animals/cats/siamese.jpg',
2342            'photos/plants/fern.jpg',
2343            'photos/plants/rose.jpg',
2344            'photos/me.jpg']
2345
2346
2347 def main():
2348     if get_user() == 'test':
2349         unittest.main(module='pithos.tools.test')
2350     else:
2351         print 'Will not run tests as any other user except \'test\' (current user: %s).' % get_user()
2352
2353
2354 if __name__ == "__main__":
2355     main()
2356