code.grnet.gr Git - pithos/blob - snf-pithos-tools/pithos/tools/test.py

   1 #!/usr/bin/env python
   2 #coding=utf8
   3
   4 # Copyright 2011-2012 GRNET S.A. All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or
   7 # without modification, are permitted provided that the following
   8 # conditions are met:
   9 #
  10 #   1. Redistributions of source code must retain the above
  11 #      copyright notice, this list of conditions and the following
  12 #      disclaimer.
  13 #
  14 #   2. Redistributions in binary form must reproduce the above
  15 #      copyright notice, this list of conditions and the following
  16 #      disclaimer in the documentation and/or other materials
  17 #      provided with the distribution.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  20 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  21 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  23 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  26 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  27 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  29 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 # POSSIBILITY OF SUCH DAMAGE.
  31 #
  32 # The views and conclusions contained in the software and
  33 # documentation are those of the authors and should not be
  34 # interpreted as representing official policies, either expressed
  35 # or implied, of GRNET S.A.
  36
  37 from pithos.tools.lib.client import Pithos_Client, Fault
  38 from pithos.tools.lib.util import get_user, get_auth, get_url
  39
  40 from xml.dom import minidom
  41 from StringIO import StringIO
  42 from hashlib import new as newhasher
  43 from binascii import hexlify
  44 from httplib import HTTPConnection
  45 from urlparse import urlparse
  46
  47 import json
  48 import unittest
  49 import time as _time
  50 import types
  51 import hashlib
  52 import mimetypes
  53 import random
  54 import datetime
  55 import string
  56 import re
  57
  58 DATE_FORMATS = ["%a %b %d %H:%M:%S %Y",
  59                 "%A, %d-%b-%y %H:%M:%S GMT",
  60                 "%a, %d %b %Y %H:%M:%S GMT"]
  61
  62 OTHER_ACCOUNTS = {
  63     '0001': 'verigak',
  64     '0002': 'chazapis',
  65     '0003': 'gtsouk',
  66     '0004': 'papagian',
  67     '0005': 'louridas',
  68     '0006': 'chstath',
  69     '0007': 'pkanavos',
  70     '0008': 'mvasilak',
  71     '0009': 'διογένης'}
  72
  73 class BaseTestCase(unittest.TestCase):
  74     #TODO unauthorized request
  75     def setUp(self):
  76         self.client = Pithos_Client(get_url(), get_auth(), get_user())
  77         self._clean_account()
  78         self.invalid_client = Pithos_Client(get_url(), get_auth(), 'invalid')
  79
  80         #keep track of initial account groups
  81         self.initial_groups = self.client.retrieve_account_groups()
  82
  83         #keep track of initial account meta
  84         self.initial_meta = self.client.retrieve_account_metadata(restricted=True)
  85
  86         self.extended = {
  87             'container':(
  88                 'name',
  89                 'count',
  90                 'bytes',
  91                 'last_modified',
  92                 'x_container_policy'),
  93             'object':(
  94                 'name',
  95                 'hash',
  96                 'bytes',
  97                 'content_type',
  98                 'content_encoding',
  99                 'last_modified',)}
 100         self.return_codes = (400, 401, 403, 404, 503,)
 101
 102     def tearDown(self):
 103         #delete additionally created meta
 104         l = []
 105         for m in self.client.retrieve_account_metadata(restricted=True):
 106             if m not in self.initial_meta:
 107                 l.append(m)
 108         self.client.delete_account_metadata(l)
 109
 110         #delete additionally created groups
 111         l = []
 112         for g in self.client.retrieve_account_groups():
 113             if g not in self.initial_groups:
 114                 l.append(g)
 115         self.client.unset_account_groups(l)
 116         self._clean_account()
 117
 118     def _clean_account(self):
 119         for c in self.client.list_containers():
 120             while True:
 121                 #list objects returns at most 10000 objects
 122                 #so repeat until there are no more objects
 123                 objects = self.client.list_objects(c)
 124                 if not objects:
 125                     break
 126                 for o in objects:
 127                     self.client.delete_object(c, o)
 128             self.client.delete_container(c)
 129
 130     def assert_status(self, status, codes):
 131         l = [elem for elem in self.return_codes]
 132         if type(codes) == types.ListType:
 133             l.extend(codes)
 134         else:
 135             l.append(codes)
 136         self.assertTrue(status in l)
 137
 138     def assert_extended(self, data, format, type, size=10000):
 139         if format == 'xml':
 140             self._assert_xml(data, type, size)
 141         elif format == 'json':
 142             self._assert_json(data, type, size)
 143
 144     def _assert_json(self, data, type, size):
 145         convert = lambda s: s.lower()
 146         info = [convert(elem) for elem in self.extended[type]]
 147         self.assertTrue(len(data) <= size)
 148         for item in info:
 149             for i in data:
 150                 if 'subdir' in i.keys():
 151                     continue
 152                 self.assertTrue(item in i.keys())
 153
 154     def _assert_xml(self, data, type, size):
 155         convert = lambda s: s.lower()
 156         info = [convert(elem) for elem in self.extended[type]]
 157         try:
 158             info.remove('content_encoding')
 159         except ValueError:
 160             pass
 161         xml = data
 162         entities = xml.getElementsByTagName(type)
 163         self.assertTrue(len(entities) <= size)
 164         for e in entities:
 165             for item in info:
 166                 self.assertTrue(e.getElementsByTagName(item))
 167
 168     def assert_raises_fault(self, status, callableObj, *args, **kwargs):
 169         """
 170         asserts that a Fault with a specific status is raised
 171         when callableObj is called with the specific arguments
 172         """
 173         try:
 174             r = callableObj(*args, **kwargs)
 175             self.fail('Should never reach here')
 176         except Fault, f:
 177             if type(status) == types.ListType:
 178                 self.failUnless(f.status in status)
 179             else:
 180                 self.failUnless(f.status == status)
 181
 182     def assert_not_raises_fault(self, status, callableObj, *args, **kwargs):
 183         """
 184         asserts that a Fault with a specific status is not raised
 185         when callableObj is called with the specific arguments
 186         """
 187         try:
 188             r = callableObj(*args, **kwargs)
 189         except Fault, f:
 190             self.failIfEqual(f.status, status)
 191
 192     def assert_container_exists(self, container):
 193         """
 194         asserts the existence of a container
 195         """
 196         try:
 197             self.client.retrieve_container_metadata(container)
 198         except Fault, f:
 199             self.failIf(f.status == 404)
 200
 201     def assert_container_not_exists(self, container):
 202         """
 203         asserts there is no such a container
 204         """
 205         self.assert_raises_fault(404, self.client.retrieve_container_metadata,
 206                                  container)
 207
 208     def assert_object_exists(self, container, object):
 209         """
 210         asserts the existence of an object
 211         """
 212         try:
 213             self.client.retrieve_object_metadata(container, object)
 214         except Fault, f:
 215             self.failIf(f.status == 404)
 216
 217     def assert_object_not_exists(self, container, object):
 218         """
 219         asserts there is no such an object
 220         """
 221         self.assert_raises_fault(404, self.client.retrieve_object_metadata,
 222                                  container, object)
 223
 224     def assert_versionlist_structure(self, versionlist):
 225         self.assertTrue(type(versionlist) == types.ListType)
 226         for elem in versionlist:
 227             self.assertTrue(type(elem) == types.ListType)
 228             self.assertEqual(len(elem), 2)
 229
 230     def upload_random_data(self, container, name, length=1024, type=None,
 231                            enc=None, **meta):
 232         data = get_random_data(length)
 233         return self.upload_data(container, name, data, type, enc, **meta)
 234
 235     def upload_data(self, container, name, data, type=None, enc=None, etag=None,
 236                     **meta):
 237         obj = {}
 238         obj['name'] = name
 239         try:
 240             obj['data'] = data
 241             obj['hash'] = compute_md5_hash(obj['data'])
 242
 243             args = {}
 244             args['etag'] = etag if etag else obj['hash']
 245
 246             try:
 247                 guess = mimetypes.guess_type(name)
 248                 type = type if type else guess[0]
 249                 enc = enc if enc else guess[1]
 250             except:
 251                 pass
 252             args['content_type'] = type if type else 'plain/text'
 253             args['content_encoding'] = enc if enc else None
 254
 255             obj['meta'] = args
 256
 257             path = '/%s/%s' % (container, name)
 258             self.client.create_object(container, name, f=StringIO(obj['data']),
 259                                       meta=meta, **args)
 260
 261             return obj
 262         except IOError:
 263             return
 264
 265 class AccountHead(BaseTestCase):
 266     def setUp(self):
 267         BaseTestCase.setUp(self)
 268         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 269         for item in self.containers:
 270             self.client.create_container(item)
 271
 272         meta = {'foo':'bar'}
 273         self.client.update_account_metadata(**meta)
 274         #self.updated_meta = self.initial_meta.update(meta)
 275
 276     def test_get_account_meta(self):
 277         meta = self.client.retrieve_account_metadata()
 278
 279         containers = self.client.list_containers()
 280         l = str(len(containers))
 281         self.assertEqual(meta['x-account-container-count'], l)
 282         size = 0
 283         for c in containers:
 284             m = self.client.retrieve_container_metadata(c)
 285             size = size + int(m['x-container-bytes-used'])
 286         self.assertEqual(meta['x-account-bytes-used'], str(size))
 287
 288     def test_get_account_403(self):
 289         self.assert_raises_fault(403,
 290                                  self.invalid_client.retrieve_account_metadata)
 291
 292     def test_get_account_meta_until(self):
 293         t = datetime.datetime.utcnow()
 294         past = t - datetime.timedelta(minutes=-15)
 295         past = int(_time.mktime(past.timetuple()))
 296
 297         meta = {'premium':True}
 298         self.client.update_account_metadata(**meta)
 299         meta = self.client.retrieve_account_metadata(restricted=True,
 300                                                      until=past)
 301         self.assertTrue('premium' not in meta)
 302
 303         meta = self.client.retrieve_account_metadata(restricted=True)
 304         self.assertTrue('premium' in meta)
 305
 306     def test_get_account_meta_until_invalid_date(self):
 307         meta = {'premium':True}
 308         self.client.update_account_metadata(**meta)
 309         meta = self.client.retrieve_account_metadata(restricted=True,
 310                                                      until='kshfksfh')
 311         self.assertTrue('premium' in meta)
 312
 313 class AccountGet(BaseTestCase):
 314     def setUp(self):
 315         BaseTestCase.setUp(self)
 316         #create some containers
 317         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 318         for item in self.containers:
 319             self.client.create_container(item)
 320
 321     def test_list(self):
 322         #list containers
 323         containers = self.client.list_containers()
 324         self.assertEquals(self.containers, containers)
 325
 326     def test_list_403(self):
 327         self.assert_raises_fault(403, self.invalid_client.list_containers)
 328
 329     def test_list_with_limit(self):
 330         limit = 2
 331         containers = self.client.list_containers(limit=limit)
 332         self.assertEquals(len(containers), limit)
 333         self.assertEquals(self.containers[:2], containers)
 334
 335     def test_list_with_marker(self):
 336         l = 2
 337         m = 'bananas'
 338         containers = self.client.list_containers(limit=l, marker=m)
 339         i = self.containers.index(m) + 1
 340         self.assertEquals(self.containers[i:(i+l)], containers)
 341
 342         m = 'oranges'
 343         containers = self.client.list_containers(limit=l, marker=m)
 344         i = self.containers.index(m) + 1
 345         self.assertEquals(self.containers[i:(i+l)], containers)
 346
 347     def test_list_json_with_marker(self):
 348         l = 2
 349         m = 'bananas'
 350         containers = self.client.list_containers(limit=l, marker=m, format='json')
 351         self.assert_extended(containers, 'json', 'container', l)
 352         self.assertEqual(containers[0]['name'], 'kiwis')
 353         self.assertEqual(containers[1]['name'], 'oranges')
 354
 355     def test_list_xml_with_marker(self):
 356         l = 2
 357         m = 'oranges'
 358         xml = self.client.list_containers(limit=l, marker=m, format='xml')
 359         self.assert_extended(xml, 'xml', 'container', l)
 360         nodes = xml.getElementsByTagName('name')
 361         self.assertEqual(len(nodes), 1)
 362         self.assertEqual(nodes[0].childNodes[0].data, 'pears')
 363
 364     def test_if_modified_since(self):
 365         t = datetime.datetime.utcnow()
 366         t2 = t - datetime.timedelta(minutes=10)
 367
 368         #add a new container
 369         self.client.create_container('dummy')
 370
 371         for f in DATE_FORMATS:
 372             past = t2.strftime(f)
 373             try:
 374                 c = self.client.list_containers(if_modified_since=past)
 375                 self.assertEqual(len(c), len(self.containers) + 1)
 376             except Fault, f:
 377                 self.failIf(f.status == 304) #fail if not modified
 378
 379     def test_if_modified_since_invalid_date(self):
 380         c = self.client.list_containers(if_modified_since='')
 381         self.assertEqual(len(c), len(self.containers))
 382
 383     def test_if_not_modified_since(self):
 384         now = datetime.datetime.utcnow()
 385         since = now + datetime.timedelta(1)
 386
 387         for f in DATE_FORMATS:
 388             args = {'if_modified_since':'%s' %since.strftime(f)}
 389
 390             #assert not modified
 391             self.assert_raises_fault(304, self.client.list_containers, **args)
 392
 393     def test_if_unmodified_since(self):
 394         now = datetime.datetime.utcnow()
 395         since = now + datetime.timedelta(1)
 396
 397         for f in DATE_FORMATS:
 398             c = self.client.list_containers(if_unmodified_since=since.strftime(f))
 399
 400             #assert success
 401             self.assertEqual(self.containers, c)
 402
 403     def test_if_unmodified_since_precondition_failed(self):
 404         t = datetime.datetime.utcnow()
 405         t2 = t - datetime.timedelta(minutes=10)
 406
 407         #add a new container
 408         self.client.create_container('dummy')
 409
 410         for f in DATE_FORMATS:
 411             past = t2.strftime(f)
 412
 413             args = {'if_unmodified_since':'%s' %past}
 414
 415             #assert precondition failed
 416             self.assert_raises_fault(412, self.client.list_containers, **args)
 417
 418 class AccountPost(BaseTestCase):
 419     def setUp(self):
 420         BaseTestCase.setUp(self)
 421         self.containers = ['apples', 'bananas', 'kiwis', 'oranges', 'pears']
 422         for item in self.containers:
 423             self.client.create_container(item)
 424
 425         meta = {'foo':'bar'}
 426         self.client.update_account_metadata(**meta)
 427         self.updated_meta = self.initial_meta.update(meta)
 428
 429     def test_update_meta(self):
 430         with AssertMappingInvariant(self.client.retrieve_account_groups):
 431             meta = {'test':'test', 'tost':'tost'}
 432             self.client.update_account_metadata(**meta)
 433
 434             meta.update(self.initial_meta)
 435             self.assertEqual(meta,
 436                              self.client.retrieve_account_metadata(
 437                                 restricted=True))
 438
 439     def test_invalid_account_update_meta(self):
 440         meta = {'test':'test', 'tost':'tost'}
 441         self.assert_raises_fault(403,
 442                                  self.invalid_client.update_account_metadata,
 443                                  **meta)
 444
 445     def test_reset_meta(self):
 446         with AssertMappingInvariant(self.client.retrieve_account_groups):
 447             meta = {'test':'test', 'tost':'tost'}
 448             self.client.update_account_metadata(**meta)
 449
 450             meta = {'test':'test33'}
 451             self.client.reset_account_metadata(**meta)
 452
 453             self.assertEqual(meta, self.client.retrieve_account_metadata(restricted=True))
 454
 455     def test_delete_meta(self):
 456         with AssertMappingInvariant(self.client.retrieve_account_groups):
 457             meta = {'test':'test', 'tost':'tost'}
 458             self.client.update_account_metadata(**meta)
 459
 460             self.client.delete_account_metadata(meta.keys())
 461
 462             account_meta = self.client.retrieve_account_metadata(restricted=True)
 463             for m in meta:
 464                 self.assertTrue(m not in account_meta.keys())
 465
 466     def test_set_account_groups(self):
 467         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 468             groups = {'pithosdev':'verigak,gtsouk,chazapis'}
 469             self.client.set_account_groups(**groups)
 470
 471             self.assertEqual(set(groups['pithosdev']),
 472                              set(self.client.retrieve_account_groups()['pithosdev']))
 473
 474             more_groups = {'clientsdev':'pkanavos,mvasilak'}
 475             self.client.set_account_groups(**more_groups)
 476
 477             groups.update(more_groups)
 478             self.assertEqual(set(groups['clientsdev']),
 479                              set(self.client.retrieve_account_groups()['clientsdev']))
 480
 481     def test_reset_account_groups(self):
 482         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 483             groups = {'pithosdev':'verigak,gtsouk,chazapis',
 484                       'clientsdev':'pkanavos,mvasilak'}
 485             self.client.set_account_groups(**groups)
 486
 487             self.assertEqual(set(groups['pithosdev'].split(',')),
 488                              set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
 489             self.assertEqual(set(groups['clientsdev'].split(',')),
 490                              set(self.client.retrieve_account_groups()['clientsdev'].split(',')))
 491
 492             groups = {'pithosdev':'verigak,gtsouk,chazapis,papagian'}
 493             self.client.reset_account_groups(**groups)
 494
 495             self.assertEqual(set(groups['pithosdev'].split(',')),
 496                              set(self.client.retrieve_account_groups()['pithosdev'].split(',')))
 497
 498     def test_delete_account_groups(self):
 499         with AssertMappingInvariant(self.client.retrieve_account_metadata):
 500             groups = {'pithosdev':'verigak,gtsouk,chazapis',
 501                       'clientsdev':'pkanavos,mvasilak'}
 502             self.client.set_account_groups(**groups)
 503
 504             self.client.unset_account_groups(groups.keys())
 505
 506             self.assertEqual({}, self.client.retrieve_account_groups())
 507
 508 class ContainerHead(BaseTestCase):
 509     def setUp(self):
 510         BaseTestCase.setUp(self)
 511         self.container = 'apples'
 512         self.client.create_container(self.container)
 513
 514     def test_get_meta(self):
 515         meta = {'trash':'true'}
 516         t1 = datetime.datetime.utcnow()
 517         o = self.upload_random_data(self.container, o_names[0], **meta)
 518         if o:
 519             headers = self.client.retrieve_container_metadata(self.container)
 520             self.assertEqual(headers['x-container-object-count'], '1')
 521             self.assertEqual(headers['x-container-bytes-used'], str(len(o['data'])))
 522             t2 = datetime.datetime.strptime(headers['last-modified'], DATE_FORMATS[2])
 523             delta = (t2 - t1)
 524             threashold = datetime.timedelta(seconds=1)
 525             self.assertTrue(delta < threashold)
 526             self.assertTrue(headers['x-container-object-meta'])
 527             self.assertTrue('Trash' in headers['x-container-object-meta'])
 528
 529 class ContainerGet(BaseTestCase):
 530     def setUp(self):
 531         BaseTestCase.setUp(self)
 532         self.container = ['pears', 'apples']
 533         for c in self.container:
 534             self.client.create_container(c)
 535         self.obj = []
 536         for o in o_names[:8]:
 537             self.obj.append(self.upload_random_data(self.container[0], o))
 538         for o in o_names[8:]:
 539             self.obj.append(self.upload_random_data(self.container[1], o))
 540
 541     def test_list_objects(self):
 542         objects = self.client.list_objects(self.container[0])
 543         l = [elem['name'] for elem in self.obj[:8]]
 544         l.sort()
 545         self.assertEqual(objects, l)
 546
 547     def test_list_objects_containing_slash(self):
 548         self.client.create_container('test')
 549         self.upload_random_data('test', '/objectname')
 550
 551         objects = self.client.list_objects('test')
 552         self.assertEqual(objects, ['/objectname'])
 553
 554         objects = self.client.list_objects('test', format='json')
 555         self.assertEqual(objects[0]['name'], '/objectname')
 556
 557         objects = self.client.list_objects('test', format='xml')
 558         self.assert_extended(objects, 'xml', 'object')
 559         node_name = objects.getElementsByTagName('name')[0]
 560         self.assertEqual(node_name.firstChild.data, '/objectname')
 561
 562     def test_list_objects_with_limit_marker(self):
 563         objects = self.client.list_objects(self.container[0], limit=2)
 564         l = [elem['name'] for elem in self.obj[:8]]
 565         l.sort()
 566         self.assertEqual(objects, l[:2])
 567
 568         markers = ['How To Win Friends And Influence People.pdf',
 569                    'moms_birthday.jpg']
 570         limit = 4
 571         for m in markers:
 572             objects = self.client.list_objects(self.container[0], limit=limit,
 573                                                marker=m)
 574             l = [elem['name'] for elem in self.obj[:8]]
 575             l.sort()
 576             start = l.index(m) + 1
 577             end = start + limit
 578             end = end if len(l) >= end else len(l)
 579             self.assertEqual(objects, l[start:end])
 580
 581     #takes too long
 582     def _test_list_limit_exceeds(self):
 583         self.client.create_container('pithos')
 584
 585         for i in range(10001):
 586             self.client.create_zero_length_object('pithos', i)
 587
 588         self.assertEqual(10000, len(self.client.list_objects('pithos')))
 589
 590     def test_list_empty_params(self):
 591         objects = self.client.get('/%s/%s' % (get_user(), self.container[0]))[2]
 592         if objects:
 593             objects = objects.strip().split('\n')
 594         self.assertEqual(objects,
 595                          self.client.list_objects(self.container[0]))
 596
 597     def test_list_pseudo_hierarchical_folders(self):
 598         objects = self.client.list_objects(self.container[1], prefix='photos',
 599                                            delimiter='/')
 600         self.assertEquals(['photos/animals/', 'photos/me.jpg',
 601                            'photos/plants/'], objects)
 602
 603         objects = self.client.list_objects(self.container[1],
 604                                            prefix='photos/animals',
 605                                            delimiter='/')
 606         l = ['photos/animals/cats/', 'photos/animals/dogs/']
 607         self.assertEquals(l, objects)
 608
 609         objects = self.client.list_objects(self.container[1], path='photos')
 610         self.assertEquals(['photos/me.jpg'], objects)
 611
 612     def test_extended_list_json(self):
 613         objects = self.client.list_objects(self.container[1], format='json',
 614                                            limit=2, prefix='photos/animals',
 615                                            delimiter='/')
 616         self.assertEqual(objects[0]['subdir'], 'photos/animals/cats/')
 617         self.assertEqual(objects[1]['subdir'], 'photos/animals/dogs/')
 618
 619     def test_extended_list_xml(self):
 620         xml = self.client.list_objects(self.container[1], format='xml', limit=4,
 621                                        prefix='photos', delimiter='/')
 622         self.assert_extended(xml, 'xml', 'object', size=4)
 623         dirs = xml.getElementsByTagName('subdir')
 624         self.assertEqual(len(dirs), 2)
 625         self.assertEqual(dirs[0].attributes['name'].value, 'photos/animals/')
 626         self.assertEqual(dirs[1].attributes['name'].value, 'photos/plants/')
 627
 628         objects = xml.getElementsByTagName('name')
 629         self.assertEqual(len(objects), 1)
 630         self.assertEqual(objects[0].childNodes[0].data, 'photos/me.jpg')
 631
 632     def test_list_meta_double_matching(self):
 633         meta = {'quality':'aaa', 'stock':'true'}
 634         self.client.update_object_metadata(self.container[0],
 635                                            self.obj[0]['name'], **meta)
 636         obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
 637         self.assertEqual(len(obj), 1)
 638         self.assertTrue(obj, self.obj[0]['name'])
 639
 640     def test_list_using_meta(self):
 641         meta = {'quality':'aaa'}
 642         for o in self.obj[:2]:
 643             self.client.update_object_metadata(self.container[0], o['name'],
 644                                                **meta)
 645         meta = {'stock':'true'}
 646         for o in self.obj[3:5]:
 647             self.client.update_object_metadata(self.container[0], o['name'],
 648                                                **meta)
 649
 650         obj = self.client.list_objects(self.container[0], meta='Quality')
 651         self.assertEqual(len(obj), 2)
 652         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 653
 654         # test case insensitive
 655         obj = self.client.list_objects(self.container[0], meta='quality')
 656         self.assertEqual(len(obj), 2)
 657         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 658
 659         # test multiple matches
 660         obj = self.client.list_objects(self.container[0], meta='Quality,Stock')
 661         self.assertEqual(len(obj), 4)
 662         self.assertTrue(obj, [o['name'] for o in self.obj[:4]])
 663
 664         # test non 1-1 multiple match
 665         obj = self.client.list_objects(self.container[0], meta='Quality,aaaa')
 666         self.assertEqual(len(obj), 2)
 667         self.assertTrue(obj, [o['name'] for o in self.obj[:2]])
 668
 669     def test_if_modified_since(self):
 670         t = datetime.datetime.utcnow()
 671         t2 = t - datetime.timedelta(minutes=10)
 672
 673         #add a new object
 674         self.upload_random_data(self.container[0], o_names[0])
 675
 676         for f in DATE_FORMATS:
 677             past = t2.strftime(f)
 678             try:
 679                 o = self.client.list_objects(self.container[0],
 680                                             if_modified_since=past)
 681                 self.assertEqual(o,
 682                                  self.client.list_objects(self.container[0]))
 683             except Fault, f:
 684                 self.failIf(f.status == 304) #fail if not modified
 685
 686     def test_if_modified_since_invalid_date(self):
 687         headers = {'if-modified-since':''}
 688         o = self.client.list_objects(self.container[0], if_modified_since='')
 689         self.assertEqual(o, self.client.list_objects(self.container[0]))
 690
 691     def test_if_not_modified_since(self):
 692         now = datetime.datetime.utcnow()
 693         since = now + datetime.timedelta(1)
 694
 695         for f in DATE_FORMATS:
 696             args = {'if_modified_since':'%s' %since.strftime(f)}
 697
 698             #assert not modified
 699             self.assert_raises_fault(304, self.client.list_objects,
 700                                      self.container[0], **args)
 701
 702     def test_if_unmodified_since(self):
 703         now = datetime.datetime.utcnow()
 704         since = now + datetime.timedelta(1)
 705
 706         for f in DATE_FORMATS:
 707             obj = self.client.list_objects(self.container[0],
 708                                            if_unmodified_since=since.strftime(f))
 709
 710             #assert unmodified
 711             self.assertEqual(obj, self.client.list_objects(self.container[0]))
 712
 713     def test_if_unmodified_since_precondition_failed(self):
 714         t = datetime.datetime.utcnow()
 715         t2 = t - datetime.timedelta(minutes=10)
 716
 717         #add a new container
 718         self.client.create_container('dummy')
 719
 720         for f in DATE_FORMATS:
 721             past = t2.strftime(f)
 722
 723             args = {'if_unmodified_since':'%s' %past}
 724
 725             #assert precondition failed
 726             self.assert_raises_fault(412, self.client.list_objects,
 727                                      self.container[0], **args)
 728
 729 class ContainerPut(BaseTestCase):
 730     def setUp(self):
 731         BaseTestCase.setUp(self)
 732         self.containers = ['c1', 'c2']
 733
 734     def test_create(self):
 735         self.client.create_container(self.containers[0])
 736         containers = self.client.list_containers()
 737         self.assertTrue(self.containers[0] in containers)
 738         self.assert_container_exists(self.containers[0])
 739
 740     def test_create_twice(self):
 741         self.client.create_container(self.containers[0])
 742         self.assertTrue(not self.client.create_container(self.containers[0]))
 743
 744     def test_quota(self):
 745         self.client.create_container(self.containers[0])
 746
 747         policy = {'quota':100}
 748         self.client.set_container_policies('c1', **policy)
 749
 750         meta = self.client.retrieve_container_metadata('c1')
 751         self.assertTrue('x-container-policy-quota' in meta)
 752         self.assertEqual(meta['x-container-policy-quota'], '100')
 753
 754         args = ['c1', 'o1']
 755         kwargs = {'length':101}
 756         self.assert_raises_fault(413, self.upload_random_data, *args, **kwargs)
 757
 758         #reset quota
 759         policy = {'quota':0}
 760         self.client.set_container_policies('c1', **policy)
 761
 762 class ContainerPost(BaseTestCase):
 763     def setUp(self):
 764         BaseTestCase.setUp(self)
 765         self.container = 'apples'
 766         self.client.create_container(self.container)
 767
 768     def test_update_meta(self):
 769         meta = {'test':'test33',
 770                 'tost':'tost22'}
 771         self.client.update_container_metadata(self.container, **meta)
 772         headers = self.client.retrieve_container_metadata(self.container)
 773         for k,v in meta.items():
 774             k = 'x-container-meta-%s' % k
 775             self.assertTrue(headers[k])
 776             self.assertEqual(headers[k], v)
 777
 778 class ContainerDelete(BaseTestCase):
 779     def setUp(self):
 780         BaseTestCase.setUp(self)
 781         self.containers = ['c1', 'c2']
 782         for c in self.containers:
 783             self.client.create_container(c)
 784
 785     def test_delete(self):
 786         status = self.client.delete_container(self.containers[0])[0]
 787         self.assertEqual(status, 204)
 788
 789     def test_delete_non_empty(self):
 790         self.upload_random_data(self.containers[1], o_names[0])
 791         self.assert_raises_fault(409, self.client.delete_container,
 792                                  self.containers[1])
 793
 794     def test_delete_invalid(self):
 795         self.assert_raises_fault(404, self.client.delete_container, 'c3')
 796
 797 class ObjectGet(BaseTestCase):
 798     def setUp(self):
 799         BaseTestCase.setUp(self)
 800         self.containers = ['c1', 'c2']
 801         #create some containers
 802         for c in self.containers:
 803             self.client.create_container(c)
 804
 805         #upload a file
 806         names = ('obj1', 'obj2')
 807         self.objects = []
 808         for n in names:
 809             self.objects.append(self.upload_random_data(self.containers[1], n))
 810
 811     def test_versions(self):
 812         c = self.containers[1]
 813         o = self.objects[0]
 814         b = self.client.retrieve_object_versionlist(c, o['name'])['versions']
 815         self.assert_versionlist_structure(b)
 816
 817         #update meta
 818         meta = {'quality':'AAA', 'stock':True}
 819         self.client.update_object_metadata(c, o['name'], **meta)
 820
 821         a = self.client.retrieve_object_versionlist(c, o['name'])['versions']
 822         self.assert_versionlist_structure(a)
 823         self.assertEqual(len(b)+1, len(a))
 824         self.assertEqual(b, a[:-1])
 825
 826         #get exact previous version metadata
 827         v = a[-2][0]
 828         v_meta = self.client.retrieve_object_metadata(c, o['name'],
 829                                                       restricted=True,
 830                                                       version=v)
 831         for k in meta.keys():
 832             self.assertTrue(k not in v_meta)
 833
 834         #update obejct
 835         data = get_random_data()
 836         self.client.update_object(c, o['name'], StringIO(data))
 837
 838         aa = self.client.retrieve_object_versionlist(c, o['name'])['versions']
 839         self.assert_versionlist_structure(aa)
 840         self.assertEqual(len(a)+1, len(aa))
 841         self.assertEqual(a, aa[:-1])
 842
 843         #get exact previous version
 844         v = aa[-3][0]
 845         v_data = self.client.retrieve_object_version(c, o['name'], version=v)
 846         self.assertEqual(o['data'], v_data)
 847         self.assertEqual(self.client.retrieve_object(c, o['name']),
 848                          '%s%s' %(v_data, data))
 849
 850     def test_get(self):
 851         #perform get
 852         o = self.client.retrieve_object(self.containers[1],
 853                                         self.objects[0]['name'],
 854                                         self.objects[0]['meta'])
 855         self.assertEqual(o, self.objects[0]['data'])
 856
 857     def test_objects_with_trailing_spaces(self):
 858         self.client.create_container('test')
 859         #create 'a' object
 860         self.upload_random_data('test', 'a')
 861         #look for 'a ' object
 862         self.assert_raises_fault(404, self.client.retrieve_object,
 863                                  'test', 'a ')
 864
 865         #delete 'a' object
 866         self.client.delete_object('test', 'a')
 867         self.assert_raises_fault(404, self.client.retrieve_object,
 868                                  'test', 'a')
 869
 870         #create 'a ' object
 871         self.upload_random_data('test', 'a ')
 872         #look for 'a' object
 873         self.assert_raises_fault(404, self.client.retrieve_object,
 874                                  'test', 'a')
 875
 876     def test_get_invalid(self):
 877         self.assert_raises_fault(404, self.client.retrieve_object,
 878                                  self.containers[0], self.objects[0]['name'])
 879
 880     def test_get_partial(self):
 881         #perform get with range
 882         status, headers, data = self.client.request_object(self.containers[1],
 883                                                             self.objects[0]['name'],
 884                                                             range='bytes=0-499')
 885
 886         #assert successful partial content
 887         self.assertEqual(status, 206)
 888
 889         #assert content-type
 890         self.assertEqual(headers['content-type'],
 891                          self.objects[0]['meta']['content_type'])
 892
 893         #assert content length
 894         self.assertEqual(int(headers['content-length']), 500)
 895
 896         #assert content
 897         self.assertEqual(self.objects[0]['data'][:500], data)
 898
 899     def test_get_final_500(self):
 900         #perform get with range
 901         headers = {'range':'bytes=-500'}
 902         status, headers, data = self.client.request_object(self.containers[1],
 903                                                             self.objects[0]['name'],
 904                                                             range='bytes=-500')
 905
 906         #assert successful partial content
 907         self.assertEqual(status, 206)
 908
 909         #assert content-type
 910         self.assertEqual(headers['content-type'],
 911                          self.objects[0]['meta']['content_type'])
 912
 913         #assert content length
 914         self.assertEqual(int(headers['content-length']), 500)
 915
 916         #assert content
 917         self.assertTrue(self.objects[0]['data'][-500:], data)
 918
 919     def test_get_rest(self):
 920         #perform get with range
 921         offset = len(self.objects[0]['data']) - 500
 922         status, headers, data = self.client.request_object(self.containers[1],
 923                                                 self.objects[0]['name'],
 924                                                 range='bytes=%s-' %offset)
 925
 926         #assert successful partial content
 927         self.assertEqual(status, 206)
 928
 929         #assert content-type
 930         self.assertEqual(headers['content-type'],
 931                          self.objects[0]['meta']['content_type'])
 932
 933         #assert content length
 934         self.assertEqual(int(headers['content-length']), 500)
 935
 936         #assert content
 937         self.assertTrue(self.objects[0]['data'][-500:], data)
 938
 939     def test_get_range_not_satisfiable(self):
 940         #perform get with range
 941         offset = len(self.objects[0]['data']) + 1
 942
 943         #assert range not satisfiable
 944         self.assert_raises_fault(416, self.client.retrieve_object,
 945                                  self.containers[1], self.objects[0]['name'],
 946                                  range='bytes=0-%s' %offset)
 947
 948     def test_multiple_range(self):
 949         #perform get with multiple range
 950         ranges = ['0-499', '-500', '1000-']
 951         bytes = 'bytes=%s' % ','.join(ranges)
 952         status, headers, data = self.client.request_object(self.containers[1],
 953                                                            self.objects[0]['name'],
 954                                                            range=bytes)
 955
 956         # assert partial content
 957         self.assertEqual(status, 206)
 958
 959         # assert Content-Type of the reply will be multipart/byteranges
 960         self.assertTrue(headers['content-type'])
 961         content_type_parts = headers['content-type'].split()
 962         self.assertEqual(content_type_parts[0], ('multipart/byteranges;'))
 963
 964         boundary = '--%s' %content_type_parts[1].split('=')[-1:][0]
 965         cparts = data.split(boundary)[1:-1]
 966
 967         # assert content parts are exactly 2
 968         self.assertEqual(len(cparts), len(ranges))
 969
 970         # for each content part assert headers
 971         i = 0
 972         for cpart in cparts:
 973             content = cpart.split('\r\n')
 974             headers = content[1:3]
 975             content_range = headers[0].split(': ')
 976             self.assertEqual(content_range[0], 'Content-Range')
 977
 978             r = ranges[i].split('-')
 979             if not r[0] and not r[1]:
 980                 pass
 981             elif not r[0]:
 982                 start = len(self.objects[0]['data']) - int(r[1])
 983                 end = len(self.objects[0]['data'])
 984             elif not r[1]:
 985                 start = int(r[0])
 986                 end = len(self.objects[0]['data'])
 987             else:
 988                 start = int(r[0])
 989                 end = int(r[1]) + 1
 990             fdata = self.objects[0]['data'][start:end]
 991             sdata = '\r\n'.join(content[4:-1])
 992             self.assertEqual(len(fdata), len(sdata))
 993             self.assertEquals(fdata, sdata)
 994             i+=1
 995
 996     def test_multiple_range_not_satisfiable(self):
 997         #perform get with multiple range
 998         out_of_range = len(self.objects[0]['data']) + 1
 999         ranges = ['0-499', '-500', '%d-' %out_of_range]
1000         bytes = 'bytes=%s' % ','.join(ranges)
1001
1002         # assert partial content
1003         self.assert_raises_fault(416, self.client.retrieve_object,
1004                                  self.containers[1],
1005                                  self.objects[0]['name'], range=bytes)
1006
1007     def test_get_with_if_match(self):
1008         #perform get with If-Match
1009         etag = self.objects[0]['hash']
1010         status, headers, data = self.client.request_object(self.containers[1],
1011                                                            self.objects[0]['name'],
1012                                                            if_match=etag)
1013         #assert get success
1014         self.assertEqual(status, 200)
1015
1016         #assert content-type
1017         self.assertEqual(headers['content-type'],
1018                          self.objects[0]['meta']['content_type'])
1019
1020         #assert response content
1021         self.assertEqual(self.objects[0]['data'], data)
1022
1023     def test_get_with_if_match_star(self):
1024         #perform get with If-Match *
1025         headers = {'if-match':'*'}
1026         status, headers, data = self.client.request_object(self.containers[1],
1027                                                 self.objects[0]['name'],
1028                                                 **headers)
1029         #assert get success
1030         self.assertEqual(status, 200)
1031
1032         #assert content-type
1033         self.assertEqual(headers['content-type'],
1034                          self.objects[0]['meta']['content_type'])
1035
1036         #assert response content
1037         self.assertEqual(self.objects[0]['data'], data)
1038
1039     def test_get_with_multiple_if_match(self):
1040         #perform get with If-Match
1041         etags = [i['hash'] for i in self.objects if i]
1042         etags = ','.join('"%s"' % etag for etag in etags)
1043         status, headers, data = self.client.request_object(self.containers[1],
1044                                                            self.objects[0]['name'],
1045                                                            if_match=etags)
1046         #assert get success
1047         self.assertEqual(status, 200)
1048
1049         #assert content-type
1050         self.assertEqual(headers['content-type'],
1051                          self.objects[0]['meta']['content_type'])
1052
1053         #assert content-type
1054         self.assertEqual(headers['content-type'],
1055                          self.objects[0]['meta']['content_type'])
1056
1057         #assert response content
1058         self.assertEqual(self.objects[0]['data'], data)
1059
1060     def test_if_match_precondition_failed(self):
1061         #assert precondition failed
1062         self.assert_raises_fault(412, self.client.retrieve_object,
1063                                  self.containers[1],
1064                                  self.objects[0]['name'], if_match='123')
1065
1066     def test_if_none_match(self):
1067         #perform get with If-None-Match
1068         status, headers, data = self.client.request_object(self.containers[1],
1069                                                            self.objects[0]['name'],
1070                                                            if_none_match='123')
1071
1072         #assert get success
1073         self.assertEqual(status, 200)
1074
1075         #assert content-type
1076         self.assertEqual(headers['content_type'],
1077                          self.objects[0]['meta']['content_type'])
1078
1079     def test_if_none_match(self):
1080         #perform get with If-None-Match * and assert not modified
1081         self.assert_raises_fault(304, self.client.retrieve_object,
1082                                  self.containers[1],
1083                                  self.objects[0]['name'],
1084                                  if_none_match='*')
1085
1086     def test_if_none_match_not_modified(self):
1087         #perform get with If-None-Match and assert not modified
1088         self.assert_raises_fault(304, self.client.retrieve_object,
1089                                  self.containers[1],
1090                                  self.objects[0]['name'],
1091                                  if_none_match=self.objects[0]['hash'])
1092
1093         meta = self.client.retrieve_object_metadata(self.containers[1],
1094                                                     self.objects[0]['name'])
1095         self.assertEqual(meta['etag'], self.objects[0]['hash'])
1096
1097     def test_if_modified_since(self):
1098         t = datetime.datetime.utcnow()
1099         t2 = t - datetime.timedelta(minutes=10)
1100
1101         #modify the object
1102         self.upload_data(self.containers[1],
1103                            self.objects[0]['name'],
1104                            self.objects[0]['data'][:200])
1105
1106         for f in DATE_FORMATS:
1107             past = t2.strftime(f)
1108
1109             headers = {'if-modified-since':'%s' %past}
1110             try:
1111                 o = self.client.retrieve_object(self.containers[1],
1112                                                 self.objects[0]['name'],
1113                                                 if_modified_since=past)
1114                 self.assertEqual(o,
1115                                  self.client.retrieve_object(self.containers[1],
1116                                                              self.objects[0]['name']))
1117             except Fault, f:
1118                 self.failIf(f.status == 304)
1119
1120     def test_if_modified_since_invalid_date(self):
1121         o = self.client.retrieve_object(self.containers[1],
1122                                         self.objects[0]['name'],
1123                                         if_modified_since='')
1124         self.assertEqual(o, self.client.retrieve_object(self.containers[1],
1125                                                         self.objects[0]['name']))
1126
1127     def test_if_not_modified_since(self):
1128         now = datetime.datetime.utcnow()
1129         since = now + datetime.timedelta(1)
1130
1131         for f in DATE_FORMATS:
1132             #assert not modified
1133             self.assert_raises_fault(304, self.client.retrieve_object,
1134                                      self.containers[1], self.objects[0]['name'],
1135                                      if_modified_since=since.strftime(f))
1136
1137     def test_if_unmodified_since(self):
1138         now = datetime.datetime.utcnow()
1139         since = now + datetime.timedelta(1)
1140
1141         for f in DATE_FORMATS:
1142             t = since.strftime(f)
1143             status, headers, data = self.client.request_object(self.containers[1],
1144                                                                self.objects[0]['name'],
1145                                                                if_unmodified_since=t)
1146             #assert success
1147             self.assertEqual(status, 200)
1148             self.assertEqual(self.objects[0]['data'], data)
1149
1150             #assert content-type
1151             self.assertEqual(headers['content-type'],
1152                              self.objects[0]['meta']['content_type'])
1153
1154     def test_if_unmodified_since_precondition_failed(self):
1155         t = datetime.datetime.utcnow()
1156         t2 = t - datetime.timedelta(minutes=10)
1157
1158         #modify the object
1159         self.upload_data(self.containers[1],
1160                            self.objects[0]['name'],
1161                            self.objects[0]['data'][:200])
1162
1163         for f in DATE_FORMATS:
1164             past = t2.strftime(f)
1165             #assert precondition failed
1166             self.assert_raises_fault(412, self.client.retrieve_object,
1167                                      self.containers[1], self.objects[0]['name'],
1168                                      if_unmodified_since=past)
1169
1170     def test_hashes(self):
1171         l = 8388609
1172         fname = 'largefile'
1173         o = self.upload_random_data(self.containers[1], fname, l)
1174         if o:
1175             body = self.client.retrieve_object(self.containers[1], fname,
1176                                                format='json')
1177             hashes = body['hashes']
1178             block_size = body['block_size']
1179             block_hash = body['block_hash']
1180             block_num = l/block_size if l/block_size == 0 else l/block_size + 1
1181             self.assertTrue(len(hashes), block_num)
1182             i = 0
1183             for h in hashes:
1184                 start = i * block_size
1185                 end = (i + 1) * block_size
1186                 hash = compute_block_hash(o['data'][start:end], block_hash)
1187                 self.assertEqual(h, hash)
1188                 i += 1
1189
1190 class ObjectPut(BaseTestCase):
1191     def setUp(self):
1192         BaseTestCase.setUp(self)
1193         self.container = 'c1'
1194         self.client.create_container(self.container)
1195
1196     def test_upload(self):
1197         name = o_names[0]
1198         meta = {'test':'test1'}
1199         o = self.upload_random_data(self.container, name, **meta)
1200
1201         headers = self.client.retrieve_object_metadata(self.container,
1202                                                        name,
1203                                                        restricted=True)
1204         self.assertTrue('test' in headers.keys())
1205         self.assertEqual(headers['test'], meta['test'])
1206
1207         #assert uploaded content
1208         status, h, data = self.client.request_object(self.container, name)
1209         self.assertEqual(len(o['data']), int(h['content-length']))
1210         self.assertEqual(o['data'], data)
1211
1212         #assert content-type
1213         self.assertEqual(h['content-type'], o['meta']['content_type'])
1214
1215     def _test_maximum_upload_size_exceeds(self):
1216         name = o_names[0]
1217         meta = {'test':'test1'}
1218         #upload 100MB
1219         length=1024*1024*100
1220         self.assert_raises_fault(400, self.upload_random_data, self.container,
1221                                  name, length, **meta)
1222
1223     def test_upload_with_name_containing_slash(self):
1224         name = '/%s' % o_names[0]
1225         meta = {'test':'test1'}
1226         o = self.upload_random_data(self.container, name, **meta)
1227
1228         self.assertEqual(o['data'],
1229                          self.client.retrieve_object(self.container, name))
1230
1231         self.assertTrue(name in self.client.list_objects(self.container))
1232
1233     def test_create_directory_marker(self):
1234         self.client.create_directory_marker(self.container, 'foo')
1235         meta = self.client.retrieve_object_metadata(self.container, 'foo')
1236         self.assertEqual(meta['content-length'], '0')
1237         self.assertEqual(meta['content-type'], 'application/directory')
1238
1239     def test_upload_unprocessable_entity(self):
1240         meta={'etag':'123', 'test':'test1'}
1241
1242         #assert unprocessable entity
1243         self.assert_raises_fault(422, self.upload_random_data, self.container,
1244                                  o_names[0], **meta)
1245
1246     def test_chunked_transfer(self):
1247         data = get_random_data()
1248         objname = 'object'
1249         self.client.create_object_using_chunks(self.container, objname,
1250                                                StringIO(data))
1251
1252         uploaded_data = self.client.retrieve_object(self.container, objname)
1253         self.assertEqual(data, uploaded_data)
1254
1255     def test_manifestation(self):
1256         prefix = 'myobject/'
1257         data = ''
1258         for i in range(5):
1259             part = '%s%d' %(prefix, i)
1260             o = self.upload_random_data(self.container, part)
1261             data += o['data']
1262
1263         manifest = '%s/%s' %(self.container, prefix)
1264         self.client.create_manifestation(self.container, 'large-object', manifest)
1265
1266         self.assert_object_exists(self.container, 'large-object')
1267         self.assertEqual(data, self.client.retrieve_object(self.container,
1268                                                            'large-object'))
1269
1270         r = self.client.retrieve_object_hashmap(self.container,'large-object')
1271         hashes = r['hashes']
1272         block_size = int(r['block_size'])
1273         block_hash = r['block_hash']
1274         l = len(data)
1275         block_num = l/block_size if l/block_size != 0 else l/block_size + 1
1276         self.assertEqual(block_num, len(hashes))
1277
1278         #wrong manifestation
1279         self.client.create_manifestation(self.container, 'large-object',
1280                                          '%s/invalid' % self.container)
1281         self.assertEqual('', self.client.retrieve_object(self.container,
1282                                                          'large-object'))
1283
1284     def test_create_zero_length_object(self):
1285         c = self.container
1286         o = 'object'
1287         zero = self.client.create_zero_length_object(c, o)
1288         zero_meta = self.client.retrieve_object_metadata(c, o)
1289         zero_hash = self.client.retrieve_object_hashmap(c, o)["hashes"]
1290         zero_data = self.client.retrieve_object(c, o)
1291
1292         self.assertEqual(int(zero_meta['content-length']), 0)
1293         hasher = newhasher('sha256')
1294         hasher.update("")
1295         emptyhash = hasher.digest()
1296         self.assertEqual(zero_hash, [hexlify(emptyhash)])
1297         self.assertEqual(zero_data, '')
1298
1299     def test_create_object_by_hashmap(self):
1300         c = self.container
1301         o = 'object'
1302         self.upload_random_data(c, o)
1303         hashmap = self.client.retrieve_object(c, o, format='json')
1304         o2 = 'object-copy'
1305         self.client.create_object_by_hashmap(c, o2, hashmap)
1306         self.assertEqual(self.client.retrieve_object(c, o),
1307                          self.client.retrieve_object(c, o))
1308
1309 class ObjectCopy(BaseTestCase):
1310     def setUp(self):
1311         BaseTestCase.setUp(self)
1312         self.containers = ['c1', 'c2']
1313         for c in self.containers:
1314             self.client.create_container(c)
1315         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1316
1317     def test_copy(self):
1318         with AssertMappingInvariant(self.client.retrieve_object_metadata,
1319                              self.containers[0], self.obj['name']):
1320             #perform copy
1321             meta = {'test':'testcopy'}
1322             status = self.client.copy_object(self.containers[0],
1323                                               self.obj['name'],
1324                                               self.containers[0],
1325                                               'testcopy',
1326                                               meta)[0]
1327
1328             #assert copy success
1329             self.assertEqual(status, 201)
1330
1331             #assert access the new object
1332             headers = self.client.retrieve_object_metadata(self.containers[0],
1333                                                            'testcopy')
1334             self.assertTrue('x-object-meta-test' in headers.keys())
1335             self.assertTrue(headers['x-object-meta-test'], 'testcopy')
1336
1337             #assert etag is the same
1338             self.assertEqual(headers['etag'], self.obj['hash'])
1339
1340             #assert src object still exists
1341             self.assert_object_exists(self.containers[0], self.obj['name'])
1342
1343     def test_copy_from_different_container(self):
1344         with AssertMappingInvariant(self.client.retrieve_object_metadata,
1345                              self.containers[0], self.obj['name']):
1346             meta = {'test':'testcopy'}
1347             status = self.client.copy_object(self.containers[0],
1348                                              self.obj['name'],
1349                                              self.containers[1],
1350                                              'testcopy',
1351                                              meta)[0]
1352             self.assertEqual(status, 201)
1353
1354             # assert updated metadata
1355             meta = self.client.retrieve_object_metadata(self.containers[1],
1356                                                            'testcopy',
1357                                                            restricted=True)
1358             self.assertTrue('test' in meta.keys())
1359             self.assertTrue(meta['test'], 'testcopy')
1360
1361             #assert src object still exists
1362             self.assert_object_exists(self.containers[0], self.obj['name'])
1363
1364     def test_copy_invalid(self):
1365         #copy from invalid object
1366         meta = {'test':'testcopy'}
1367         self.assert_raises_fault(404, self.client.copy_object, self.containers[0],
1368                                  'test.py', self.containers[1], 'testcopy', meta)
1369
1370         #copy from invalid container
1371         meta = {'test':'testcopy'}
1372         self.assert_raises_fault(404, self.client.copy_object, self.containers[1],
1373                                  self.obj['name'], self.containers[1],
1374                                  'testcopy', meta)
1375
1376 class ObjectMove(BaseTestCase):
1377     def setUp(self):
1378         BaseTestCase.setUp(self)
1379         self.containers = ['c1', 'c2']
1380         for c in self.containers:
1381             self.client.create_container(c)
1382         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1383
1384     def test_move(self):
1385         meta = self.client.retrieve_object_metadata(self.containers[0],
1386                                                     self.obj['name'])
1387         self.assertTrue('x-object-uuid' in meta)
1388         uuid = meta['x-object-uuid']
1389
1390         #perform move
1391         meta = {'test':'testcopy'}
1392         src_path = '/'.join(('/', self.containers[0], self.obj['name']))
1393         status = self.client.move_object(self.containers[0], self.obj['name'],
1394                                          self.containers[0], 'testcopy',
1395                                          meta)[0]
1396
1397         #assert successful move
1398         self.assertEqual(status, 201)
1399
1400         #assert updated metadata
1401         meta = self.client.retrieve_object_metadata(self.containers[0],
1402                                                     'testcopy')
1403         self.assertTrue('x-object-meta-test' in meta.keys())
1404         self.assertTrue(meta['x-object-meta-test'], 'testcopy')
1405
1406         #assert same uuid
1407         self.assertTrue(meta['x-object-uuid'], uuid)
1408
1409         #assert src object no more exists
1410         self.assert_object_not_exists(self.containers[0], self.obj['name'])
1411
1412 class ObjectPost(BaseTestCase):
1413     def setUp(self):
1414         BaseTestCase.setUp(self)
1415         self.containers = ['c1', 'c2']
1416         for c in self.containers:
1417             self.client.create_container(c)
1418         self.obj = []
1419         for i in range(2):
1420             self.obj.append(self.upload_random_data(self.containers[0], o_names[i]))
1421
1422     def test_update_meta(self):
1423         with AssertUUidInvariant(self.client.retrieve_object_metadata,
1424                                  self.containers[0],
1425                                  self.obj[0]['name']):
1426             #perform update metadata
1427             more = {'foo': 'foo', 'bar': 'bar', 'f' * 114: 'b' * 256}
1428             status = self.client.update_object_metadata(self.containers[0],
1429                                                         self.obj[0]['name'],
1430                                                         **more)[0]
1431             #assert request accepted
1432             self.assertEqual(status, 202)
1433
1434             #assert old metadata are still there
1435             headers = self.client.retrieve_object_metadata(self.containers[0],
1436                                                            self.obj[0]['name'],
1437                                                            restricted=True)
1438             #assert new metadata have been updated
1439             for k,v in more.items():
1440                 self.assertTrue(k in headers.keys())
1441                 self.assertTrue(headers[k], v)
1442
1443             #out of limits
1444             more = {'f' * 114: 'b' * 257}
1445             self.assert_raises_fault(400, self.client.update_object_metadata,
1446                                                         self.containers[0],
1447                                                         self.obj[0]['name'],
1448                                                         **more)
1449
1450             #perform update metadata
1451             more = {'α': 'β' * 256}
1452             status = self.client.update_object_metadata(self.containers[0],
1453                                                         self.obj[0]['name'],
1454                                                         **more)[0]
1455             #assert request accepted
1456             self.assertEqual(status, 202)
1457
1458             #assert old metadata are still there
1459             headers = self.client.retrieve_object_metadata(self.containers[0],
1460                                                            self.obj[0]['name'],
1461                                                            restricted=True)
1462             #assert new metadata have been updated
1463             for k,v in more.items():
1464                 self.assertTrue(k in headers.keys())
1465                 self.assertTrue(headers[k], v)
1466
1467             #out of limits
1468             more = {'α': 'β' * 257}
1469             self.assert_raises_fault(400, self.client.update_object_metadata,
1470                                                         self.containers[0],
1471                                                         self.obj[0]['name'],
1472                                                         **more)
1473
1474     def test_update_object(self,
1475                            first_byte_pos=0,
1476                            last_byte_pos=499,
1477                            instance_length = True,
1478                            content_length = 500):
1479         with AssertUUidInvariant(self.client.retrieve_object_metadata,
1480                                  self.containers[0],
1481                                  self.obj[0]['name']):
1482             l = len(self.obj[0]['data'])
1483             range = 'bytes %d-%d/%s' %(first_byte_pos,
1484                                            last_byte_pos,
1485                                             l if instance_length else '*')
1486             partial = last_byte_pos - first_byte_pos + 1
1487             length = first_byte_pos + partial
1488             data = get_random_data(partial)
1489             args = {'content_type':'application/octet-stream',
1490                     'content_range':'%s' %range}
1491             if content_length:
1492                 args['content_length'] = content_length
1493
1494             r = self.client.update_object(self.containers[0], self.obj[0]['name'],
1495                                       StringIO(data), **args)
1496             status = r[0]
1497             etag = r[1]['etag']
1498             if partial < 0 or (instance_length and l <= last_byte_pos):
1499                 self.assertEqual(status, 202)
1500             else:
1501                 self.assertEqual(status, 204)
1502                 #check modified object
1503                 content = self.client.retrieve_object(self.containers[0],
1504                                                   self.obj[0]['name'])
1505                 self.assertEqual(content[:first_byte_pos], self.obj[0]['data'][:first_byte_pos])
1506                 self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1507                 self.assertEqual(content[last_byte_pos+1:], self.obj[0]['data'][last_byte_pos+1:])
1508                 self.assertEqual(etag, compute_md5_hash(content))
1509
1510     def test_update_object_lt_blocksize(self):
1511         self.test_update_object(10, 20, content_length=None)
1512
1513     def test_update_object_gt_blocksize(self):
1514         o = self.upload_random_data(self.containers[0], o_names[1],
1515                                 length=4*1024*1024+5)
1516         c = self.containers[0]
1517         o_name = o['name']
1518         o_data = o['data']
1519         first_byte_pos = 4*1024*1024+1
1520         last_byte_pos = 4*1024*1024+4
1521         l = last_byte_pos - first_byte_pos + 1
1522         data = get_random_data(l)
1523         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1524         self.client.update_object(c, o_name, StringIO(data), content_range=range)
1525         content = self.client.retrieve_object(c, o_name)
1526         self.assertEqual(content[:first_byte_pos], o_data[:first_byte_pos])
1527         self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1528         self.assertEqual(content[last_byte_pos+1:], o_data[last_byte_pos+1:])
1529
1530     def test_update_object_divided_by_blocksize(self):
1531         o = self.upload_random_data(self.containers[0], o_names[1],
1532                                 length=4*1024*1024+5)
1533         c = self.containers[0]
1534         o_name = o['name']
1535         o_data = o['data']
1536         first_byte_pos = 4*1024*1024
1537         last_byte_pos = 5*1024*1024
1538         l = last_byte_pos - first_byte_pos + 1
1539         data = get_random_data(l)
1540         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1541         self.client.update_object(c, o_name, StringIO(data), content_range=range)
1542         content = self.client.retrieve_object(c, o_name)
1543         self.assertEqual(content[:first_byte_pos], o_data[:first_byte_pos])
1544         self.assertEqual(content[first_byte_pos:last_byte_pos+1], data)
1545         self.assertEqual(content[last_byte_pos+1:], o_data[last_byte_pos+1:])
1546
1547     def test_update_object_no_content_length(self):
1548         self.test_update_object(content_length = None)
1549
1550     def test_update_object_invalid_content_length(self):
1551         with AssertContentInvariant(self.client.retrieve_object,
1552                                     self.containers[0], self.obj[0]['name']):
1553             self.assert_raises_fault(400, self.test_update_object,
1554                                      content_length = 1000)
1555
1556     def _test_update_object_invalid_range(self):
1557         with AssertContentInvariant(self.client.retrieve_object,
1558                                     self.containers[0], self.obj[0]['name']):
1559             self.assert_raises_fault(416, self.test_update_object, 499, 0, True)
1560
1561     def _test_update_object_invalid_range_and_length(self):
1562         with AssertContentInvariant(self.client.retrieve_object,
1563                                     self.containers[0], self.obj[0]['name']):
1564             self.assert_raises_fault([400, 416], self.test_update_object, 499, 0, True,
1565                                      -1)
1566
1567     def test_update_object_invalid_range_with_no_content_length(self):
1568         with AssertContentInvariant(self.client.retrieve_object,
1569                                     self.containers[0], self.obj[0]['name']):
1570             self.assert_raises_fault(416, self.test_update_object, 499, 0, True,
1571                                      content_length = None)
1572
1573     def test_update_object_out_of_limits(self):
1574         with AssertContentInvariant(self.client.retrieve_object,
1575                                     self.containers[0], self.obj[0]['name']):
1576             l = len(self.obj[0]['data'])
1577             self.assert_raises_fault(416, self.test_update_object, 0, l+1, True)
1578
1579     def test_append(self):
1580         data = get_random_data(500)
1581         headers = {}
1582         self.client.update_object(self.containers[0], self.obj[0]['name'],
1583                                   StringIO(data), content_length=500,
1584                                   content_type='application/octet-stream')
1585
1586         content = self.client.retrieve_object(self.containers[0],
1587                                               self.obj[0]['name'])
1588         self.assertEqual(len(content), len(self.obj[0]['data']) + 500)
1589         self.assertEqual(content[:-500], self.obj[0]['data'])
1590
1591     def test_update_with_chunked_transfer(self):
1592         data = get_random_data(500)
1593         dl = len(data)
1594         fl = len(self.obj[0]['data'])
1595
1596         self.client.update_object_using_chunks(self.containers[0],
1597                                                self.obj[0]['name'],
1598                                                StringIO(data),
1599                                                offset=0,
1600                                                content_type='application/octet-stream')
1601
1602         #check modified object
1603         content = self.client.retrieve_object(self.containers[0],
1604                                               self.obj[0]['name'])
1605         self.assertEqual(content[0:dl], data)
1606         self.assertEqual(content[dl:fl], self.obj[0]['data'][dl:fl])
1607
1608     def test_update_from_other_object(self):
1609         c = self.containers[0]
1610         src = o_names[0]
1611         dest = 'object'
1612
1613         source_data = self.client.retrieve_object(c, src)
1614         source_meta = self.client.retrieve_object_metadata(c, src)
1615         source_hash = self.client.retrieve_object_hashmap(c, src)["hashes"]
1616
1617         #update zero length object
1618         self.client.create_zero_length_object(c, dest)
1619         source_object = '/%s/%s' % (c, src)
1620         self.client.update_from_other_source(c, dest, source_object)
1621         dest_data = self.client.retrieve_object(c, src)
1622         dest_meta = self.client.retrieve_object_metadata(c, dest)
1623         dest_hash = self.client.retrieve_object_hashmap(c, src)["hashes"]
1624         self.assertEqual(source_data, dest_data)
1625         self.assertEqual(source_hash, dest_hash)
1626
1627         #test append
1628         self.client.update_from_other_source(c, dest, source_object)
1629         content = self.client.retrieve_object(c, dest)
1630         self.assertEqual(source_data * 2, content)
1631
1632     def test_update_range_from_other_object(self):
1633         c = self.containers[0]
1634         dest = 'object'
1635
1636         #test update range
1637         src = self.obj[1]['name']
1638         src_data = self.client.retrieve_object(c, src)
1639
1640         #update zero length object
1641         prev_data = self.upload_random_data(c, dest, length=4*1024*1024+10)['data']
1642         source_object = '/%s/%s' % (c, src)
1643         first_byte_pos = 4*1024*1024+1
1644         last_byte_pos = 4*1024*1024+4
1645         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1646         self.client.update_from_other_source(c, dest, source_object,
1647                                              content_range=range)
1648         content = self.client.retrieve_object(c, dest)
1649         self.assertEqual(content[:first_byte_pos], prev_data[:first_byte_pos])
1650         self.assertEqual(content[first_byte_pos:last_byte_pos+1], src_data[:last_byte_pos - first_byte_pos + 1])
1651         self.assertEqual(content[last_byte_pos+1:], prev_data[last_byte_pos+1:])
1652
1653     def test_update_hashes_from_other_object(self):
1654         c = self.containers[0]
1655         dest = 'object'
1656
1657         #test update range
1658         src_data = self.upload_random_data(c, o_names[0], length=1024*1024+10)['data']
1659
1660         #update zero length object
1661         prev_data = self.upload_random_data(c, dest, length=5*1024*1024+10)['data']
1662         source_object = '/%s/%s' % (c, o_names[0])
1663         first_byte_pos = 4*1024*1024
1664         last_byte_pos = 5*1024*1024
1665         range = 'bytes %d-%d/*' %(first_byte_pos, last_byte_pos)
1666         self.client.update_from_other_source(c, dest, source_object,
1667                                              content_range=range)
1668         content = self.client.retrieve_object(c, dest)
1669         self.assertEqual(content[:first_byte_pos], prev_data[:first_byte_pos])
1670         self.assertEqual(content[first_byte_pos:last_byte_pos+1], src_data[:last_byte_pos - first_byte_pos + 1])
1671         self.assertEqual(content[last_byte_pos+1:], prev_data[last_byte_pos+1:])
1672
1673
1674     def test_update_zero_length_object(self):
1675         c = self.containers[0]
1676         o = 'object'
1677         other = 'other'
1678         zero = self.client.create_zero_length_object(c, o)
1679
1680         data = get_random_data()
1681         self.client.update_object(c, o, StringIO(data))
1682         self.client.create_object(c, other, StringIO(data))
1683
1684         self.assertEqual(self.client.retrieve_object(c, o),
1685                          self.client.retrieve_object(c, other))
1686
1687         self.assertEqual(self.client.retrieve_object_hashmap(c, o)["hashes"],
1688                          self.client.retrieve_object_hashmap(c, other)["hashes"])
1689
1690 class ObjectDelete(BaseTestCase):
1691     def setUp(self):
1692         BaseTestCase.setUp(self)
1693         self.containers = ['c1', 'c2']
1694         for c in self.containers:
1695             self.client.create_container(c)
1696         self.obj = self.upload_random_data(self.containers[0], o_names[0])
1697
1698     def test_delete(self):
1699         #perform delete object
1700         self.client.delete_object(self.containers[0], self.obj['name'])[0]
1701
1702     def test_delete_invalid(self):
1703         #assert item not found
1704         self.assert_raises_fault(404, self.client.delete_object, self.containers[1],
1705                                  self.obj['name'])
1706
1707 class ListSharing(BaseTestCase):
1708     def setUp(self):
1709         BaseTestCase.setUp(self)
1710         for i in range(2):
1711             self.client.create_container('c%s' %i)
1712         self.client.create_container('c')
1713         for i in range(2):
1714             self.upload_random_data('c1', 'o%s' %i)
1715         accounts = OTHER_ACCOUNTS.copy()
1716         self.o1_sharing_with = accounts.popitem()
1717         self.o1_sharing = [self.o1_sharing_with[1]]
1718         self.client.share_object('c1', 'o1', self.o1_sharing, read=True)
1719
1720         l = []
1721         for i in range(2):
1722             l.append(accounts.popitem())
1723
1724     def test_list_other_shared(self):
1725         self.other = Pithos_Client(get_url(),
1726                               self.o1_sharing_with[0],
1727                               self.o1_sharing_with[1])
1728         self.assertTrue(get_user() in self.other.list_shared_by_others())
1729
1730     def test_list_my_shared(self):
1731         my_shared_containers = self.client.list_containers(shared=True)
1732         self.assertTrue('c1' in my_shared_containers)
1733         self.assertTrue('c2' not in my_shared_containers)
1734
1735         my_shared_objects = self.client.list_objects('c1', shared=True)
1736         self.assertTrue('o1' in my_shared_objects)
1737         self.assertTrue('o2' not in my_shared_objects)
1738
1739 class TestGreek(BaseTestCase):
1740     def test_create_container(self):
1741         self.client.create_container('φάκελος')
1742         self.assert_container_exists('φάκελος')
1743
1744         self.assertTrue('φάκελος' in self.client.list_containers())
1745
1746     def test_create_object(self):
1747         self.client.create_container('φάκελος')
1748         self.upload_random_data('φάκελος', 'αντικείμενο')
1749
1750         self.assert_object_exists('φάκελος', 'αντικείμενο')
1751         self.assertTrue('αντικείμενο' in self.client.list_objects('φάκελος'))
1752
1753     def test_copy_object(self):
1754         src_container = 'φάκελος'
1755         src_object = 'αντικείμενο'
1756         dest_container = 'αντίγραφα'
1757         dest_object = 'ασφαλές-αντίγραφο'
1758
1759         self.client.create_container(src_container)
1760         self.upload_random_data(src_container, src_object)
1761
1762         self.client.create_container(dest_container)
1763         self.client.copy_object(src_container, src_object, dest_container,
1764                                 dest_object)
1765
1766         self.assert_object_exists(src_container, src_object)
1767         self.assert_object_exists(dest_container, dest_object)
1768         self.assertTrue(dest_object in self.client.list_objects(dest_container))
1769
1770     def test_move_object(self):
1771         src_container = 'φάκελος'
1772         src_object = 'αντικείμενο'
1773         dest_container = 'αντίγραφα'
1774         dest_object = 'ασφαλές-αντίγραφο'
1775
1776         self.client.create_container(src_container)
1777         self.upload_random_data(src_container, src_object)
1778
1779         self.client.create_container(dest_container)
1780         self.client.move_object(src_container, src_object, dest_container,
1781                                 dest_object)
1782
1783         self.assert_object_not_exists(src_container, src_object)
1784         self.assert_object_exists(dest_container, dest_object)
1785         self.assertTrue(dest_object in self.client.list_objects(dest_container))
1786
1787     def test_delete_object(self):
1788         self.client.create_container('φάκελος')
1789         self.upload_random_data('φάκελος', 'αντικείμενο')
1790         self.assert_object_exists('φάκελος', 'αντικείμενο')
1791
1792         self.client.delete_object('φάκελος', 'αντικείμενο')
1793         self.assert_object_not_exists('φάκελος', 'αντικείμενο')
1794         self.assertTrue('αντικείμενο' not in self.client.list_objects('φάκελος'))
1795
1796     def test_delete_container(self):
1797         self.client.create_container('φάκελος')
1798         self.assert_container_exists('φάκελος')
1799
1800         self.client.delete_container('φάκελος')
1801         self.assert_container_not_exists('φάκελος')
1802         self.assertTrue('φάκελος' not in self.client.list_containers())
1803
1804     def test_account_meta(self):
1805         meta = {'ποιότητα':'ΑΑΑ'}
1806         self.client.update_account_metadata(**meta)
1807         meta = self.client.retrieve_account_metadata(restricted=True)
1808         self.assertTrue('ποιότητα' in meta.keys())
1809         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1810
1811     def test_container_meta(self):
1812         meta = {'ποιότητα':'ΑΑΑ'}
1813         self.client.create_container('φάκελος', meta=meta)
1814
1815         meta = self.client.retrieve_container_metadata('φάκελος', restricted=True)
1816         self.assertTrue('ποιότητα' in meta.keys())
1817         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1818
1819     def test_object_meta(self):
1820         self.client.create_container('φάκελος')
1821         meta = {'ποιότητα':'ΑΑΑ'}
1822         self.upload_random_data('φάκελος', 'αντικείμενο', **meta)
1823
1824         meta = self.client.retrieve_object_metadata('φάκελος', 'αντικείμενο',
1825                                                     restricted=True)
1826         self.assertTrue('ποιότητα' in meta.keys())
1827         self.assertEqual(meta['ποιότητα'], 'ΑΑΑ')
1828
1829     def test_list_meta_filtering(self):
1830         self.client.create_container('φάκελος')
1831         meta = {'ποιότητα':'ΑΑΑ'}
1832         self.upload_random_data('φάκελος', 'ο1', **meta)
1833         self.upload_random_data('φάκελος', 'ο2')
1834         self.upload_random_data('φάκελος', 'ο3')
1835
1836         meta = {'ποσότητα':'μεγάλη'}
1837         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
1838         objects = self.client.list_objects('φάκελος', meta='ποιότητα, ποσότητα')
1839         self.assertEquals(objects, ['ο1', 'ο2'])
1840
1841         objects = self.client.list_objects('φάκελος', meta='!ποιότητα')
1842         self.assertEquals(objects, ['ο2', 'ο3'])
1843
1844         objects = self.client.list_objects('φάκελος', meta='!ποιότητα, !ποσότητα')
1845         self.assertEquals(objects, ['ο3'])
1846
1847         meta = {'ποιότητα':'ΑΒ'}
1848         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
1849         objects = self.client.list_objects('φάκελος', meta='ποιότητα=ΑΑΑ')
1850         self.assertEquals(objects, ['ο1'])
1851         objects = self.client.list_objects('φάκελος', meta='ποιότητα!=ΑΑΑ')
1852         self.assertEquals(objects, ['ο2'])
1853
1854         meta = {'έτος':'2011'}
1855         self.client.update_object_metadata('φάκελος', 'ο3', **meta)
1856         meta = {'έτος':'2012'}
1857         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
1858         objects = self.client.list_objects('φάκελος', meta='έτος<2012')
1859         self.assertEquals(objects, ['ο3'])
1860         objects = self.client.list_objects('φάκελος', meta='έτος<=2012')
1861         self.assertEquals(objects, ['ο2', 'ο3'])
1862         objects = self.client.list_objects('φάκελος', meta='έτος<2012,έτος!=2011')
1863         self.assertEquals(objects, '')
1864
1865     def test_groups(self):
1866         #create a group
1867         groups = {'σεφς':'chazapis,διογένης'}
1868         self.client.set_account_groups(**groups)
1869         groups.update(self.initial_groups)
1870         self.assertEqual(groups['σεφς'],
1871                          self.client.retrieve_account_groups()['σεφς'])
1872
1873         #check read access
1874         self.client.create_container('φάκελος')
1875         o = self.upload_random_data('φάκελος', 'ο1')
1876         self.client.share_object('φάκελος', 'ο1', ['%s:σεφς' % get_user()])
1877         chef = Pithos_Client(get_url(),
1878                             '0009',
1879                             'διογένης')
1880         self.assert_not_raises_fault(403, chef.retrieve_object_metadata,
1881                                      'φάκελος', 'ο1', account=get_user())
1882
1883         #check write access
1884         self.client.share_object('φάκελος', 'ο1', ['διογένης'], read=False)
1885         new_data = get_random_data()
1886         self.assert_not_raises_fault(403, chef.update_object,
1887                                      'φάκελος', 'ο1', StringIO(new_data),
1888                                      account=get_user())
1889
1890         server_data = self.client.retrieve_object('φάκελος', 'ο1')
1891         self.assertEqual(server_data[:len(o['data'])], o['data'])
1892         self.assertEqual(server_data[len(o['data']):], new_data)
1893
1894     def test_manifestation(self):
1895         self.client.create_container('κουβάς')
1896         prefix = 'μέρη/'
1897         data = ''
1898         for i in range(5):
1899             part = '%s%d' %(prefix, i)
1900             o = self.upload_random_data('κουβάς', part)
1901             data += o['data']
1902
1903         self.client.create_container('φάκελος')
1904         manifest = '%s/%s' %('κουβάς', prefix)
1905         self.client.create_manifestation('φάκελος', 'άπαντα', manifest)
1906
1907         self.assert_object_exists('φάκελος', 'άπαντα')
1908         self.assertEqual(data, self.client.retrieve_object('φάκελος',
1909                                                            'άπαντα'))
1910
1911         #wrong manifestation
1912         self.client.create_manifestation('φάκελος', 'άπαντα', 'κουβάς/άκυρο')
1913         self.assertEqual('', self.client.retrieve_object('φάκελος', 'άπαντα'))
1914
1915     def test_update_from_another_object(self):
1916         self.client.create_container('κουβάς')
1917         src_data = self.upload_random_data('κουβάς', 'πηγή')['data']
1918         initial_data = self.upload_random_data('κουβάς', 'νέο')['data']
1919         source_object = '/%s/%s' % ('κουβάς', 'πηγή')
1920         self.client.update_from_other_source('κουβάς', 'νέο', source_object)
1921
1922         self.assertEqual(
1923             self.client.retrieve_object('κουβάς', 'νέο'),
1924             '%s%s' % (initial_data, self.client.retrieve_object('κουβάς', 'πηγή')))
1925
1926 class TestPermissions(BaseTestCase):
1927     def setUp(self):
1928         BaseTestCase.setUp(self)
1929
1930         #create a group
1931         self.authorized = ['chazapis', 'verigak', 'gtsouk']
1932         groups = {'pithosdev':','.join(self.authorized)}
1933         self.client.set_account_groups(**groups)
1934
1935         self.container = 'c'
1936         self.object = 'o'
1937         self.client.create_container(self.container)
1938         self.upload_random_data(self.container, self.object)
1939         self.upload_random_data(self.container, self.object+'/')
1940         self.upload_random_data(self.container, self.object+'/a')
1941         self.upload_random_data(self.container, self.object+'a')
1942         self.upload_random_data(self.container, self.object+'a/')
1943         self.dir_content_types = ('application/directory', 'application/folder')
1944
1945     def assert_read(self, authorized=[], any=False, depth=0):
1946         for token, account in OTHER_ACCOUNTS.items():
1947             cl = Pithos_Client(get_url(), token, account)
1948             if account in authorized or any:
1949                 self.assert_not_raises_fault(403, cl.retrieve_object_metadata,
1950                                              self.container, self.object,
1951                                              account=get_user())
1952             else:
1953                 self.assert_raises_fault(403, cl.retrieve_object_metadata,
1954                                          self.container, self.object,
1955                                          account=get_user())
1956
1957         #check inheritance
1958         meta = self.client.retrieve_object_metadata(self.container, self.object)
1959         type = meta['content-type']
1960         derivatives = self.client.list_objects(self.container, prefix=self.object)
1961         #exclude the self.object
1962         del derivatives[derivatives.index(self.object)]
1963         for o in derivatives:
1964             for token, account in OTHER_ACCOUNTS.items():
1965                 cl = Pithos_Client(get_url(), token, account)
1966                 prefix = self.object if self.object.endswith('/') else self.object+'/'
1967                 if (account in authorized or any) and \
1968                 (type in self.dir_content_types) and \
1969                 o.startswith(prefix):
1970                     self.assert_not_raises_fault(403, cl.retrieve_object_metadata,
1971                                              self.container, o, account=get_user())
1972                 else:
1973                     self.assert_raises_fault(403, cl.retrieve_object_metadata,
1974                                          self.container, o, account=get_user())
1975
1976     def assert_write(self, authorized=[], any=False):
1977         o_data = self.client.retrieve_object(self.container, self.object)
1978         for token, account in OTHER_ACCOUNTS.items():
1979             cl = Pithos_Client(get_url(), token, account)
1980             new_data = get_random_data()
1981             if account in authorized or any:
1982                 # test write access
1983                 self.assert_not_raises_fault(403, cl.update_object,
1984                                              self.container, self.object, StringIO(new_data),
1985                                              account=get_user())
1986                 try:
1987                     # test read access
1988                     server_data = cl.retrieve_object(self.container, self.object, account=get_user())
1989                     self.assertEqual(o_data, server_data[:len(o_data)])
1990                     self.assertEqual(new_data, server_data[len(o_data):])
1991                     o_data = server_data
1992                 except Fault, f:
1993                     self.failIf(f.status == 403)
1994             else:
1995                 self.assert_raises_fault(403, cl.update_object,
1996                                              self.container, self.object, StringIO(new_data),
1997                                              account=get_user())
1998         #check inheritance
1999         meta = self.client.retrieve_object_metadata(self.container, self.object)
2000         type = meta['content-type']
2001         derivatives = self.client.list_objects(self.container, prefix=self.object)
2002         #exclude the object
2003         del derivatives[derivatives.index(self.object)]
2004         for o in derivatives:
2005             for token, account in OTHER_ACCOUNTS.items():
2006                 prefix = self.object if self.object.endswith('/') else self.object+'/'
2007                 cl = Pithos_Client(get_url(), token, account)
2008                 new_data = get_random_data()
2009                 if (account in authorized or any) and \
2010                 (type in self.dir_content_types) and \
2011                 o.startswith(prefix):
2012                     # test write access
2013                     self.assert_not_raises_fault(403, cl.update_object,
2014                                                  self.container, o,
2015                                                  StringIO(new_data),
2016                                                  account=get_user())
2017                     try:
2018                         server_data = cl.retrieve_object(self.container, o, account=get_user())
2019                         self.assertEqual(new_data, server_data[-len(new_data):])
2020                     except Fault, f:
2021                         self.failIf(f.status == 403)
2022                 else:
2023                     self.assert_raises_fault(403, cl.update_object,
2024                                                  self.container, o,
2025                                                  StringIO(new_data),
2026                                                  account=get_user())
2027
2028     def test_group_read(self):
2029         self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()])
2030         self.assert_read(authorized=self.authorized)
2031
2032     def test_read_many(self):
2033         self.client.share_object(self.container, self.object, self.authorized)
2034         self.assert_read(authorized=self.authorized)
2035
2036     def test_read_by_everyone(self):
2037         self.client.share_object(self.container, self.object, ['*'])
2038         self.assert_read(any=True)
2039
2040     def test_read_directory(self):
2041         for type in self.dir_content_types:
2042             #change content type
2043             self.client.move_object(self.container, self.object, self.container, self.object, content_type=type)
2044             self.client.share_object(self.container, self.object, ['*'])
2045             self.assert_read(any=True)
2046             self.client.share_object(self.container, self.object, self.authorized)
2047             self.assert_read(authorized=self.authorized)
2048             self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()])
2049             self.assert_read(authorized=self.authorized)
2050
2051     def test_group_write(self):
2052         self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()], read=False)
2053         self.assert_write(authorized=self.authorized)
2054
2055     def test_write_many(self):
2056         self.client.share_object(self.container, self.object, self.authorized, read=False)
2057         self.assert_write(authorized=self.authorized)
2058
2059     def test_write_by_everyone(self):
2060         self.client.share_object(self.container, self.object, ['*'], read=False)
2061         self.assert_write(any=True)
2062
2063     def test_write_directory(self):
2064         dir_content_types = ('application/directory', 'application/foler')
2065         for type in dir_content_types:
2066             #change content type
2067             self.client.move_object(self.container, self.object, self.container, self.object, content_type='application/folder')
2068             self.client.share_object(self.container, self.object, ['*'], read=False)
2069             self.assert_write(any=True)
2070             self.client.share_object(self.container, self.object, self.authorized, read=False)
2071             self.assert_write(authorized=self.authorized)
2072             self.client.share_object(self.container, self.object, ['%s:pithosdev' % get_user()], read=False)
2073             self.assert_write(authorized=self.authorized)
2074
2075     def test_shared_listing(self):
2076         self.client.share_object(self.container, self.object, self.authorized)
2077
2078         my_shared_containers = self.client.list_containers(shared=True)
2079         self.assertEqual(['c'], my_shared_containers)
2080         my_shared_objects = self.client.list_objects('c', shared=True)
2081         self.assertEqual(['o'], my_shared_objects)
2082
2083         dir_content_types = ('application/directory', 'application/foler')
2084         for type in dir_content_types:
2085             #change content type
2086             self.client.move_object(self.container, self.object, self.container, self.object, content_type='application/folder')
2087             my_shared_objects = self.client.list_objects('c', shared=True)
2088             self.assertEqual(['o', 'o/', 'o/a'], my_shared_objects)
2089
2090         for token, account in OTHER_ACCOUNTS.items():
2091             if account in self.authorized:
2092                 self.other = Pithos_Client(get_url(), token, account)
2093                 self.assertTrue(get_user() in self.other.list_shared_by_others())
2094
2095 class TestPublish(BaseTestCase):
2096     def test_publish(self):
2097         self.client.create_container('c')
2098         o_data = self.upload_random_data('c', 'o')['data']
2099         self.client.publish_object('c', 'o')
2100         meta = self.client.retrieve_object_metadata('c', 'o')
2101         self.assertTrue('x-object-public' in meta)
2102         url = meta['x-object-public']
2103
2104         p = urlparse(get_url())
2105         if p.scheme == 'http':
2106             conn = HTTPConnection(p.netloc)
2107         elif p.scheme == 'https':
2108             conn = HTTPSConnection(p.netloc)
2109         else:
2110             raise Exception('Unknown URL scheme')
2111
2112         conn.request('GET', url)
2113         resp = conn.getresponse()
2114         length = resp.getheader('content-length', None)
2115         data = resp.read(length)
2116         self.assertEqual(o_data, data)
2117
2118 class TestPolicies(BaseTestCase):
2119     def test_none_versioning(self):
2120         self.client.create_container('c', policies={'versioning':'none'})
2121         o = self.upload_random_data('c', 'o')
2122         meta = self.client.retrieve_object_metadata('c', 'o')
2123         v = meta['x-object-version']
2124         more_data = get_random_data()
2125         self.client.update_object('c', 'o', StringIO(more_data))
2126         vlist = self.client.retrieve_object_versionlist('c', 'o')
2127         self.assert_raises_fault(404, self.client.retrieve_object_version,
2128                                  'c', 'o', v)
2129         data = self.client.retrieve_object('c', 'o')
2130         end = len(o['data'])
2131         self.assertEqual(data[:end], o['data'])
2132         self.assertEqual(data[end:], more_data)
2133
2134     def test_quota(self):
2135         self.client.create_container('c', policies={'quota':'1'})
2136         meta = self.client.retrieve_container_metadata('c')
2137         self.assertEqual(meta['x-container-policy-quota'], '1')
2138         self.assert_raises_fault(413, self.upload_random_data, 'c', 'o',
2139                                  length=1024*1024+1)
2140
2141     def test_quota_none(self):
2142         self.client.create_container('c', policies={'quota':'0'})
2143         meta = self.client.retrieve_container_metadata('c')
2144         self.assertEqual(meta['x-container-policy-quota'], '0')
2145         self.assert_not_raises_fault(413, self.upload_random_data, 'c', 'o',
2146                                  length=1024*1024+1)
2147
2148 class AssertUUidInvariant(object):
2149     def __init__(self, callable, *args, **kwargs):
2150         self.callable = callable
2151         self.args = args
2152         self.kwargs = kwargs
2153
2154     def __enter__(self):
2155         self.map = self.callable(*self.args, **self.kwargs)
2156         assert('x-object-uuid' in self.map)
2157         self.uuid = self.map['x-object-uuid']
2158         return self.map
2159
2160     def __exit__(self, type, value, tb):
2161         map = self.callable(*self.args, **self.kwargs)
2162         assert('x-object-uuid' in self.map)
2163         uuid = map['x-object-uuid']
2164         assert(uuid == self.uuid)
2165
2166 class AssertMappingInvariant(object):
2167     def __init__(self, callable, *args, **kwargs):
2168         self.callable = callable
2169         self.args = args
2170         self.kwargs = kwargs
2171
2172     def __enter__(self):
2173         self.map = self.callable(*self.args, **self.kwargs)
2174         return self.map
2175
2176     def __exit__(self, type, value, tb):
2177         map = self.callable(*self.args, **self.kwargs)
2178         for k, v in self.map.items():
2179             if is_date(v):
2180                 continue
2181             #print '#', k, v, map[k]
2182             assert(k in map)
2183             assert v == map[k]
2184
2185 class AssertContentInvariant(object):
2186     def __init__(self, callable, *args, **kwargs):
2187         self.callable = callable
2188         self.args = args
2189         self.kwargs = kwargs
2190
2191     def __enter__(self):
2192         self.content = self.callable(*self.args, **self.kwargs)[2]
2193         return self.content
2194
2195     def __exit__(self, type, value, tb):
2196         content = self.callable(*self.args, **self.kwargs)[2]
2197         assert self.content == content
2198
2199 def get_content_splitted(response):
2200     if response:
2201         return response.content.split('\n')
2202
2203 def compute_md5_hash(data):
2204     md5 = hashlib.md5()
2205     offset = 0
2206     md5.update(data)
2207     return md5.hexdigest().lower()
2208
2209 def compute_block_hash(data, algorithm):
2210     h = hashlib.new(algorithm)
2211     h.update(data.rstrip('\x00'))
2212     return h.hexdigest()
2213
2214 def get_random_data(length=500):
2215     char_set = string.ascii_uppercase + string.digits
2216     return ''.join(random.choice(char_set) for x in xrange(length))
2217
2218 def is_date(date):
2219     MONTHS = 'jan feb mar apr may jun jul aug sep oct nov dec'.split()
2220     __D = r'(?P<day>\d{2})'
2221     __D2 = r'(?P<day>[ \d]\d)'
2222     __M = r'(?P<mon>\w{3})'
2223     __Y = r'(?P<year>\d{4})'
2224     __Y2 = r'(?P<year>\d{2})'
2225     __T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
2226     RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
2227     RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
2228     ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
2229     for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
2230         m = regex.match(date)
2231         if m is not None:
2232             return True
2233     return False
2234
2235 o_names = ['kate.jpg',
2236            'kate_beckinsale.jpg',
2237            'How To Win Friends And Influence People.pdf',
2238            'moms_birthday.jpg',
2239            'poodle_strut.mov',
2240            'Disturbed - Down With The Sickness.mp3',
2241            'army_of_darkness.avi',
2242            'the_mad.avi',
2243            'photos/animals/dogs/poodle.jpg',
2244            'photos/animals/dogs/terrier.jpg',
2245            'photos/animals/cats/persian.jpg',
2246            'photos/animals/cats/siamese.jpg',
2247            'photos/plants/fern.jpg',
2248            'photos/plants/rose.jpg',
2249            'photos/me.jpg']
2250
2251
2252 def main():
2253     if get_user() == 'test':
2254         unittest.main(module='pithos.tools.test')
2255     else:
2256         print 'Will not run tests as any other user except \'test\' (current user: %s).' % get_user()
2257
2258
2259 if __name__ == "__main__":
2260     main()
2261