Revision 4a1c29ea

b/README.upgrade
1
Upgrade notes.
1
UPGRADE
2
=======
3

  
4
0.7.9 -> 0.7.10
5
---------------
6
* Update settings.py (BACKEND*, SERVICE_NAME, *_EMAIL, *_TARGET)
7
* Update 'attributes' table in mysql:
8
    
9
	mysql> update attributes set `key`='ETag' where `key`='hash';
10

  
b/docs/source/devguide.rst
25 25
=========================  ================================
26 26
Revision                   Description
27 27
=========================  ================================
28
0.7 (Oct 21, 2011)         Suggest upload/download methods using hashmaps.
28
0.7 (Nov 21, 2011)         Suggest upload/download methods using hashmaps.
29 29
\                          Propose syncing algorithm.
30 30
\                          Support cross-account object copy and move.
31 31
\                          Pass token as a request parameter when using ``POST`` via an HTML form.
......
33 33
\                          Use container ``POST`` to upload missing blocks of data.
34 34
\                          Report policy in account headers.
35 35
\                          Add insufficient quota reply.
36
\                          Use special meta to always report Merkle hash.
36 37
0.6 (Sept 13, 2011)        Reply with Merkle hash as the ETag when updating objects.
37 38
\                          Include version id in object replace/change replies.
38 39
\                          Change conflict (409) replies format to text.
......
437 438
content_encoding            The encoding of the object (optional)
438 439
content-disposition         The presentation style of the object (optional)
439 440
last_modified               The last object modification date (regardless of version)
441
x_object_hash               The Merkle hash
440 442
x_object_version            The object's version identifier
441 443
x_object_version_timestamp  The object's version timestamp
442 444
x_object_modified_by        The user that committed the object's version
......
600 602
Last-Modified               The last object modification date (regardless of version)
601 603
Content-Encoding            The encoding of the object (optional)
602 604
Content-Disposition         The presentation style of the object (optional)
605
X-Object-Hash               The Merkle hash
603 606
X-Object-Version            The object's version identifier
604 607
X-Object-Version-Timestamp  The object's version timestamp
605 608
X-Object-Modified-By        The user that comitted the object's version
......
695 698
Last-Modified               The last object modification date (regardless of version)
696 699
Content-Encoding            The encoding of the object (optional)
697 700
Content-Disposition         The presentation style of the object (optional)
701
X-Object-Hash               The Merkle hash
698 702
X-Object-Version            The object's version identifier
699 703
X-Object-Version-Timestamp  The object's version timestamp
700 704
X-Object-Modified-By        The user that comitted the object's version
......
979 983
* Multi-range object ``GET`` support as outlined in RFC2616.
980 984
* Object hashmap retrieval through ``GET`` and the ``format`` parameter.
981 985
* Object create via hashmap through ``PUT`` and the ``format`` parameter.
986
* The object's Merkle hash is always returned in the ``X-Object-Hash`` header.
982 987
* Object create using ``POST`` to support standard HTML forms.
983 988
* Partial object updates through ``POST``, using the ``Content-Length``, ``Content-Type``, ``Content-Range`` and ``Transfer-Encoding`` headers. Use another object's data to update with ``X-Source-Object`` and ``X-Source-Version``. Truncate with ``X-Object-Bytes``. New ETag corresponds to the Merkle hash of the object's hashmap.
984 989
* Include new version identifier in replies for object replace/change requests.
b/pithos/api/functions.py
532 532
            except NameError:
533 533
                pass
534 534
            else:
535
                rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by ETag.
536
                rename_meta_key(meta, 'ETag', 'hash')
535 537
                rename_meta_key(meta, 'modified', 'last_modified')
536 538
                rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
537 539
                rename_meta_key(meta, 'version', 'x_object_version')
......
584 586
        validate_matching_preconditions(request, meta)
585 587
    except NotModified:
586 588
        response = HttpResponse(status=304)
587
        response['ETag'] = meta['hash']
589
        response['ETag'] = meta['ETag']
588 590
        return response
589 591
    
590 592
    response = HttpResponse(status=200)
......
653 655
        validate_matching_preconditions(request, meta)
654 656
    except NotModified:
655 657
        response = HttpResponse(status=304)
656
        response['ETag'] = meta['hash']
658
        response['ETag'] = meta['ETag']
657 659
        return response
658 660
    
659 661
    sizes = []
......
805 807
            except:
806 808
                raise BadRequest('Invalid data formatting')
807 809
        
808
        meta.update({'hash': hashmap_hash(request, hashmap)}) # Update ETag.
810
        meta.update({'ETag': hashmap_hash(request, hashmap)}) # Update ETag.
809 811
    else:
810 812
        md5 = hashlib.md5()
811 813
        size = 0
......
818 820
            hashmap.append(request.backend.put_block(data))
819 821
            md5.update(data)
820 822
        
821
        meta['hash'] = md5.hexdigest().lower()
823
        meta['ETag'] = md5.hexdigest().lower()
822 824
        etag = request.META.get('HTTP_ETAG')
823
        if etag and parse_etags(etag)[0].lower() != meta['hash']:
825
        if etag and parse_etags(etag)[0].lower() != meta['ETag']:
824 826
            raise UnprocessableEntity('Object ETag does not match')
825 827
    
826 828
    try:
......
849 851
            raise ItemNotFound('Object does not exist')
850 852
    
851 853
    response = HttpResponse(status=201)
852
    response['ETag'] = meta['hash']
854
    response['ETag'] = meta['ETag']
853 855
    response['X-Object-Version'] = version_id
854 856
    return response
855 857

  
......
876 878
        hashmap.append(request.backend.put_block(data))
877 879
        md5.update(data)
878 880
    
879
    meta['hash'] = md5.hexdigest().lower()
881
    meta['ETag'] = md5.hexdigest().lower()
880 882
    
881 883
    try:
882 884
        version_id = request.backend.update_object_hashmap(request.user_uniq,
......
889 891
        raise RequestEntityTooLarge('Quota exceeded')
890 892
    
891 893
    response = HttpResponse(status=201)
892
    response['ETag'] = meta['hash']
894
    response['ETag'] = meta['ETag']
893 895
    response['X-Object-Version'] = version_id
894 896
    return response
895 897

  
......
991 993
    if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
992 994
        validate_matching_preconditions(request, prev_meta)
993 995
    
994
    # If replacing, keep previous values of 'Content-Type' and 'hash'.
996
    # If replacing, keep previous values of 'Content-Type' and 'ETag'.
995 997
    replace = True
996 998
    if 'update' in request.GET:
997 999
        replace = False
998 1000
    if replace:
999
        for k in ('Content-Type', 'hash'):
1001
        for k in ('Content-Type', 'ETag'):
1000 1002
            if k in prev_meta:
1001 1003
                meta[k] = prev_meta[k]
1002 1004
    
......
1153 1155
    if dest_bytes is not None and dest_bytes < size:
1154 1156
        size = dest_bytes
1155 1157
        hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)]
1156
    meta.update({'hash': hashmap_hash(request, hashmap)}) # Update ETag.
1158
    meta.update({'ETag': hashmap_hash(request, hashmap)}) # Update ETag.
1157 1159
    try:
1158 1160
        version_id = request.backend.update_object_hashmap(request.user_uniq,
1159 1161
                        v_account, v_container, v_object, size, hashmap, meta,
......
1178 1180
            raise ItemNotFound('Object does not exist')
1179 1181
    
1180 1182
    response = HttpResponse(status=204)
1181
    response['ETag'] = meta['hash']
1183
    response['ETag'] = meta['ETag']
1182 1184
    response['X-Object-Version'] = version_id
1183 1185
    return response
1184 1186

  
b/pithos/api/util.py
173 173
    return meta, get_sharing(request), get_public(request)
174 174

  
175 175
def put_object_headers(response, meta, restricted=False):
176
    response['ETag'] = meta['hash']
176
    response['ETag'] = meta['ETag']
177 177
    response['Content-Length'] = meta['bytes']
178 178
    response['Content-Type'] = meta.get('Content-Type', 'application/octet-stream')
179 179
    response['Last-Modified'] = http_date(int(meta['modified']))
180 180
    if not restricted:
181
        response['X-Object-Hash'] = meta['hash']
181 182
        response['X-Object-Modified-By'] = smart_str(meta['modified_by'], strings_only=True)
182 183
        response['X-Object-Version'] = meta['version']
183 184
        response['X-Object-Version-Timestamp'] = http_date(int(meta['version_timestamp']))
......
197 198
    """Update metadata if the object has an X-Object-Manifest."""
198 199
    
199 200
    if 'X-Object-Manifest' in meta:
200
        hash = ''
201
        etag = ''
201 202
        bytes = 0
202 203
        try:
203 204
            src_container, src_name = split_container_object_string('/' + meta['X-Object-Manifest'])
......
206 207
            for x in objects:
207 208
                src_meta = request.backend.get_object_meta(request.user_uniq,
208 209
                                        v_account, src_container, x[0], x[1])
209
                hash += src_meta['hash']
210
                etag += src_meta['ETag']
210 211
                bytes += src_meta['bytes']
211 212
        except:
212 213
            # Ignore errors.
213 214
            return
214 215
        meta['bytes'] = bytes
215 216
        md5 = hashlib.md5()
216
        md5.update(hash)
217
        meta['hash'] = md5.hexdigest().lower()
217
        md5.update(etag)
218
        meta['ETag'] = md5.hexdigest().lower()
218 219

  
219 220
def update_sharing_meta(request, permissions, v_account, v_container, v_object, meta):
220 221
    if permissions is None:
......
261 262
def validate_matching_preconditions(request, meta):
262 263
    """Check that the ETag conforms with the preconditions set."""
263 264
    
264
    hash = meta.get('hash', None)
265
    etag = meta.get('ETag', None)
265 266
    
266 267
    if_match = request.META.get('HTTP_IF_MATCH')
267 268
    if if_match is not None:
268
        if hash is None:
269
        if etag is None:
269 270
            raise PreconditionFailed('Resource does not exist')
270
        if if_match != '*' and hash not in [x.lower() for x in parse_etags(if_match)]:
271
        if if_match != '*' and etag not in [x.lower() for x in parse_etags(if_match)]:
271 272
            raise PreconditionFailed('Resource ETag does not match')
272 273
    
273 274
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
274 275
    if if_none_match is not None:
275 276
        # TODO: If this passes, must ignore If-Modified-Since header.
276
        if hash is not None:
277
            if if_none_match == '*' or hash in [x.lower() for x in parse_etags(if_none_match)]:
277
        if etag is not None:
278
            if if_none_match == '*' or etag in [x.lower() for x in parse_etags(if_none_match)]:
278 279
                # TODO: Continue if an If-Modified-Since header is present.
279 280
                if request.method in ('HEAD', 'GET'):
280 281
                    raise NotModified('Resource ETag matches')
......
665 666
                    ranges = [(0, size)]
666 667
                    ret = 200
667 668
            except ValueError:
668
                if if_range != meta['hash']:
669
                if if_range != meta['ETag']:
669 670
                    ranges = [(0, size)]
670 671
                    ret = 200
671 672
    
b/pithos/backends/base.py
316 316
            
317 317
            'bytes': The total data size
318 318
            
319
            'hash': The hashmap hash
320
            
319 321
            'modified': Last modification timestamp (overall)
320 322
            
321 323
            'modified_by': The user that committed the object (version requested)
b/pithos/backends/modular.py
384 384
        
385 385
        if self._get_statistics(node)[0] > 0:
386 386
            raise IndexError('Container is not empty')
387
        hashes = self.node.node_purge_children(node, until, CLUSTER_HISTORY)
387
        hashes = self.node.node_purge_children(node, inf, CLUSTER_HISTORY)
388 388
        for h in hashes:
389 389
            self.store.map_delete(h)
390 390
        self.node.node_purge_children(node, inf, CLUSTER_DELETED)
......
446 446
                modified = del_props[self.MTIME]
447 447
        
448 448
        meta = dict(self.node.attribute_get(props[self.SERIAL]))
449
        meta.update({'name': name, 'bytes': props[self.SIZE]})
449
        meta.update({'name': name, 'bytes': props[self.SIZE], 'hash':props[self.HASH]})
450 450
        meta.update({'version': props[self.SERIAL], 'version_timestamp': props[self.MTIME]})
451 451
        meta.update({'modified': modified, 'modified_by': props[self.MUSER]})
452 452
        return meta
......
623 623
            hashes += self.node.node_purge(node, until, CLUSTER_HISTORY)
624 624
            for h in hashes:
625 625
                self.store.map_delete(h)
626
            self.node.node_purge_children(node, until, CLUSTER_DELETED)
626
            self.node.node_purge(node, until, CLUSTER_DELETED)
627 627
            try:
628 628
                props = self._get_version(node)
629 629
            except NameError:
630
                pass
631
            else:
632 630
                self.permissions.access_clear(path)
633 631
            return
634 632
        
b/pithos/public/functions.py
71 71
        raise ItemNotFound('Object does not exist')
72 72
    update_manifest_meta(request, v_account, meta)
73 73
    
74
    response = HttpResponse(status=204)
75
    put_object_meta(response, meta, True)
74
    response = HttpResponse(status=200)
75
    put_object_headers(response, meta, True)
76 76
    return response
77 77

  
78 78
@api_method('GET', user_required=False)
......
103 103
        validate_matching_preconditions(request, meta)
104 104
    except NotModified:
105 105
        response = HttpResponse(status=304)
106
        response['ETag'] = meta['hash']
106
        response['ETag'] = meta['ETag']
107 107
        return response
108 108
    
109 109
    sizes = []

Also available in: Unified diff