Revision 133e3fcf

b/docs/pithos-api-guide.rst
28 28
Revision                   Description
29 29
=========================  ================================
30 30
0.14 (Jun 18, 2013)        Forbidden response for public listing by non path owners
31
0.14 (Apr 23, 2013)        Reply with Merkle hash in the ETag if MD5 is not computed.
31 32
0.13 (Mar 27, 2013)        Restrict public object listing only to the owner.
32 33
\                          Do not propagate public URL information in shared objects.
33 34
0.13 (Jan 21, 2013)        Proxy identity management services
34 35
\                          UUID to displayname translation
35
0.9 (Feb 17, 2012)         Change permissions model.
36 36
0.10 (Jul 18, 2012)        Support for bulk COPY/MOVE/DELETE
37 37
\                          Optionally include public objects in listings.
38 38
0.9 (Feb 17, 2012)         Change permissions model.
......
947 947
==========================  ===============================
948 948
Reply Header Name           Value
949 949
==========================  ===============================
950
ETag                        The MD5 hash of the object
950
ETag                        The MD5 (or the Merkle if MD5 is deactivated) hash of the object
951 951
X-Object-Version            The object's new version
952 952
==========================  ===============================
953 953

  
......
960 960
409 (Conflict)                  The object can not be created from the provided hashmap (a list of missing hashes will be included in the reply)
961 961
411 (Length Required)           Missing ``Content-Length`` or ``Content-Type`` in the request
962 962
413 (Request Entity Too Large)  Insufficient quota to complete the request
963
422 (Unprocessable Entity)      The MD5 checksum of the data written to the storage system does not match the (optionally) supplied ETag value
963
422 (Unprocessable Entity)      The MD5 (or the Merkle if MD5 is deactivated) checksum of the data written to the storage system does not match the (optionally) supplied ETag value
964 964
==============================  ==============================
965 965

  
966 966

  
......
1102 1102
==========================  ===============================
1103 1103
Reply Header Name           Value
1104 1104
==========================  ===============================
1105
ETag                        The MD5 hash of the object
1105
ETag                        The MD5 (or the Merkle if MD5 is deactivated) hash of the object
1106 1106
X-Object-Version            The object's new version
1107 1107
==========================  ===============================
1108 1108

  
b/snf-pithos-app/pithos/api/functions.py
55 55
    copy_or_move_object, get_int_parameter, get_content_length,
56 56
    get_content_range, socket_read_iterator, SaveToBackendHandler,
57 57
    object_data_response, put_object_block, hashmap_md5, simple_list_response,
58
    api_method, is_uuid,
59
    retrieve_uuid, retrieve_uuids, retrieve_displaynames,
60
    get_pithos_usage
58
    api_method, is_uuid, retrieve_uuid, retrieve_uuids,
59
    retrieve_displaynames, get_pithos_usage, Checksum, NoChecksum
61 60
)
62 61

  
63 62
from pithos.api.settings import (UPDATE_MD5, TRANSLATE_UUIDS,
......
65 64

  
66 65
from pithos.api import settings
67 66

  
68
from pithos.api import settings
69

  
70 67
from pithos.backends.base import (
71 68
    NotAllowedError, QuotaError, ContainerNotEmpty, ItemNotExists,
72 69
    VersionNotExists, ContainerExists)
73 70

  
74 71
from pithos.backends.filter import parse_filters
75 72

  
76
import hashlib
77

  
78 73
import logging
79 74
logger = logging.getLogger(__name__)
80 75

  
......
818 813
        validate_matching_preconditions(request, meta)
819 814
    except faults.NotModified:
820 815
        response = HttpResponse(status=304)
821
        response['ETag'] = meta['checksum']
816
        response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
822 817
        return response
823 818

  
824 819
    response = HttpResponse(status=200)
......
899 894
        validate_matching_preconditions(request, meta)
900 895
    except faults.NotModified:
901 896
        response = HttpResponse(status=304)
902
        response['ETag'] = meta['checksum']
897
        response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
903 898
        return response
904 899

  
905 900
    hashmap_reply = False
......
1083 1078

  
1084 1079
        checksum = ''  # Do not set to None (will copy previous value).
1085 1080
    else:
1086
        md5 = hashlib.md5()
1081
        etag = request.META.get('HTTP_ETAG')
1082
        checksum_compute = Checksum() if etag or UPDATE_MD5 else NoChecksum()
1087 1083
        size = 0
1088 1084
        hashmap = []
1089 1085
        for data in socket_read_iterator(request, content_length,
......
1093 1089
            #       and we stop before getting this much data.
1094 1090
            size += len(data)
1095 1091
            hashmap.append(request.backend.put_block(data))
1096
            md5.update(data)
1092
            checksum_compute.update(data)
1097 1093

  
1098
        checksum = md5.hexdigest().lower()
1099
        etag = request.META.get('HTTP_ETAG')
1094
        checksum = checksum_compute.hexdigest()
1100 1095
        if etag and parse_etags(etag)[0].lower() != checksum:
1101 1096
            raise faults.UnprocessableEntity('Object ETag does not match')
1102 1097

  
1103 1098
    try:
1104
        version_id = \
1105
            request.backend.update_object_hashmap(request.user_uniq,
1106
                                                  v_account, v_container,
1107
                                                  v_object, size, content_type,
1108
                                                  hashmap, checksum,
1109
                                                  'pithos', meta, True,
1110
                                                  permissions)
1099
        version_id, merkle = request.backend.update_object_hashmap(
1100
            request.user_uniq, v_account, v_container, v_object, size,
1101
            content_type, hashmap, checksum, 'pithos', meta, True, permissions
1102
        )
1111 1103
    except NotAllowedError:
1112 1104
        raise faults.Forbidden('Not allowed')
1113 1105
    except IndexError, e:
......
1141 1133
            raise faults.ItemNotFound('Object does not exist')
1142 1134

  
1143 1135
    response = HttpResponse(status=201)
1144
    if checksum:
1145
        response['ETag'] = checksum
1136
    response['ETag'] = merkle if not UPDATE_MD5 else checksum
1146 1137
    response['X-Object-Version'] = version_id
1147 1138
    return response
1148 1139

  
......
1163 1154

  
1164 1155
    checksum = file.etag
1165 1156
    try:
1166
        version_id = \
1167
            request.backend.update_object_hashmap(request.user_uniq,
1168
                                                  v_account, v_container,
1169
                                                  v_object, file.size,
1170
                                                  file.content_type,
1171
                                                  file.hashmap, checksum,
1172
                                                  'pithos', {}, True)
1157
        version_id, merkle = request.backend.update_object_hashmap(
1158
            request.user_uniq, v_account, v_container, v_object, file.size,
1159
            file.content_type, file.hashmap, checksum, 'pithos', {}, True
1160
        )
1173 1161
    except NotAllowedError:
1174 1162
        raise faults.Forbidden('Not allowed')
1175 1163
    except ItemNotExists:
......
1178 1166
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
1179 1167

  
1180 1168
    response = HttpResponse(status=201)
1181
    response['ETag'] = checksum
1169
    response['ETag'] = merkle if not UPDATE_MD5 else checksum
1182 1170
    response['X-Object-Version'] = version_id
1183 1171
    response.content = checksum
1184 1172
    return response
......
1466 1454
    checksum = hashmap_md5(
1467 1455
        request.backend, hashmap, size) if UPDATE_MD5 else ''
1468 1456
    try:
1469
        version_id = \
1470
            request.backend.update_object_hashmap(request.user_uniq,
1471
                                                  v_account, v_container,
1472
                                                  v_object, size,
1473
                                                  prev_meta['type'],
1474
                                                  hashmap, checksum, 'pithos',
1475
                                                  meta, replace, permissions)
1457
        version_id, merkle = request.backend.update_object_hashmap(
1458
            request.user_uniq, v_account, v_container, v_object, size,
1459
            prev_meta['type'], hashmap, checksum, 'pithos', meta, replace,
1460
            permissions
1461
        )
1476 1462
    except NotAllowedError:
1477 1463
        raise faults.Forbidden('Not allowed')
1478 1464
    except ItemNotExists:
......
1491 1477
            raise faults.ItemNotFound('Object does not exist')
1492 1478

  
1493 1479
    response = HttpResponse(status=204)
1494
    response['ETag'] = checksum
1480
    response['ETag'] = merkle if not UPDATE_MD5 else checksum
1495 1481
    response['X-Object-Version'] = version_id
1496 1482
    return response
1497 1483

  
b/snf-pithos-app/pithos/api/manage_accounts/__init__.py
31 31
# interpreted as representing official policies, either expressed
32 32
# or implied, of GRNET S.A.
33 33

  
34
from pithos.api.util import get_backend, split_container_object_string
35

  
34
from pithos.api.util import (
35
    get_backend, split_container_object_string, Checksum, NoChecksum
36
)
36 37
import re
37
import hashlib
38 38
import os
39 39

  
40 40

  
......
307 307
        return self.backend._lookup_account(account, create=True)
308 308

  
309 309
    def create_update_object(self, account, container, name, content_type,
310
                             data, meta=None, permissions=None, request_user=None):
310
                             data, meta=None, permissions=None,
311
                             request_user=None,
312
                             checksum_compute_class=NoChecksum):
311 313
        meta = meta or {}
312 314
        permissions = permissions or {}
313
        md5 = hashlib.md5()
315

  
316
        assert checksum_compute_class in (NoChecksum, Checksum), 'Invalid checksum_compute_class'
317
        checksum_compute = checksum_compute_class()
314 318
        size = 0
315 319
        hashmap = []
316 320
        for block_data in data_read_iterator(data, self.backend.block_size):
317 321
            size += len(block_data)
318 322
            hashmap.append(self.backend.put_block(block_data))
319
            md5.update(block_data)
323
            checksum_compute.update(block_data)
320 324

  
321
        checksum = md5.hexdigest().lower()
325
        checksum = checksum_compute.hexdigest()
322 326

  
323 327
        request_user = request_user or account
324 328
        return self.backend.update_object_hashmap(request_user, account,
b/snf-pithos-app/pithos/api/manage_accounts/tests.py
195 195
                                                   'object1',
196 196
                                                   'application/octet-stream',
197 197
                                                   data))
198
        self.assertEquals(sorted(versions[:-1]),
198
        self.assertEquals(sorted([i[0] for i in versions[:-1]]),
199 199
                          self.utils.list_past_versions('account1',
200 200
                                                        'container1',
201 201
                                                        'object1'))
......
226 226
                               dry=False, silent=True)
227 227

  
228 228
        expected = {'meta': meta,
229
                    'versions': versions[:-1],
229
                    'versions': [i[0] for i in versions[:-1]],
230 230
                    'permissions': permissions}
231 231
        self._verify_object('account1', 'container1', 'object1', expected)
232 232

  
......
275 275
                    'permissions': permissions}
276 276
        for c, o_dict in versions.iteritems():
277 277
            for o, versions in o_dict.iteritems():
278
                expected['versions'] = versions[:-1]
278
                expected['versions'] = [i[0] for i in versions[:-1]]
279 279
                self._verify_object('account1', c, o, expected)
280 280

  
281 281
    def test_merge_existing_dest_container(self):
......
314 314
            self.fail(e)
315 315

  
316 316
        expected = {'meta': meta,
317
                    'versions': versions[:-1],
317
                    'versions': [i[0] for i in versions[:-1]],
318 318
                    'permissions': permissions}
319 319
        self._verify_object('account1', 'container1', 'object1', expected)
320 320

  
......
361 361

  
362 362
        expected = {'meta': meta,
363 363
                    'permissions': permissions,
364
                    'versions': versions[:-1]}
364
                    'versions': [i[0] for i in versions[:-1]]}
365 365
        self._verify_object('account1', container, object, expected,
366 366
                            strict=False)
367 367

  
b/snf-pithos-app/pithos/api/util.py
62 62
                                 BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM,
63 63
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
64 64
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
65
                                 PUBLIC_URL_SECURITY,
66
                                 PUBLIC_URL_ALPHABET,
67
                                 COOKIE_NAME, BASE_HOST)
65
                                 PUBLIC_URL_SECURITY, PUBLIC_URL_ALPHABET,
66
                                 COOKIE_NAME, BASE_HOST, UPDATE_MD5)
68 67
from pithos.api.resources import resources
69 68
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
70 69
                                  VersionNotExists)
......
216 215

  
217 216

  
218 217
def put_object_headers(response, meta, restricted=False, token=None):
219
    response['ETag'] = meta['checksum']
218
    response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
220 219
    response['Content-Length'] = meta['bytes']
221 220
    response.override_serialization = True
222 221
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
......
260 259
                request.user_uniq, v_account,
261 260
                src_container, prefix=src_name, virtual=False)
262 261
            for x in objects:
263
                src_meta = request.backend.get_object_meta(request.user_uniq,
264
                                                           v_account,
265
                                                           src_container,
266
                                                           x[0], 'pithos', x[1])
267
                etag += src_meta['checksum']
262
                src_meta = request.backend.get_object_meta(
263
                    request.user_uniq, v_account, src_container, x[0],
264
                    'pithos', x[1])
265
                etag += (src_meta['hash'] if not UPDATE_MD5 else
266
                         src_meta['checksum'])
268 267
                bytes += src_meta['bytes']
269 268
        except:
270 269
            # Ignore errors.
......
424 423
def validate_matching_preconditions(request, meta):
425 424
    """Check that the ETag conforms with the preconditions set."""
426 425

  
427
    etag = meta['checksum']
426
    etag = meta['hash'] if not UPDATE_MD5 else meta['checksum']
428 427
    if not etag:
429 428
        etag = None
430 429

  
......
773 772
        if len(self.data) >= length:
774 773
            block = self.data[:length]
775 774
            self.file.hashmap.append(self.backend.put_block(block))
776
            self.md5.update(block)
775
            self.checksum_compute.update(block)
777 776
            self.data = self.data[length:]
778 777

  
779 778
    def new_file(self, field_name, file_name, content_type,
780 779
                 content_length, charset=None):
781
        self.md5 = hashlib.md5()
780
        self.checksum_compute = NoChecksum() if not UPDATE_MD5 else Checksum()
782 781
        self.data = ''
783 782
        self.file = UploadedFile(
784 783
            name=file_name, content_type=content_type, charset=charset)
......
795 794
        l = len(self.data)
796 795
        if l > 0:
797 796
            self.put_data(l)
798
        self.file.etag = self.md5.hexdigest().lower()
797
        self.file.etag = self.checksum_compute.hexdigest()
799 798
        return self.file
800 799

  
801 800

  
......
1161 1160
                raise Exception(response)
1162 1161
        return wrapper
1163 1162
    return decorator
1163

  
1164

  
1165
class Checksum:
1166
    def __init__(self):
1167
        self.md5 = hashlib.md5()
1168

  
1169
    def update(self, data):
1170
        self.md5.update(data)
1171

  
1172
    def hexdigest(self):
1173
        return self.md5.hexdigest().lower()
1174

  
1175
class NoChecksum:
1176
    def update(self, data):
1177
        pass
1178

  
1179
    def hexdigest(self):
1180
        return ''
b/snf-pithos-backend/pithos/backends/modular.py
925 925
            raise ie
926 926

  
927 927
        hash = map.hash()
928
        dest_version_id = self._update_object_hash(user, account, container, name, size, type, binascii.hexlify(hash), checksum, domain, meta, replace_meta, permissions)
928
        hexlified = binascii.hexlify(hash)
929
        dest_version_id = self._update_object_hash(user, account, container, name, size, type, hexlified, checksum, domain, meta, replace_meta, permissions)
929 930
        self.store.map_put(hash, map)
930
        return dest_version_id
931
        return dest_version_id, hexlified
931 932

  
932 933
    def update_object_checksum(self, user, account, container, name, version, checksum):
933 934
        """Update an object's checksum."""

Also available in: Unified diff