Revision 133e3fcf
b/docs/pithos-api-guide.rst | ||
---|---|---|
28 | 28 |
Revision Description |
29 | 29 |
========================= ================================ |
30 | 30 |
0.14 (Jun 18, 2013) Forbidden response for public listing by non path owners |
31 |
0.14 (Apr 23, 2013) Reply with Merkle hash in the ETag if MD5 is not computed. |
|
31 | 32 |
0.13 (Mar 27, 2013) Restrict public object listing only to the owner. |
32 | 33 |
\ Do not propagate public URL information in shared objects. |
33 | 34 |
0.13 (Jan 21, 2013) Proxy identity management services |
34 | 35 |
\ UUID to displayname translation |
35 |
0.9 (Feb 17, 2012) Change permissions model. |
|
36 | 36 |
0.10 (Jul 18, 2012) Support for bulk COPY/MOVE/DELETE |
37 | 37 |
\ Optionally include public objects in listings. |
38 | 38 |
0.9 (Feb 17, 2012) Change permissions model. |
... | ... | |
947 | 947 |
========================== =============================== |
948 | 948 |
Reply Header Name Value |
949 | 949 |
========================== =============================== |
950 |
ETag The MD5 hash of the object |
|
950 |
ETag The MD5 (or the Merkle if MD5 is deactivated) hash of the object
|
|
951 | 951 |
X-Object-Version The object's new version |
952 | 952 |
========================== =============================== |
953 | 953 |
|
... | ... | |
960 | 960 |
409 (Conflict) The object can not be created from the provided hashmap (a list of missing hashes will be included in the reply) |
961 | 961 |
411 (Length Required) Missing ``Content-Length`` or ``Content-Type`` in the request |
962 | 962 |
413 (Request Entity Too Large) Insufficient quota to complete the request |
963 |
422 (Unprocessable Entity) The MD5 checksum of the data written to the storage system does not match the (optionally) supplied ETag value |
|
963 |
422 (Unprocessable Entity) The MD5 (or the Merkle if MD5 is deactivated) checksum of the data written to the storage system does not match the (optionally) supplied ETag value
|
|
964 | 964 |
============================== ============================== |
965 | 965 |
|
966 | 966 |
|
... | ... | |
1102 | 1102 |
========================== =============================== |
1103 | 1103 |
Reply Header Name Value |
1104 | 1104 |
========================== =============================== |
1105 |
ETag The MD5 hash of the object |
|
1105 |
ETag The MD5 (or the Merkle if MD5 is deactivated) hash of the object
|
|
1106 | 1106 |
X-Object-Version The object's new version |
1107 | 1107 |
========================== =============================== |
1108 | 1108 |
|
b/snf-pithos-app/pithos/api/functions.py | ||
---|---|---|
55 | 55 |
copy_or_move_object, get_int_parameter, get_content_length, |
56 | 56 |
get_content_range, socket_read_iterator, SaveToBackendHandler, |
57 | 57 |
object_data_response, put_object_block, hashmap_md5, simple_list_response, |
58 |
api_method, is_uuid, |
|
59 |
retrieve_uuid, retrieve_uuids, retrieve_displaynames, |
|
60 |
get_pithos_usage |
|
58 |
api_method, is_uuid, retrieve_uuid, retrieve_uuids, |
|
59 |
retrieve_displaynames, get_pithos_usage, Checksum, NoChecksum |
|
61 | 60 |
) |
62 | 61 |
|
63 | 62 |
from pithos.api.settings import (UPDATE_MD5, TRANSLATE_UUIDS, |
... | ... | |
65 | 64 |
|
66 | 65 |
from pithos.api import settings |
67 | 66 |
|
68 |
from pithos.api import settings |
|
69 |
|
|
70 | 67 |
from pithos.backends.base import ( |
71 | 68 |
NotAllowedError, QuotaError, ContainerNotEmpty, ItemNotExists, |
72 | 69 |
VersionNotExists, ContainerExists) |
73 | 70 |
|
74 | 71 |
from pithos.backends.filter import parse_filters |
75 | 72 |
|
76 |
import hashlib |
|
77 |
|
|
78 | 73 |
import logging |
79 | 74 |
logger = logging.getLogger(__name__) |
80 | 75 |
|
... | ... | |
818 | 813 |
validate_matching_preconditions(request, meta) |
819 | 814 |
except faults.NotModified: |
820 | 815 |
response = HttpResponse(status=304) |
821 |
response['ETag'] = meta['checksum'] |
|
816 |
response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
|
|
822 | 817 |
return response |
823 | 818 |
|
824 | 819 |
response = HttpResponse(status=200) |
... | ... | |
899 | 894 |
validate_matching_preconditions(request, meta) |
900 | 895 |
except faults.NotModified: |
901 | 896 |
response = HttpResponse(status=304) |
902 |
response['ETag'] = meta['checksum'] |
|
897 |
response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
|
|
903 | 898 |
return response |
904 | 899 |
|
905 | 900 |
hashmap_reply = False |
... | ... | |
1083 | 1078 |
|
1084 | 1079 |
checksum = '' # Do not set to None (will copy previous value). |
1085 | 1080 |
else: |
1086 |
md5 = hashlib.md5() |
|
1081 |
etag = request.META.get('HTTP_ETAG') |
|
1082 |
checksum_compute = Checksum() if etag or UPDATE_MD5 else NoChecksum() |
|
1087 | 1083 |
size = 0 |
1088 | 1084 |
hashmap = [] |
1089 | 1085 |
for data in socket_read_iterator(request, content_length, |
... | ... | |
1093 | 1089 |
# and we stop before getting this much data. |
1094 | 1090 |
size += len(data) |
1095 | 1091 |
hashmap.append(request.backend.put_block(data)) |
1096 |
md5.update(data)
|
|
1092 |
checksum_compute.update(data)
|
|
1097 | 1093 |
|
1098 |
checksum = md5.hexdigest().lower() |
|
1099 |
etag = request.META.get('HTTP_ETAG') |
|
1094 |
checksum = checksum_compute.hexdigest() |
|
1100 | 1095 |
if etag and parse_etags(etag)[0].lower() != checksum: |
1101 | 1096 |
raise faults.UnprocessableEntity('Object ETag does not match') |
1102 | 1097 |
|
1103 | 1098 |
try: |
1104 |
version_id = \ |
|
1105 |
request.backend.update_object_hashmap(request.user_uniq, |
|
1106 |
v_account, v_container, |
|
1107 |
v_object, size, content_type, |
|
1108 |
hashmap, checksum, |
|
1109 |
'pithos', meta, True, |
|
1110 |
permissions) |
|
1099 |
version_id, merkle = request.backend.update_object_hashmap( |
|
1100 |
request.user_uniq, v_account, v_container, v_object, size, |
|
1101 |
content_type, hashmap, checksum, 'pithos', meta, True, permissions |
|
1102 |
) |
|
1111 | 1103 |
except NotAllowedError: |
1112 | 1104 |
raise faults.Forbidden('Not allowed') |
1113 | 1105 |
except IndexError, e: |
... | ... | |
1141 | 1133 |
raise faults.ItemNotFound('Object does not exist') |
1142 | 1134 |
|
1143 | 1135 |
response = HttpResponse(status=201) |
1144 |
if checksum: |
|
1145 |
response['ETag'] = checksum |
|
1136 |
response['ETag'] = merkle if not UPDATE_MD5 else checksum |
|
1146 | 1137 |
response['X-Object-Version'] = version_id |
1147 | 1138 |
return response |
1148 | 1139 |
|
... | ... | |
1163 | 1154 |
|
1164 | 1155 |
checksum = file.etag |
1165 | 1156 |
try: |
1166 |
version_id = \ |
|
1167 |
request.backend.update_object_hashmap(request.user_uniq, |
|
1168 |
v_account, v_container, |
|
1169 |
v_object, file.size, |
|
1170 |
file.content_type, |
|
1171 |
file.hashmap, checksum, |
|
1172 |
'pithos', {}, True) |
|
1157 |
version_id, merkle = request.backend.update_object_hashmap( |
|
1158 |
request.user_uniq, v_account, v_container, v_object, file.size, |
|
1159 |
file.content_type, file.hashmap, checksum, 'pithos', {}, True |
|
1160 |
) |
|
1173 | 1161 |
except NotAllowedError: |
1174 | 1162 |
raise faults.Forbidden('Not allowed') |
1175 | 1163 |
except ItemNotExists: |
... | ... | |
1178 | 1166 |
raise faults.RequestEntityTooLarge('Quota error: %s' % e) |
1179 | 1167 |
|
1180 | 1168 |
response = HttpResponse(status=201) |
1181 |
response['ETag'] = checksum |
|
1169 |
response['ETag'] = merkle if not UPDATE_MD5 else checksum
|
|
1182 | 1170 |
response['X-Object-Version'] = version_id |
1183 | 1171 |
response.content = checksum |
1184 | 1172 |
return response |
... | ... | |
1466 | 1454 |
checksum = hashmap_md5( |
1467 | 1455 |
request.backend, hashmap, size) if UPDATE_MD5 else '' |
1468 | 1456 |
try: |
1469 |
version_id = \ |
|
1470 |
request.backend.update_object_hashmap(request.user_uniq, |
|
1471 |
v_account, v_container, |
|
1472 |
v_object, size, |
|
1473 |
prev_meta['type'], |
|
1474 |
hashmap, checksum, 'pithos', |
|
1475 |
meta, replace, permissions) |
|
1457 |
version_id, merkle = request.backend.update_object_hashmap( |
|
1458 |
request.user_uniq, v_account, v_container, v_object, size, |
|
1459 |
prev_meta['type'], hashmap, checksum, 'pithos', meta, replace, |
|
1460 |
permissions |
|
1461 |
) |
|
1476 | 1462 |
except NotAllowedError: |
1477 | 1463 |
raise faults.Forbidden('Not allowed') |
1478 | 1464 |
except ItemNotExists: |
... | ... | |
1491 | 1477 |
raise faults.ItemNotFound('Object does not exist') |
1492 | 1478 |
|
1493 | 1479 |
response = HttpResponse(status=204) |
1494 |
response['ETag'] = checksum |
|
1480 |
response['ETag'] = merkle if not UPDATE_MD5 else checksum
|
|
1495 | 1481 |
response['X-Object-Version'] = version_id |
1496 | 1482 |
return response |
1497 | 1483 |
|
b/snf-pithos-app/pithos/api/manage_accounts/__init__.py | ||
---|---|---|
31 | 31 |
# interpreted as representing official policies, either expressed |
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 |
from pithos.api.util import get_backend, split_container_object_string |
|
35 |
|
|
34 |
from pithos.api.util import ( |
|
35 |
get_backend, split_container_object_string, Checksum, NoChecksum |
|
36 |
) |
|
36 | 37 |
import re |
37 |
import hashlib |
|
38 | 38 |
import os |
39 | 39 |
|
40 | 40 |
|
... | ... | |
307 | 307 |
return self.backend._lookup_account(account, create=True) |
308 | 308 |
|
309 | 309 |
def create_update_object(self, account, container, name, content_type, |
310 |
data, meta=None, permissions=None, request_user=None): |
|
310 |
data, meta=None, permissions=None, |
|
311 |
request_user=None, |
|
312 |
checksum_compute_class=NoChecksum): |
|
311 | 313 |
meta = meta or {} |
312 | 314 |
permissions = permissions or {} |
313 |
md5 = hashlib.md5() |
|
315 |
|
|
316 |
assert checksum_compute_class in (NoChecksum, Checksum), 'Invalid checksum_compute_class' |
|
317 |
checksum_compute = checksum_compute_class() |
|
314 | 318 |
size = 0 |
315 | 319 |
hashmap = [] |
316 | 320 |
for block_data in data_read_iterator(data, self.backend.block_size): |
317 | 321 |
size += len(block_data) |
318 | 322 |
hashmap.append(self.backend.put_block(block_data)) |
319 |
md5.update(block_data)
|
|
323 |
checksum_compute.update(block_data)
|
|
320 | 324 |
|
321 |
checksum = md5.hexdigest().lower()
|
|
325 |
checksum = checksum_compute.hexdigest()
|
|
322 | 326 |
|
323 | 327 |
request_user = request_user or account |
324 | 328 |
return self.backend.update_object_hashmap(request_user, account, |
b/snf-pithos-app/pithos/api/manage_accounts/tests.py | ||
---|---|---|
195 | 195 |
'object1', |
196 | 196 |
'application/octet-stream', |
197 | 197 |
data)) |
198 |
self.assertEquals(sorted(versions[:-1]),
|
|
198 |
self.assertEquals(sorted([i[0] for i in versions[:-1]]),
|
|
199 | 199 |
self.utils.list_past_versions('account1', |
200 | 200 |
'container1', |
201 | 201 |
'object1')) |
... | ... | |
226 | 226 |
dry=False, silent=True) |
227 | 227 |
|
228 | 228 |
expected = {'meta': meta, |
229 |
'versions': versions[:-1],
|
|
229 |
'versions': [i[0] for i in versions[:-1]],
|
|
230 | 230 |
'permissions': permissions} |
231 | 231 |
self._verify_object('account1', 'container1', 'object1', expected) |
232 | 232 |
|
... | ... | |
275 | 275 |
'permissions': permissions} |
276 | 276 |
for c, o_dict in versions.iteritems(): |
277 | 277 |
for o, versions in o_dict.iteritems(): |
278 |
expected['versions'] = versions[:-1]
|
|
278 |
expected['versions'] = [i[0] for i in versions[:-1]]
|
|
279 | 279 |
self._verify_object('account1', c, o, expected) |
280 | 280 |
|
281 | 281 |
def test_merge_existing_dest_container(self): |
... | ... | |
314 | 314 |
self.fail(e) |
315 | 315 |
|
316 | 316 |
expected = {'meta': meta, |
317 |
'versions': versions[:-1],
|
|
317 |
'versions': [i[0] for i in versions[:-1]],
|
|
318 | 318 |
'permissions': permissions} |
319 | 319 |
self._verify_object('account1', 'container1', 'object1', expected) |
320 | 320 |
|
... | ... | |
361 | 361 |
|
362 | 362 |
expected = {'meta': meta, |
363 | 363 |
'permissions': permissions, |
364 |
'versions': versions[:-1]}
|
|
364 |
'versions': [i[0] for i in versions[:-1]]}
|
|
365 | 365 |
self._verify_object('account1', container, object, expected, |
366 | 366 |
strict=False) |
367 | 367 |
|
b/snf-pithos-app/pithos/api/util.py | ||
---|---|---|
62 | 62 |
BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM, |
63 | 63 |
RADOS_STORAGE, RADOS_POOL_BLOCKS, |
64 | 64 |
RADOS_POOL_MAPS, TRANSLATE_UUIDS, |
65 |
PUBLIC_URL_SECURITY, |
|
66 |
PUBLIC_URL_ALPHABET, |
|
67 |
COOKIE_NAME, BASE_HOST) |
|
65 |
PUBLIC_URL_SECURITY, PUBLIC_URL_ALPHABET, |
|
66 |
COOKIE_NAME, BASE_HOST, UPDATE_MD5) |
|
68 | 67 |
from pithos.api.resources import resources |
69 | 68 |
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists, |
70 | 69 |
VersionNotExists) |
... | ... | |
216 | 215 |
|
217 | 216 |
|
218 | 217 |
def put_object_headers(response, meta, restricted=False, token=None): |
219 |
response['ETag'] = meta['checksum'] |
|
218 |
response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
|
|
220 | 219 |
response['Content-Length'] = meta['bytes'] |
221 | 220 |
response.override_serialization = True |
222 | 221 |
response['Content-Type'] = meta.get('type', 'application/octet-stream') |
... | ... | |
260 | 259 |
request.user_uniq, v_account, |
261 | 260 |
src_container, prefix=src_name, virtual=False) |
262 | 261 |
for x in objects: |
263 |
src_meta = request.backend.get_object_meta(request.user_uniq,
|
|
264 |
v_account,
|
|
265 |
src_container,
|
|
266 |
x[0], 'pithos', x[1])
|
|
267 |
etag += src_meta['checksum']
|
|
262 |
src_meta = request.backend.get_object_meta( |
|
263 |
request.user_uniq, v_account, src_container, x[0],
|
|
264 |
'pithos', x[1])
|
|
265 |
etag += (src_meta['hash'] if not UPDATE_MD5 else
|
|
266 |
src_meta['checksum'])
|
|
268 | 267 |
bytes += src_meta['bytes'] |
269 | 268 |
except: |
270 | 269 |
# Ignore errors. |
... | ... | |
424 | 423 |
def validate_matching_preconditions(request, meta): |
425 | 424 |
"""Check that the ETag conforms with the preconditions set.""" |
426 | 425 |
|
427 |
etag = meta['checksum'] |
|
426 |
etag = meta['hash'] if not UPDATE_MD5 else meta['checksum']
|
|
428 | 427 |
if not etag: |
429 | 428 |
etag = None |
430 | 429 |
|
... | ... | |
773 | 772 |
if len(self.data) >= length: |
774 | 773 |
block = self.data[:length] |
775 | 774 |
self.file.hashmap.append(self.backend.put_block(block)) |
776 |
self.md5.update(block)
|
|
775 |
self.checksum_compute.update(block)
|
|
777 | 776 |
self.data = self.data[length:] |
778 | 777 |
|
779 | 778 |
def new_file(self, field_name, file_name, content_type, |
780 | 779 |
content_length, charset=None): |
781 |
self.md5 = hashlib.md5()
|
|
780 |
self.checksum_compute = NoChecksum() if not UPDATE_MD5 else Checksum()
|
|
782 | 781 |
self.data = '' |
783 | 782 |
self.file = UploadedFile( |
784 | 783 |
name=file_name, content_type=content_type, charset=charset) |
... | ... | |
795 | 794 |
l = len(self.data) |
796 | 795 |
if l > 0: |
797 | 796 |
self.put_data(l) |
798 |
self.file.etag = self.md5.hexdigest().lower()
|
|
797 |
self.file.etag = self.checksum_compute.hexdigest()
|
|
799 | 798 |
return self.file |
800 | 799 |
|
801 | 800 |
|
... | ... | |
1161 | 1160 |
raise Exception(response) |
1162 | 1161 |
return wrapper |
1163 | 1162 |
return decorator |
1163 |
|
|
1164 |
|
|
1165 |
class Checksum: |
|
1166 |
def __init__(self): |
|
1167 |
self.md5 = hashlib.md5() |
|
1168 |
|
|
1169 |
def update(self, data): |
|
1170 |
self.md5.update(data) |
|
1171 |
|
|
1172 |
def hexdigest(self): |
|
1173 |
return self.md5.hexdigest().lower() |
|
1174 |
|
|
1175 |
class NoChecksum: |
|
1176 |
def update(self, data): |
|
1177 |
pass |
|
1178 |
|
|
1179 |
def hexdigest(self): |
|
1180 |
return '' |
b/snf-pithos-backend/pithos/backends/modular.py | ||
---|---|---|
925 | 925 |
raise ie |
926 | 926 |
|
927 | 927 |
hash = map.hash() |
928 |
dest_version_id = self._update_object_hash(user, account, container, name, size, type, binascii.hexlify(hash), checksum, domain, meta, replace_meta, permissions) |
|
928 |
hexlified = binascii.hexlify(hash) |
|
929 |
dest_version_id = self._update_object_hash(user, account, container, name, size, type, hexlified, checksum, domain, meta, replace_meta, permissions) |
|
929 | 930 |
self.store.map_put(hash, map) |
930 |
return dest_version_id |
|
931 |
return dest_version_id, hexlified
|
|
931 | 932 |
|
932 | 933 |
def update_object_checksum(self, user, account, container, name, version, checksum): |
933 | 934 |
"""Update an object's checksum.""" |
Also available in: Unified diff