Revision b3155065
| b/snf-pithos-app/README | ||
|---|---|---|
| 43 | 43 |
PITHOS_UPDATE_MD5 True Update object checksums when using hashmaps |
| 44 | 44 |
=============================== ================================================ ============================================================ |
| 45 | 45 |
|
| 46 |
To update checksums asynchronously, enable the queue, install snf-pithos-tools and use ``pithos-dispatcher``:: |
|
| 47 |
|
|
| 48 |
pithos-dispatcher --exchange=pithos --callback=pithos.api.dispatch.update_md5 |
|
| 49 |
|
|
| 46 | 50 |
Administrator functions |
| 47 | 51 |
----------------------- |
| 48 | 52 |
|
| b/snf-pithos-app/pithos/api/dispatch.py | ||
|---|---|---|
| 1 |
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION, |
|
| 2 |
BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH, |
|
| 3 |
BACKEND_QUEUE_MODULE, BACKEND_QUEUE_CONNECTION, |
|
| 4 |
BACKEND_QUOTA, BACKEND_VERSIONING) |
|
| 5 |
from pithos.backends import connect_backend |
|
| 6 |
from pithos.api.util import hashmap_md5 |
|
| 7 |
|
|
| 8 |
def update_md5(m): |
|
| 9 |
if m['resource'] != 'object' or m['details']['action'] != 'object update': |
|
| 10 |
return |
|
| 11 |
|
|
| 12 |
backend = connect_backend(db_module=BACKEND_DB_MODULE, |
|
| 13 |
db_connection=BACKEND_DB_CONNECTION, |
|
| 14 |
block_module=BACKEND_BLOCK_MODULE, |
|
| 15 |
block_path=BACKEND_BLOCK_PATH, |
|
| 16 |
queue_module=BACKEND_QUEUE_MODULE, |
|
| 17 |
queue_connection=BACKEND_QUEUE_CONNECTION) |
|
| 18 |
backend.default_policy['quota'] = BACKEND_QUOTA |
|
| 19 |
backend.default_policy['versioning'] = BACKEND_VERSIONING |
|
| 20 |
|
|
| 21 |
path = m['value'] |
|
| 22 |
account, container, name = path.split('/', 2)
|
|
| 23 |
version = m['details']['version'] |
|
| 24 |
meta = None |
|
| 25 |
try: |
|
| 26 |
meta = backend.get_object_meta(account, account, container, name, 'pithos', version) |
|
| 27 |
if meta['checksum'] == '': |
|
| 28 |
size, hashmap = backend.get_object_hashmap(account, account, container, name, version) |
|
| 29 |
checksum = hashmap_md5(backend, hashmap, size) |
|
| 30 |
backend.update_object_checksum(account, account, container, name, version, checksum) |
|
| 31 |
print 'INFO: Updated checksum for path "%s"' % (path,) |
|
| 32 |
except Exception, e: |
|
| 33 |
print 'WARNING: Can not update checksum for path "%s" (%s)' % (path, e) |
|
| 34 |
|
|
| 35 |
backend.close() |
|
| b/snf-pithos-app/pithos/api/functions.py | ||
|---|---|---|
| 886 | 886 |
raise RequestEntityTooLarge('Quota exceeded')
|
| 887 | 887 |
if not checksum and UPDATE_MD5: |
| 888 | 888 |
# Update the MD5 after the hashmap, as there may be missing hashes. |
| 889 |
checksum = hashmap_md5(request, hashmap, size) |
|
| 889 |
checksum = hashmap_md5(request.backend, hashmap, size)
|
|
| 890 | 890 |
try: |
| 891 |
version_id = request.backend.update_object_checksum(request.user_uniq,
|
|
| 892 |
v_account, v_container, v_object, version_id, checksum)
|
|
| 891 |
request.backend.update_object_checksum(request.user_uniq, |
|
| 892 |
v_account, v_container, v_object, version_id, checksum) |
|
| 893 | 893 |
except NotAllowedError: |
| 894 | 894 |
raise Forbidden('Not allowed')
|
| 895 | 895 |
if public is not None: |
| ... | ... | |
| 1187 | 1187 |
if dest_bytes is not None and dest_bytes < size: |
| 1188 | 1188 |
size = dest_bytes |
| 1189 | 1189 |
hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)] |
| 1190 |
checksum = hashmap_md5(request, hashmap, size) if UPDATE_MD5 else '' |
|
| 1190 |
checksum = hashmap_md5(request.backend, hashmap, size) if UPDATE_MD5 else ''
|
|
| 1191 | 1191 |
try: |
| 1192 | 1192 |
version_id = request.backend.update_object_hashmap(request.user_uniq, |
| 1193 | 1193 |
v_account, v_container, v_object, size, prev_meta['type'], |
| b/snf-pithos-app/pithos/api/util.py | ||
|---|---|---|
| 761 | 761 |
hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
|
| 762 | 762 |
return bl # Return ammount of data written. |
| 763 | 763 |
|
| 764 |
def hashmap_md5(request, hashmap, size):
|
|
| 764 |
def hashmap_md5(backend, hashmap, size):
|
|
| 765 | 765 |
"""Produce the MD5 sum from the data in the hashmap.""" |
| 766 | 766 |
|
| 767 | 767 |
# TODO: Search backend for the MD5 of another object with the same hashmap and size... |
| 768 | 768 |
md5 = hashlib.md5() |
| 769 |
bs = request.backend.block_size
|
|
| 769 |
bs = backend.block_size |
|
| 770 | 770 |
for bi, hash in enumerate(hashmap): |
| 771 |
data = request.backend.get_block(hash) # Blocks come in padded.
|
|
| 771 |
data = backend.get_block(hash) # Blocks come in padded. |
|
| 772 | 772 |
if bi == len(hashmap) - 1: |
| 773 | 773 |
data = data[:size % bs] |
| 774 | 774 |
md5.update(data) |
Also available in: Unified diff