From b31550659b64469bf19c768660adb97ed1688b1b Mon Sep 17 00:00:00 2001
From: Antony Chazapis <chazapis@gmail.com>
Date: Tue, 3 Apr 2012 10:48:33 +0300
Subject: [PATCH] Add dispatcher function for updating object checksums
 asynchronously.

Refs #1792
---
 snf-pithos-app/README                  |    4 ++++
 snf-pithos-app/pithos/api/dispatch.py  |   35 ++++++++++++++++++++++++++++++++
 snf-pithos-app/pithos/api/functions.py |    8 ++++----
 snf-pithos-app/pithos/api/util.py      |    6 +++---
 4 files changed, 46 insertions(+), 7 deletions(-)
 create mode 100644 snf-pithos-app/pithos/api/dispatch.py

diff --git a/snf-pithos-app/README b/snf-pithos-app/README
index 2921482..b506e8b 100644
--- a/snf-pithos-app/README
+++ b/snf-pithos-app/README
@@ -43,6 +43,10 @@ PITHOS_BACKEND_VERSIONING        auto
 PITHOS_UPDATE_MD5                True                                              Update object checksums when using hashmaps
 ===============================  ================================================  ============================================================
 
+To update checksums asynchronously, enable the queue, install snf-pithos-tools and use ``pithos-dispatcher``::
+
+    pithos-dispatcher --exchange=pithos --callback=pithos.api.dispatch.update_md5
+
 Administrator functions
 -----------------------
 
diff --git a/snf-pithos-app/pithos/api/dispatch.py b/snf-pithos-app/pithos/api/dispatch.py
new file mode 100644
index 0000000..09761da
--- /dev/null
+++ b/snf-pithos-app/pithos/api/dispatch.py
@@ -0,0 +1,35 @@
+from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
+                                    BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
+                                    BACKEND_QUEUE_MODULE, BACKEND_QUEUE_CONNECTION,
+                                    BACKEND_QUOTA, BACKEND_VERSIONING)
+from pithos.backends import connect_backend
+from pithos.api.util import hashmap_md5
+
+def update_md5(m):
+    if m['resource'] != 'object' or m['details']['action'] != 'object update':
+        return
+    
+    backend = connect_backend(db_module=BACKEND_DB_MODULE,
+                              db_connection=BACKEND_DB_CONNECTION,
+                              block_module=BACKEND_BLOCK_MODULE,
+                              block_path=BACKEND_BLOCK_PATH,
+                              queue_module=BACKEND_QUEUE_MODULE,
+                              queue_connection=BACKEND_QUEUE_CONNECTION)
+    backend.default_policy['quota'] = BACKEND_QUOTA
+    backend.default_policy['versioning'] = BACKEND_VERSIONING
+    
+    path = m['value']
+    account, container, name = path.split('/', 2)
+    version = m['details']['version']
+    meta = None
+    try:
+        meta = backend.get_object_meta(account, account, container, name, 'pithos', version)
+        if meta['checksum'] == '':
+            size, hashmap = backend.get_object_hashmap(account, account, container, name, version)
+            checksum = hashmap_md5(backend, hashmap, size)
+            backend.update_object_checksum(account, account, container, name, version, checksum)
+            print 'INFO: Updated checksum for path "%s"' % (path,)
+    except Exception, e:
+        print 'WARNING: Can not update checksum for path "%s" (%s)' % (path, e)
+    
+    backend.close()
diff --git a/snf-pithos-app/pithos/api/functions.py b/snf-pithos-app/pithos/api/functions.py
index 521b28f..368b9e7 100644
--- a/snf-pithos-app/pithos/api/functions.py
+++ b/snf-pithos-app/pithos/api/functions.py
@@ -886,10 +886,10 @@ def object_write(request, v_account, v_container, v_object):
         raise RequestEntityTooLarge('Quota exceeded')
     if not checksum and UPDATE_MD5:
         # Update the MD5 after the hashmap, as there may be missing hashes.
-        checksum = hashmap_md5(request, hashmap, size)
+        checksum = hashmap_md5(request.backend, hashmap, size)
         try:
-            version_id = request.backend.update_object_checksum(request.user_uniq,
-                            v_account, v_container, v_object, version_id, checksum)
+            request.backend.update_object_checksum(request.user_uniq,
+              v_account, v_container, v_object, version_id, checksum)
         except NotAllowedError:
             raise Forbidden('Not allowed')
     if public is not None:
@@ -1187,7 +1187,7 @@ def object_update(request, v_account, v_container, v_object):
     if dest_bytes is not None and dest_bytes < size:
         size = dest_bytes
         hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)]
-    checksum = hashmap_md5(request, hashmap, size) if UPDATE_MD5 else ''
+    checksum = hashmap_md5(request.backend, hashmap, size) if UPDATE_MD5 else ''
     try:
         version_id = request.backend.update_object_hashmap(request.user_uniq,
                         v_account, v_container, v_object, size, prev_meta['type'],
diff --git a/snf-pithos-app/pithos/api/util.py b/snf-pithos-app/pithos/api/util.py
index 11642a7..4c4513c 100644
--- a/snf-pithos-app/pithos/api/util.py
+++ b/snf-pithos-app/pithos/api/util.py
@@ -761,14 +761,14 @@ def put_object_block(request, hashmap, data, offset):
         hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
     return bl # Return ammount of data written.
 
-def hashmap_md5(request, hashmap, size):
+def hashmap_md5(backend, hashmap, size):
     """Produce the MD5 sum from the data in the hashmap."""
     
     # TODO: Search backend for the MD5 of another object with the same hashmap and size...
     md5 = hashlib.md5()
-    bs = request.backend.block_size
+    bs = backend.block_size
     for bi, hash in enumerate(hashmap):
-        data = request.backend.get_block(hash) # Blocks come in padded.
+        data = backend.get_block(hash) # Blocks come in padded.
         if bi == len(hashmap) - 1:
             data = data[:size % bs]
         md5.update(data)
-- 
1.7.10.4