From: Antony Chazapis <chazapis@gmail.com>
Date: Thu, 16 Feb 2012 16:53:43 +0000 (+0200)
Subject: Blocks always come padded from the backend.
X-Git-Tag: pithos/v0.9.0~4^2~2^2~1
X-Git-Url: https://code.grnet.gr/git/pithos/commitdiff_plain/c9865fe1cc9ed34a66b64b88f5a4caa90732a956

Blocks always come padded from the backend.

Refs #2038
---

diff --git a/pithos/api/util.py b/pithos/api/util.py
index a0b432b..9c1b4b1 100644
--- a/pithos/api/util.py
+++ b/pithos/api/util.py
@@ -610,6 +610,7 @@ class ObjectWrapper(object):
     """
     
     def __init__(self, backend, ranges, sizes, hashmaps, boundary):
+        print '***', ranges, sizes, hashmaps, boundary
         self.backend = backend
         self.ranges = ranges
         self.sizes = sizes
@@ -648,7 +649,10 @@ class ObjectWrapper(object):
             
             # Get the data from the block.
             bo = self.offset % self.backend.block_size
-            bl = min(self.length, len(self.block) - bo)
+            bs = self.backend.block_size
+            if self.block_index == len(self.hashmaps[self.file_index]) - 1:
+                bs = self.sizes[self.file_index] % self.backend.block_size
+            bl = min(self.length, bs - bo)
             data = self.block[bo:bo + bl]
             self.offset += bl
             self.length -= bl
@@ -753,11 +757,10 @@ def hashmap_md5(request, hashmap, size):
     md5 = hashlib.md5()
     bs = request.backend.block_size
     for bi, hash in enumerate(hashmap):
-        data = request.backend.get_block(hash)
+        data = request.backend.get_block(hash) # Blocks come in padded.
         if bi == len(hashmap) - 1:
-            bs = size % bs
-        pad = bs - min(len(data), bs)
-        md5.update(data + ('\x00' * pad))
+            data = data[:size % bs]
+        md5.update(data)
     return md5.hexdigest().lower()
 
 def simple_list_response(request, l):
diff --git a/pithos/backends/lib/hashfiler/blocker.py b/pithos/backends/lib/hashfiler/blocker.py
index 3e98b65..b5ecb61 100644
--- a/pithos/backends/lib/hashfiler/blocker.py
+++ b/pithos/backends/lib/hashfiler/blocker.py
@@ -74,6 +74,9 @@ class Blocker(object):
         self.hashlen = len(emptyhash)
         self.emptyhash = emptyhash
 
+    def _pad(self, block):
+        return block + ('\x00' * (self.blocksize - len(block)))
+
     def _get_rear_block(self, blkhash, create=0):
         filename = hexlify(blkhash)
         dir = join(self.blockpath, filename[0:2], filename[2:4], filename[4:6])
@@ -116,7 +119,7 @@ class Blocker(object):
 
         for h in hashes:
             if h == self.emptyhash:
-                append('')
+                append(self._pad(''))
                 continue
             with self._get_rear_block(h, 0) as rbl:
                 if not rbl:
@@ -125,7 +128,7 @@ class Blocker(object):
                     break # there should be just one block there
             if not block:
                 break
-            append(block)
+            append(self._pad(block))
 
         return blocks
 
@@ -145,36 +148,24 @@ class Blocker(object):
 
         return hashlist, missing
 
-    def block_delta(self, blkhash, offdata=()):
+    def block_delta(self, blkhash, offset, data):
         """Construct and store a new block from a given block
-           and a list of (offset, data) 'patches'. Return:
+           and a data 'patch' applied at offset. Return:
            (the hash of the new block, if the block already existed)
         """
-        if not offdata:
-            return None, None
 
         blocksize = self.blocksize
+        if offset >= blocksize or not data:
+            return None, None
+
         block = self.block_retr((blkhash,))
         if not block:
             return None, None
-
+        
         block = block[0]
-        newblock = ''
-        idx = 0
-        size = 0
-        trunc = 0
-        for off, data in offdata:
-            if not data:
-                trunc = 1
-                break
-            newblock += block[idx:off] + data
-            size += off - idx + len(data)
-            if size >= blocksize:
-                break
-            off = size
-
-        if not trunc:
-            newblock += block[size:len(block)]
+        newblock = block[:offset] + data
+        if len(block) >= blocksize:
+            newblock = newblock[:blocksize]
 
         h, a = self.block_stor((newblock,))
         return h[0], 1 if a else 0
diff --git a/pithos/backends/lib/hashfiler/store.py b/pithos/backends/lib/hashfiler/store.py
index afd5d87..54cd6f3 100644
--- a/pithos/backends/lib/hashfiler/store.py
+++ b/pithos/backends/lib/hashfiler/store.py
@@ -76,7 +76,7 @@ class Store(object):
         return hashes[0]
     
     def block_update(self, hash, offset, data):
-        h, e = self.blocker.block_delta(hash, ((offset, data),))
+        h, e = self.blocker.block_delta(hash, offset, data)
         return h
     
     def block_search(self, map):