Revision a1557c9c snf-pithos-backend/pithos/backends/lib/hashfiler/blocker.py
| b/snf-pithos-backend/pithos/backends/lib/hashfiler/blocker.py | ||
|---|---|---|
| 31 | 31 |
# interpreted as representing official policies, either expressed |
| 32 | 32 |
# or implied, of GRNET S.A. |
| 33 | 33 |
|
| 34 |
from fileblocker import FileBlocker |
|
| 35 |
|
|
| 36 |
|
|
| 37 |
def intersect(a, b): |
|
| 38 |
""" return the intersection of two lists """ |
|
| 39 |
return list(set(a) & set(b)) |
|
| 40 |
|
|
| 41 |
|
|
| 42 |
def union(a, b): |
|
| 43 |
""" return the union of two lists """ |
|
| 44 |
return list(set(a) | set(b)) |
|
| 34 |
from archipelagoblocker import ArchipelagoBlocker |
|
| 45 | 35 |
|
| 46 | 36 |
|
| 47 | 37 |
class Blocker(object): |
| ... | ... | |
| 51 | 41 |
""" |
| 52 | 42 |
|
| 53 | 43 |
def __init__(self, **params): |
| 54 |
self.rblocker = None |
|
| 55 |
try: |
|
| 56 |
if params['blockpool']: |
|
| 57 |
from radosblocker import RadosBlocker |
|
| 58 |
self.rblocker = RadosBlocker(**params) |
|
| 59 |
except KeyError: |
|
| 60 |
pass |
|
| 61 |
|
|
| 62 |
self.fblocker = FileBlocker(**params) |
|
| 63 |
self.hashlen = self.fblocker.hashlen |
|
| 44 |
self.archip_blocker = ArchipelagoBlocker(**params) |
|
| 45 |
self.hashlen = self.archip_blocker.hashlen |
|
| 64 | 46 |
self.blocksize = params['blocksize'] |
| 65 | 47 |
|
| 66 | 48 |
def block_hash(self, data): |
| 67 | 49 |
"""Hash a block of data""" |
| 68 |
return self.fblocker.block_hash(data)
|
|
| 50 |
return self.archip_blocker.block_hash(data)
|
|
| 69 | 51 |
|
| 70 | 52 |
def block_ping(self, hashes): |
| 71 | 53 |
"""Check hashes for existence and |
| 72 | 54 |
return those missing from block storage. |
| 73 | 55 |
""" |
| 74 |
r = [] |
|
| 75 |
if self.rblocker: |
|
| 76 |
r = self.rblocker.block_ping(hashes) |
|
| 77 |
f = self.fblocker.block_ping(hashes) |
|
| 78 |
return union(r, f) |
|
| 56 |
return self.archip_blocker.block_ping(hashes) |
|
| 79 | 57 |
|
| 80 | 58 |
def block_retr(self, hashes): |
| 81 | 59 |
"""Retrieve blocks from storage by their hashes.""" |
| 82 |
return self.fblocker.block_retr(hashes) |
|
| 60 |
return self.archip_blocker.block_retr(hashes) |
|
| 61 |
|
|
| 62 |
def block_retr_archipelago(self, hashes): |
|
| 63 |
"""Retrieve blocks from storage by theri hashes.""" |
|
| 64 |
return self.archip_blocker.block_retr_archipelago(hashes) |
|
| 83 | 65 |
|
| 84 | 66 |
def block_stor(self, blocklist): |
| 85 | 67 |
"""Store a bunch of blocks and return (hashes, missing). |
| ... | ... | |
| 87 | 69 |
missing is a list of indices in that list indicating |
| 88 | 70 |
which blocks were missing from the store. |
| 89 | 71 |
""" |
| 90 |
r_missing = [] |
|
| 91 |
(hashes, f_missing) = self.fblocker.block_stor(blocklist) |
|
| 92 |
if self.rblocker: |
|
| 93 |
(_, r_missing) = self.rblocker.block_stor(blocklist) |
|
| 94 |
return (hashes, union(r_missing, f_missing)) |
|
| 72 |
(hashes, missing) = self.archip_blocker.block_stor(blocklist) |
|
| 73 |
return (hashes, missing) |
|
| 95 | 74 |
|
| 96 | 75 |
def block_delta(self, blkhash, offset, data): |
| 97 | 76 |
"""Construct and store a new block from a given block |
| ... | ... | |
| 99 | 78 |
(the hash of the new block, if the block already existed) |
| 100 | 79 |
""" |
| 101 | 80 |
blocksize = self.blocksize |
| 102 |
r_hash = None |
|
| 103 |
r_existed = True |
|
| 104 |
(f_hash, f_existed) = self.fblocker.block_delta(blkhash, offset, data) |
|
| 105 |
if self.rblocker: |
|
| 106 |
(r_hash, r_existed) = self.rblocker.block_delta(blkhash, offset, |
|
| 107 |
data) |
|
| 108 |
if not r_hash and not f_hash: |
|
| 81 |
archip_hash = None |
|
| 82 |
archip_existed = True |
|
| 83 |
(archip_hash, archip_existed) = \ |
|
| 84 |
self.archip_blocker.block_delta(blkhash, offset, data) |
|
| 85 |
|
|
| 86 |
if not archip_hash: |
|
| 109 | 87 |
return None, None |
| 110 |
if self.rblocker and not r_hash: |
|
| 111 |
block = self.fblocker.block_retr((blkhash,)) |
|
| 88 |
|
|
| 89 |
if self.archip_blocker and not archip_hash: |
|
| 90 |
block = self.archip_blocker.block_retr((blkhash,)) |
|
| 112 | 91 |
if not block: |
| 113 | 92 |
return None, None |
| 114 | 93 |
block = block[0] |
| ... | ... | |
| 117 | 96 |
newblock = newblock[:blocksize] |
| 118 | 97 |
elif len(newblock) < blocksize: |
| 119 | 98 |
newblock += block[len(newblock):] |
| 120 |
r_hash, r_existed = self.rblocker.block_stor((newblock,))
|
|
| 99 |
archip_hash, archip_existed = self.rblocker.block_stor((newblock,))
|
|
| 121 | 100 |
|
| 122 |
return f_hash, 1 if r_existed and f_existed else 0 |
|
| 101 |
return archip_hash, 1 if archip_existed else 0 |
|
Also available in: Unified diff