Revision 7ca7bb08
b/pithos/backends/lib/hashfiler/__init__.py | ||
---|---|---|
31 | 31 |
# interpreted as representing official policies, either expressed |
32 | 32 |
# or implied, of GRNET S.A. |
33 | 33 |
|
34 |
from blocker import Blocker |
|
35 |
from mapper import Mapper |
|
34 |
from store import Store |
|
36 | 35 |
|
37 |
__all__ = ["Blocker", "Mapper"]
|
|
36 |
__all__ = ["Store"]
|
|
38 | 37 |
|
b/pithos/backends/lib/hashfiler/store.py | ||
---|---|---|
1 |
# Copyright 2011 GRNET S.A. All rights reserved. |
|
2 |
# |
|
3 |
# Redistribution and use in source and binary forms, with or |
|
4 |
# without modification, are permitted provided that the following |
|
5 |
# conditions are met: |
|
6 |
# |
|
7 |
# 1. Redistributions of source code must retain the above |
|
8 |
# copyright notice, this list of conditions and the following |
|
9 |
# disclaimer. |
|
10 |
# |
|
11 |
# 2. Redistributions in binary form must reproduce the above |
|
12 |
# copyright notice, this list of conditions and the following |
|
13 |
# disclaimer in the documentation and/or other materials |
|
14 |
# provided with the distribution. |
|
15 |
# |
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
27 |
# POSSIBILITY OF SUCH DAMAGE. |
|
28 |
# |
|
29 |
# The views and conclusions contained in the software and |
|
30 |
# documentation are those of the authors and should not be |
|
31 |
# interpreted as representing official policies, either expressed |
|
32 |
# or implied, of GRNET S.A. |
|
33 |
|
|
34 |
import os |
|
35 |
|
|
36 |
from blocker import Blocker |
|
37 |
from mapper import Mapper |
|
38 |
|
|
39 |
class Store(object): |
|
40 |
"""Store. |
|
41 |
Required contstructor parameters: path, block_size, hash_algorithm. |
|
42 |
""" |
|
43 |
|
|
44 |
def __init__(self, **params): |
|
45 |
path = params['path'] |
|
46 |
if path and not os.path.exists(path): |
|
47 |
os.makedirs(path) |
|
48 |
if not os.path.isdir(path): |
|
49 |
raise RuntimeError("Cannot open path '%s'" % (path,)) |
|
50 |
|
|
51 |
p = {'blocksize': params['block_size'], |
|
52 |
'blockpath': os.path.join(path + '/blocks'), |
|
53 |
'hashtype': params['hash_algorithm']} |
|
54 |
self.blocker = Blocker(**p) |
|
55 |
p = {'mappath': os.path.join(path + '/maps'), |
|
56 |
'namelen': self.blocker.hashlen} |
|
57 |
self.mapper = Mapper(**p) |
|
58 |
|
|
59 |
def map_get(hash): |
|
60 |
return self.mapper.map_retr(hash) |
|
61 |
|
|
62 |
def map_put(hash, map): |
|
63 |
self.mapper.map_stor(hash, map) |
|
64 |
|
|
65 |
def block_get(hash): |
|
66 |
blocks = self.blocker.block_retr((hash,)) |
|
67 |
if not blocks: |
|
68 |
return None |
|
69 |
return blocks[0] |
|
70 |
|
|
71 |
def block_put(data): |
|
72 |
hashes, absent = self.blocker.block_stor((data,)) |
|
73 |
return hashes[0] |
|
74 |
|
|
75 |
def block_update(hash, offset, data): |
|
76 |
h, e = self.blocker.block_delta(hash, ((offset, data),)) |
|
77 |
return h |
|
78 |
|
|
79 |
def block_search(map): |
|
80 |
return self.blocker.block_ping(map) |
|
81 |
|
b/pithos/backends/modular.py | ||
---|---|---|
123 | 123 |
__import__(block_module) |
124 | 124 |
self.block_module = sys.modules[block_module] |
125 | 125 |
|
126 |
if block_path and not os.path.exists(block_path): |
|
127 |
os.makedirs(block_path) |
|
128 |
if not os.path.isdir(block_path): |
|
129 |
raise RuntimeError("Cannot open path '%s'" % (block_path,)) |
|
130 |
|
|
131 |
params = {'blocksize': self.block_size, |
|
132 |
'blockpath': os.path.join(block_path + '/blocks'), |
|
133 |
'hashtype': self.hash_algorithm} |
|
134 |
self.blocker = self.block_module.Blocker(**params) |
|
135 |
params = {'mappath': os.path.join(block_path + '/maps'), |
|
136 |
'namelen': self.blocker.hashlen} |
|
137 |
self.mapper = self.block_module.Mapper(**params) |
|
126 |
params = {'path': block_path, |
|
127 |
'block_size': self.block_size, |
|
128 |
'hash_algorithm': self.hash_algorithm} |
|
129 |
self.store = self.block_module.Store(**params) |
|
138 | 130 |
|
139 | 131 |
def close(self): |
140 | 132 |
self.wrapper.close() |
... | ... | |
525 | 517 |
self._can_read(user, account, container, name) |
526 | 518 |
path, node = self._lookup_object(account, container, name) |
527 | 519 |
props = self._get_version(node, version) |
528 |
hashmap = self.mapper.map_retr(binascii.unhexlify(props[self.HASH]))
|
|
520 |
hashmap = self.store.map_get(binascii.unhexlify(props[self.HASH]))
|
|
529 | 521 |
return props[self.SIZE], [binascii.hexlify(x) for x in hashmap] |
530 | 522 |
|
531 | 523 |
def _update_object_hash(self, user, account, container, name, size, hash, meta={}, replace_meta=False, permissions=None): |
... | ... | |
567 | 559 |
hashmap = [self.put_block('')] |
568 | 560 |
map = HashMap(self.block_size, self.hash_algorithm) |
569 | 561 |
map.extend([binascii.unhexlify(x) for x in hashmap]) |
570 |
missing = self.blocker.block_ping(map)
|
|
562 |
missing = self.store.block_search(map)
|
|
571 | 563 |
if missing: |
572 | 564 |
ie = IndexError() |
573 | 565 |
ie.data = [binascii.hexlify(x) for x in missing] |
... | ... | |
575 | 567 |
|
576 | 568 |
hash = map.hash() |
577 | 569 |
dest_version_id = self._update_object_hash(user, account, container, name, size, binascii.hexlify(hash), meta, replace_meta, permissions) |
578 |
self.mapper.map_stor(hash, map)
|
|
570 |
self.store.map_put(hash, map)
|
|
579 | 571 |
return dest_version_id |
580 | 572 |
|
581 | 573 |
def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_meta={}, replace_meta=False, permissions=None, src_version=None): |
... | ... | |
660 | 652 |
"""Return a block's data.""" |
661 | 653 |
|
662 | 654 |
logger.debug("get_block: %s", hash) |
663 |
blocks = self.blocker.block_retr((binascii.unhexlify(hash),))
|
|
664 |
if not blocks:
|
|
655 |
block = self.store.block_get(binascii.unhexlify(hash))
|
|
656 |
if not block: |
|
665 | 657 |
raise NameError('Block does not exist') |
666 |
return blocks[0]
|
|
658 |
return block |
|
667 | 659 |
|
668 | 660 |
@backend_method(autocommit=0) |
669 | 661 |
def put_block(self, data): |
670 | 662 |
"""Store a block and return the hash.""" |
671 | 663 |
|
672 | 664 |
logger.debug("put_block: %s", len(data)) |
673 |
hashes, absent = self.blocker.block_stor((data,)) |
|
674 |
return binascii.hexlify(hashes[0]) |
|
665 |
return binascii.hexlify(self.store.block_put(data)) |
|
675 | 666 |
|
676 | 667 |
@backend_method(autocommit=0) |
677 | 668 |
def update_block(self, hash, data, offset=0): |
... | ... | |
680 | 671 |
logger.debug("update_block: %s %s %s", hash, len(data), offset) |
681 | 672 |
if offset == 0 and len(data) == self.block_size: |
682 | 673 |
return self.put_block(data) |
683 |
h, e = self.blocker.block_delta(binascii.unhexlify(hash), ((offset, data),))
|
|
674 |
h = self.store.block_update(binascii.unhexlify(hash), offset, data)
|
|
684 | 675 |
return binascii.hexlify(h) |
685 | 676 |
|
686 | 677 |
# Path functions. |
Also available in: Unified diff