Revision 7ca7bb08

b/pithos/backends/lib/hashfiler/__init__.py
31 31
# interpreted as representing official policies, either expressed
32 32
# or implied, of GRNET S.A.
33 33

  
34
from blocker import Blocker
35
from mapper import Mapper
34
from store import Store
36 35

  
37
__all__ = ["Blocker", "Mapper"]
36
__all__ = ["Store"]
38 37

  
b/pithos/backends/lib/hashfiler/store.py
1
# Copyright 2011 GRNET S.A. All rights reserved.
2
# 
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
# 
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
# 
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
# 
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
# 
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
34
import os
35

  
36
from blocker import Blocker
37
from mapper import Mapper
38

  
39
class Store(object):
40
    """Store.
41
       Required contstructor parameters: path, block_size, hash_algorithm.
42
    """
43
    
44
    def __init__(self, **params):
45
        path = params['path']
46
        if path and not os.path.exists(path):
47
            os.makedirs(path)
48
        if not os.path.isdir(path):
49
            raise RuntimeError("Cannot open path '%s'" % (path,))
50
        
51
        p = {'blocksize': params['block_size'],
52
             'blockpath': os.path.join(path + '/blocks'),
53
             'hashtype': params['hash_algorithm']}
54
        self.blocker = Blocker(**p)
55
        p = {'mappath': os.path.join(path + '/maps'),
56
             'namelen': self.blocker.hashlen}
57
        self.mapper = Mapper(**p)
58
    
59
    def map_get(hash):
60
        return self.mapper.map_retr(hash)
61
    
62
    def map_put(hash, map):
63
        self.mapper.map_stor(hash, map)
64
    
65
    def block_get(hash):
66
        blocks = self.blocker.block_retr((hash,))
67
        if not blocks:
68
            return None
69
        return blocks[0]
70
    
71
    def block_put(data):
72
        hashes, absent = self.blocker.block_stor((data,))
73
        return hashes[0]
74
    
75
    def block_update(hash, offset, data):
76
        h, e = self.blocker.block_delta(hash, ((offset, data),))
77
        return h
78
    
79
    def block_search(map):
80
        return self.blocker.block_ping(map)
81

  
b/pithos/backends/modular.py
123 123
        __import__(block_module)
124 124
        self.block_module = sys.modules[block_module]
125 125
        
126
        if block_path and not os.path.exists(block_path):
127
            os.makedirs(block_path)
128
        if not os.path.isdir(block_path):
129
            raise RuntimeError("Cannot open path '%s'" % (block_path,))
130
        
131
        params = {'blocksize': self.block_size,
132
                  'blockpath': os.path.join(block_path + '/blocks'),
133
                  'hashtype': self.hash_algorithm}
134
        self.blocker = self.block_module.Blocker(**params)
135
        params = {'mappath': os.path.join(block_path + '/maps'),
136
                  'namelen': self.blocker.hashlen}
137
        self.mapper = self.block_module.Mapper(**params)
126
        params = {'path': block_path,
127
                  'block_size': self.block_size,
128
                  'hash_algorithm': self.hash_algorithm}
129
        self.store = self.block_module.Store(**params)
138 130
    
139 131
    def close(self):
140 132
        self.wrapper.close()
......
525 517
        self._can_read(user, account, container, name)
526 518
        path, node = self._lookup_object(account, container, name)
527 519
        props = self._get_version(node, version)
528
        hashmap = self.mapper.map_retr(binascii.unhexlify(props[self.HASH]))
520
        hashmap = self.store.map_get(binascii.unhexlify(props[self.HASH]))
529 521
        return props[self.SIZE], [binascii.hexlify(x) for x in hashmap]
530 522
    
531 523
    def _update_object_hash(self, user, account, container, name, size, hash, meta={}, replace_meta=False, permissions=None):
......
567 559
            hashmap = [self.put_block('')]
568 560
        map = HashMap(self.block_size, self.hash_algorithm)
569 561
        map.extend([binascii.unhexlify(x) for x in hashmap])
570
        missing = self.blocker.block_ping(map)
562
        missing = self.store.block_search(map)
571 563
        if missing:
572 564
            ie = IndexError()
573 565
            ie.data = [binascii.hexlify(x) for x in missing]
......
575 567
        
576 568
        hash = map.hash()
577 569
        dest_version_id = self._update_object_hash(user, account, container, name, size, binascii.hexlify(hash), meta, replace_meta, permissions)
578
        self.mapper.map_stor(hash, map)
570
        self.store.map_put(hash, map)
579 571
        return dest_version_id
580 572
    
581 573
    def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_meta={}, replace_meta=False, permissions=None, src_version=None):
......
660 652
        """Return a block's data."""
661 653
        
662 654
        logger.debug("get_block: %s", hash)
663
        blocks = self.blocker.block_retr((binascii.unhexlify(hash),))
664
        if not blocks:
655
        block = self.store.block_get(binascii.unhexlify(hash))
656
        if not block:
665 657
            raise NameError('Block does not exist')
666
        return blocks[0]
658
        return block
667 659
    
668 660
    @backend_method(autocommit=0)
669 661
    def put_block(self, data):
670 662
        """Store a block and return the hash."""
671 663
        
672 664
        logger.debug("put_block: %s", len(data))
673
        hashes, absent = self.blocker.block_stor((data,))
674
        return binascii.hexlify(hashes[0])
665
        return binascii.hexlify(self.store.block_put(data))
675 666
    
676 667
    @backend_method(autocommit=0)
677 668
    def update_block(self, hash, data, offset=0):
......
680 671
        logger.debug("update_block: %s %s %s", hash, len(data), offset)
681 672
        if offset == 0 and len(data) == self.block_size:
682 673
            return self.put_block(data)
683
        h, e = self.blocker.block_delta(binascii.unhexlify(hash), ((offset, data),))
674
        h = self.store.block_update(binascii.unhexlify(hash), offset, data)
684 675
        return binascii.hexlify(h)
685 676
    
686 677
    # Path functions.

Also available in: Unified diff