Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-backend / pithos / backends / lib / hashfiler / blocker.py @ 4a7b190f

History | View | Annotate | Download (4.5 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from fileblocker import FileBlocker
35

    
36

    
37
def intersect(a, b):
38
    """ return the intersection of two lists """
39
    return list(set(a) & set(b))
40

    
41

    
42
def union(a, b):
43
    """ return the union of two lists """
44
    return list(set(a) | set(b))
45

    
46

    
47
class Blocker(object):
48
    """Blocker.
49
       Required constructor parameters: blocksize, blockpath, hashtype.
50
       Optional blockpool.
51
    """
52

    
53
    def __init__(self, **params):
54
        self.rblocker = None
55
        try:
56
            if params['blockpool']:
57
                from radosblocker import RadosBlocker
58
                self.rblocker = RadosBlocker(**params)
59
        except KeyError:
60
            pass
61

    
62
        self.fblocker = FileBlocker(**params)
63
        self.hashlen = self.fblocker.hashlen
64
        self.blocksize = params['blocksize']
65

    
66
    def block_hash(self, data):
67
        """Hash a block of data"""
68
        return self.fblocker.block_hash(data)
69

    
70
    def block_ping(self, hashes):
71
        """Check hashes for existence and
72
           return those missing from block storage.
73
        """
74
        r = []
75
        if self.rblocker:
76
            r = self.rblocker.block_ping(hashes)
77
        f = self.fblocker.block_ping(hashes)
78
        return union(r, f)
79

    
80
    def block_retr(self, hashes):
81
        """Retrieve blocks from storage by their hashes."""
82
        return self.fblocker.block_retr(hashes)
83

    
84
    def block_stor(self, blocklist):
85
        """Store a bunch of blocks and return (hashes, missing).
86
           Hashes is a list of the hashes of the blocks,
87
           missing is a list of indices in that list indicating
88
           which blocks were missing from the store.
89
        """
90
        r_missing = []
91
        (hashes, f_missing) = self.fblocker.block_stor(blocklist)
92
        if self.rblocker:
93
            (_, r_missing) = self.rblocker.block_stor(blocklist)
94
        return (hashes, union(r_missing, f_missing))
95

    
96
    def block_delta(self, blkhash, offset, data):
97
        """Construct and store a new block from a given block
98
           and a data 'patch' applied at offset. Return:
99
           (the hash of the new block, if the block already existed)
100
        """
101
        blocksize = self.blocksize
102
        r_hash = None
103
        r_existed = True
104
        (f_hash, f_existed) = self.fblocker.block_delta(blkhash, offset, data)
105
        if self.rblocker:
106
            (r_hash, r_existed) = self.rblocker.block_delta(blkhash, offset,
107
                                                            data)
108
        if not r_hash and not f_hash:
109
            return None, None
110
        if self.rblocker and not r_hash:
111
            block = self.fblocker.block_retr((blkhash,))
112
            if not block:
113
                return None, None
114
            block = block[0]
115
            newblock = block[:offset] + data
116
            if len(newblock) > blocksize:
117
                newblock = newblock[:blocksize]
118
            elif len(newblock) < blocksize:
119
                newblock += block[len(newblock):]
120
            r_hash, r_existed = self.rblocker.block_stor((newblock,))
121

    
122
        return f_hash, 1 if r_existed and f_existed else 0