Statistics
| Branch: | Tag: | Revision:

root / kamaki / clients / pithos.py @ 2bcb595a

History | View | Annotate | Download (4.2 kB)

1
# Copyright 2011 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
import hashlib
35
import os
36

    
37
from ..utils import OrderedDict
38

    
39
from .storage import StorageClient
40

    
41

    
42
def pithos_hash(block, blockhash):
43
    h = hashlib.new(blockhash)
44
    h.update(block.rstrip('\x00'))
45
    return h.hexdigest()
46

    
47

    
48
class PithosClient(StorageClient):
49
    """GRNet Pithos API client"""
50
    
51
    def put_block(self, data, hash):
52
        path = '/%s/%s' % (self.account, self.container)
53
        params = {'update': ''}
54
        headers = {'Content-Type': 'application/octet-stream',
55
                   'Content-Length': str(len(data))}
56
        r = self.post(path, params=params, data=data, headers=headers,
57
                      success=202)
58
        assert r.text.strip() == hash, 'Local hash does not match server'
59
    
60
    def create_object(self, object, f, hash_cb=None, upload_cb=None):
61
        """Create an object by uploading only the missing blocks
62
        
63
        hash_cb is a generator function taking the total number of blocks to
64
        be hashed as an argument. Its next() will be called every time a block
65
        is hashed.
66
        
67
        upload_cb is a generator function with the same properties that is
68
        called every time a block is uploaded.
69
        """
70
        self.assert_container()
71
        
72
        meta = self.get_container_meta(self.container)
73
        blocksize = int(meta['block-size'])
74
        blockhash = meta['block-hash']
75
        
76
        file_size = os.fstat(f.fileno()).st_size
77
        nblocks = 1 + (file_size - 1) // blocksize
78
        hashes = OrderedDict()
79
        
80
        size = 0
81
        
82
        if hash_cb:
83
            hash_gen = hash_cb(nblocks)
84
            hash_gen.next()
85
        for i in range(nblocks):
86
            block = f.read(blocksize)
87
            bytes = len(block)
88
            hash = pithos_hash(block, blockhash)
89
            hashes[hash] = (size, bytes)
90
            size += bytes
91
            if hash_cb:
92
                hash_gen.next()
93
        
94
        assert size == file_size
95
                
96
        path = '/%s/%s/%s' % (self.account, self.container, object)
97
        params = dict(format='json', hashmap='')
98
        hashmap = dict(bytes=size, hashes=hashes.keys())
99
        r = self.put(path, params=params, json=hashmap, success=(201, 409))
100
        
101
        if r.status_code == 201:
102
            return
103
        
104
        missing = r.json
105
        
106
        if upload_cb:
107
            upload_gen = upload_cb(len(missing))
108
            upload_gen.next()
109
        for hash in missing:
110
            offset, bytes = hashes[hash]
111
            f.seek(offset)
112
            data = f.read(bytes)
113
            self.put_block(data, hash)
114
            if upload_cb:
115
                upload_gen.next()
116
        
117
        self.put(path, params=params, json=hashmap, success=201)