Statistics
| Branch: | Tag: | Revision:

root / kamaki / clients / pithos.py @ d9e46d62

History | View | Annotate | Download (4.6 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
import hashlib
35
import os
36

    
37
from time import time
38

    
39
from .storage import StorageClient
40

    
41

    
42
def pithos_hash(block, blockhash):
43
    h = hashlib.new(blockhash)
44
    h.update(block.rstrip('\x00'))
45
    return h.hexdigest()
46

    
47

    
48
class PithosClient(StorageClient):
49
    """GRNet Pithos API client"""
50

    
51
    def purge_container(self, container):
52
        self.assert_account()
53

    
54
        path = '/%s/%s' % (self.account, container)
55
        params = {'until': int(time())}
56
        self.delete(path, params=params, success=204)
57

    
58
    def put_block(self, data, hash):
59
        path = '/%s/%s' % (self.account, self.container)
60
        params = {'update': ''}
61
        headers = {'Content-Type': 'application/octet-stream',
62
                   'Content-Length': str(len(data))}
63
        r = self.post(path, params=params, data=data, headers=headers,
64
                      success=202)
65
        assert r.text.strip() == hash, 'Local hash does not match server'
66
    
67
    def create_object(self, object, f, size=None, hash_cb=None,
68
                      upload_cb=None):
69
        """Create an object by uploading only the missing blocks
70
        
71
        hash_cb is a generator function taking the total number of blocks to
72
        be hashed as an argument. Its next() will be called every time a block
73
        is hashed.
74
        
75
        upload_cb is a generator function with the same properties that is
76
        called every time a block is uploaded.
77
        """
78
        self.assert_container()
79
        
80
        meta = self.get_container_meta(self.container)
81
        blocksize = int(meta['block-size'])
82
        blockhash = meta['block-hash']
83
        
84
        file_size = size if size is not None else os.fstat(f.fileno()).st_size
85
        nblocks = 1 + (file_size - 1) // blocksize
86
        hashes = []
87
        map = {}
88

    
89
        size = 0
90
        
91
        if hash_cb:
92
            hash_gen = hash_cb(nblocks)
93
            hash_gen.next()
94
    
95
        for i in range(nblocks):
96
            block = f.read(blocksize)
97
            bytes = len(block)
98
            hash = pithos_hash(block, blockhash)
99
            hashes.append(hash)
100
            map[hash] = (size, bytes)
101
            size += bytes
102
            if hash_cb:
103
                hash_gen.next()
104
        
105
        assert size == file_size
106
                
107
        path = '/%s/%s/%s' % (self.account, self.container, object)
108
        params = dict(format='json', hashmap='')
109
        hashmap = dict(bytes=size, hashes=hashes)
110
        headers = {'Content-Type': 'application/octet-stream'}
111
        r = self.put(path, params=params, headers=headers, json=hashmap,
112
                     success=(201, 409))
113

    
114
        if r.status_code == 201:
115
            return
116
        
117
        missing = r.json
118
        
119
        if upload_cb:
120
            upload_gen = upload_cb(len(missing))
121
            upload_gen.next()
122

    
123
        for hash in missing:
124
            offset, bytes = map[hash]
125
            f.seek(offset)
126
            data = f.read(bytes)
127
            self.put_block(data, hash)
128
            if upload_cb:
129
                upload_gen.next()
130

    
131
        self.put(path, params=params, headers=headers, json=hashmap,
132
                 success=201)