root / kamaki / clients / pithos.py @ c4922a05
History | View | Annotate | Download (4.2 kB)
1 | d2cea1e2 | Giorgos Verigakis | # Copyright 2011 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | d2cea1e2 | Giorgos Verigakis | #
|
3 | d2cea1e2 | Giorgos Verigakis | # Redistribution and use in source and binary forms, with or
|
4 | d2cea1e2 | Giorgos Verigakis | # without modification, are permitted provided that the following
|
5 | d2cea1e2 | Giorgos Verigakis | # conditions are met:
|
6 | d2cea1e2 | Giorgos Verigakis | #
|
7 | d2cea1e2 | Giorgos Verigakis | # 1. Redistributions of source code must retain the above
|
8 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
9 | d2cea1e2 | Giorgos Verigakis | # disclaimer.
|
10 | d2cea1e2 | Giorgos Verigakis | #
|
11 | d2cea1e2 | Giorgos Verigakis | # 2. Redistributions in binary form must reproduce the above
|
12 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
13 | d2cea1e2 | Giorgos Verigakis | # disclaimer in the documentation and/or other materials
|
14 | d2cea1e2 | Giorgos Verigakis | # provided with the distribution.
|
15 | d2cea1e2 | Giorgos Verigakis | #
|
16 | d2cea1e2 | Giorgos Verigakis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | d2cea1e2 | Giorgos Verigakis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | d2cea1e2 | Giorgos Verigakis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | d2cea1e2 | Giorgos Verigakis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | d2cea1e2 | Giorgos Verigakis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | d2cea1e2 | Giorgos Verigakis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | d2cea1e2 | Giorgos Verigakis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | d2cea1e2 | Giorgos Verigakis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | d2cea1e2 | Giorgos Verigakis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | d2cea1e2 | Giorgos Verigakis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | d2cea1e2 | Giorgos Verigakis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | d2cea1e2 | Giorgos Verigakis | # POSSIBILITY OF SUCH DAMAGE.
|
28 | d2cea1e2 | Giorgos Verigakis | #
|
29 | d2cea1e2 | Giorgos Verigakis | # The views and conclusions contained in the software and
|
30 | d2cea1e2 | Giorgos Verigakis | # documentation are those of the authors and should not be
|
31 | d2cea1e2 | Giorgos Verigakis | # interpreted as representing official policies, either expressed
|
32 | d2cea1e2 | Giorgos Verigakis | # or implied, of GRNET S.A.
|
33 | d2cea1e2 | Giorgos Verigakis | |
34 | d2cea1e2 | Giorgos Verigakis | import hashlib |
35 | 6a0b1658 | Giorgos Verigakis | import os |
36 | d2cea1e2 | Giorgos Verigakis | |
37 | c4ab2af9 | Giorgos Verigakis | from ..utils import OrderedDict |
38 | d2cea1e2 | Giorgos Verigakis | |
39 | 6a0b1658 | Giorgos Verigakis | from .storage import StorageClient |
40 | 6a0b1658 | Giorgos Verigakis | |
41 | 6a0b1658 | Giorgos Verigakis | |
42 | 6a0b1658 | Giorgos Verigakis | def pithos_hash(block, blockhash): |
43 | 6a0b1658 | Giorgos Verigakis | h = hashlib.new(blockhash) |
44 | 6a0b1658 | Giorgos Verigakis | h.update(block.rstrip('\x00'))
|
45 | 6a0b1658 | Giorgos Verigakis | return h.hexdigest()
|
46 | 6a0b1658 | Giorgos Verigakis | |
47 | d2cea1e2 | Giorgos Verigakis | |
48 | d2cea1e2 | Giorgos Verigakis | class PithosClient(StorageClient): |
49 | d2cea1e2 | Giorgos Verigakis | """GRNet Pithos API client"""
|
50 | d2cea1e2 | Giorgos Verigakis | |
51 | 53129af9 | Giorgos Verigakis | def put_block(self, data, hash): |
52 | 6a0b1658 | Giorgos Verigakis | path = '/%s/%s' % (self.account, self.container) |
53 | 6a0b1658 | Giorgos Verigakis | params = {'update': ''} |
54 | 53129af9 | Giorgos Verigakis | headers = {'Content-Type': 'application/octet-stream', |
55 | 6a0b1658 | Giorgos Verigakis | 'Content-Length': str(len(data))} |
56 | 6a0b1658 | Giorgos Verigakis | r = self.post(path, params=params, data=data, headers=headers,
|
57 | 6a0b1658 | Giorgos Verigakis | success=202)
|
58 | 6a0b1658 | Giorgos Verigakis | assert r.text.strip() == hash, 'Local hash does not match server' |
59 | d2cea1e2 | Giorgos Verigakis | |
60 | 6a0b1658 | Giorgos Verigakis | def create_object(self, object, f, hash_cb=None, upload_cb=None): |
61 | 6a0b1658 | Giorgos Verigakis | """Create an object by uploading only the missing blocks
|
62 | 6a0b1658 | Giorgos Verigakis |
|
63 | 6a0b1658 | Giorgos Verigakis | hash_cb is a generator function taking the total number of blocks to
|
64 | 6a0b1658 | Giorgos Verigakis | be hashed as an argument. Its next() will be called every time a block
|
65 | 6a0b1658 | Giorgos Verigakis | is hashed.
|
66 | 6a0b1658 | Giorgos Verigakis |
|
67 | 6a0b1658 | Giorgos Verigakis | upload_cb is a generator function with the same properties that is
|
68 | 6a0b1658 | Giorgos Verigakis | called every time a block is uploaded.
|
69 | 6a0b1658 | Giorgos Verigakis | """
|
70 | 6a0b1658 | Giorgos Verigakis | self.assert_container()
|
71 | 6a0b1658 | Giorgos Verigakis | |
72 | 6a0b1658 | Giorgos Verigakis | meta = self.get_container_meta(self.container) |
73 | d2cea1e2 | Giorgos Verigakis | blocksize = int(meta['block-size']) |
74 | d2cea1e2 | Giorgos Verigakis | blockhash = meta['block-hash']
|
75 | d2cea1e2 | Giorgos Verigakis | |
76 | 6a0b1658 | Giorgos Verigakis | file_size = os.fstat(f.fileno()).st_size |
77 | 6a0b1658 | Giorgos Verigakis | nblocks = 1 + (file_size - 1) // blocksize |
78 | c4ab2af9 | Giorgos Verigakis | hashes = OrderedDict() |
79 | 6a0b1658 | Giorgos Verigakis | |
80 | 6a0b1658 | Giorgos Verigakis | size = 0
|
81 | 6a0b1658 | Giorgos Verigakis | |
82 | 6a0b1658 | Giorgos Verigakis | if hash_cb:
|
83 | 6a0b1658 | Giorgos Verigakis | hash_gen = hash_cb(nblocks) |
84 | 6a0b1658 | Giorgos Verigakis | hash_gen.next() |
85 | 6a0b1658 | Giorgos Verigakis | for i in range(nblocks): |
86 | 6a0b1658 | Giorgos Verigakis | block = f.read(blocksize) |
87 | 6a0b1658 | Giorgos Verigakis | bytes = len(block)
|
88 | 6a0b1658 | Giorgos Verigakis | hash = pithos_hash(block, blockhash) |
89 | 53129af9 | Giorgos Verigakis | hashes[hash] = (size, bytes) |
90 | 53129af9 | Giorgos Verigakis | size += bytes
|
91 | 6a0b1658 | Giorgos Verigakis | if hash_cb:
|
92 | 6a0b1658 | Giorgos Verigakis | hash_gen.next() |
93 | 6a0b1658 | Giorgos Verigakis | |
94 | 6a0b1658 | Giorgos Verigakis | assert size == file_size
|
95 | d2cea1e2 | Giorgos Verigakis | |
96 | 6a0b1658 | Giorgos Verigakis | path = '/%s/%s/%s' % (self.account, self.container, object) |
97 | 2bcb595a | Giorgos Verigakis | params = dict(format='json', hashmap='') |
98 | c4ab2af9 | Giorgos Verigakis | hashmap = dict(bytes=size, hashes=hashes.keys())
|
99 | 6a0b1658 | Giorgos Verigakis | r = self.put(path, params=params, json=hashmap, success=(201, 409)) |
100 | d2cea1e2 | Giorgos Verigakis | |
101 | 6a0b1658 | Giorgos Verigakis | if r.status_code == 201: |
102 | d2cea1e2 | Giorgos Verigakis | return
|
103 | d2cea1e2 | Giorgos Verigakis | |
104 | 6a0b1658 | Giorgos Verigakis | missing = r.json |
105 | d2cea1e2 | Giorgos Verigakis | |
106 | 6a0b1658 | Giorgos Verigakis | if upload_cb:
|
107 | 6a0b1658 | Giorgos Verigakis | upload_gen = upload_cb(len(missing))
|
108 | 6a0b1658 | Giorgos Verigakis | upload_gen.next() |
109 | 53129af9 | Giorgos Verigakis | for hash in missing: |
110 | 53129af9 | Giorgos Verigakis | offset, bytes = hashes[hash]
|
111 | 53129af9 | Giorgos Verigakis | f.seek(offset) |
112 | 53129af9 | Giorgos Verigakis | data = f.read(bytes)
|
113 | 53129af9 | Giorgos Verigakis | self.put_block(data, hash) |
114 | 6a0b1658 | Giorgos Verigakis | if upload_cb:
|
115 | 6a0b1658 | Giorgos Verigakis | upload_gen.next() |
116 | d2cea1e2 | Giorgos Verigakis | |
117 | 6a0b1658 | Giorgos Verigakis | self.put(path, params=params, json=hashmap, success=201) |