root / kamaki / clients / pithos.py @ 466c1ef7
History | View | Annotate | Download (4.5 kB)
1 | 0c6d7489 | Giorgos Verigakis | # Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | d2cea1e2 | Giorgos Verigakis | #
|
3 | d2cea1e2 | Giorgos Verigakis | # Redistribution and use in source and binary forms, with or
|
4 | d2cea1e2 | Giorgos Verigakis | # without modification, are permitted provided that the following
|
5 | d2cea1e2 | Giorgos Verigakis | # conditions are met:
|
6 | d2cea1e2 | Giorgos Verigakis | #
|
7 | d2cea1e2 | Giorgos Verigakis | # 1. Redistributions of source code must retain the above
|
8 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
9 | d2cea1e2 | Giorgos Verigakis | # disclaimer.
|
10 | d2cea1e2 | Giorgos Verigakis | #
|
11 | d2cea1e2 | Giorgos Verigakis | # 2. Redistributions in binary form must reproduce the above
|
12 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
13 | d2cea1e2 | Giorgos Verigakis | # disclaimer in the documentation and/or other materials
|
14 | d2cea1e2 | Giorgos Verigakis | # provided with the distribution.
|
15 | d2cea1e2 | Giorgos Verigakis | #
|
16 | d2cea1e2 | Giorgos Verigakis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | d2cea1e2 | Giorgos Verigakis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | d2cea1e2 | Giorgos Verigakis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | d2cea1e2 | Giorgos Verigakis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | d2cea1e2 | Giorgos Verigakis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | d2cea1e2 | Giorgos Verigakis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | d2cea1e2 | Giorgos Verigakis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | d2cea1e2 | Giorgos Verigakis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | d2cea1e2 | Giorgos Verigakis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | d2cea1e2 | Giorgos Verigakis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | d2cea1e2 | Giorgos Verigakis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | d2cea1e2 | Giorgos Verigakis | # POSSIBILITY OF SUCH DAMAGE.
|
28 | d2cea1e2 | Giorgos Verigakis | #
|
29 | d2cea1e2 | Giorgos Verigakis | # The views and conclusions contained in the software and
|
30 | d2cea1e2 | Giorgos Verigakis | # documentation are those of the authors and should not be
|
31 | d2cea1e2 | Giorgos Verigakis | # interpreted as representing official policies, either expressed
|
32 | d2cea1e2 | Giorgos Verigakis | # or implied, of GRNET S.A.
|
33 | d2cea1e2 | Giorgos Verigakis | |
34 | d2cea1e2 | Giorgos Verigakis | import hashlib |
35 | 6a0b1658 | Giorgos Verigakis | import os |
36 | d2cea1e2 | Giorgos Verigakis | |
37 | a91e0293 | Giorgos Verigakis | from time import time |
38 | a91e0293 | Giorgos Verigakis | |
39 | 6a0b1658 | Giorgos Verigakis | from .storage import StorageClient |
40 | 6a0b1658 | Giorgos Verigakis | |
41 | 6a0b1658 | Giorgos Verigakis | |
42 | 6a0b1658 | Giorgos Verigakis | def pithos_hash(block, blockhash): |
43 | 6a0b1658 | Giorgos Verigakis | h = hashlib.new(blockhash) |
44 | 6a0b1658 | Giorgos Verigakis | h.update(block.rstrip('\x00'))
|
45 | 6a0b1658 | Giorgos Verigakis | return h.hexdigest()
|
46 | 6a0b1658 | Giorgos Verigakis | |
47 | d2cea1e2 | Giorgos Verigakis | |
48 | d2cea1e2 | Giorgos Verigakis | class PithosClient(StorageClient): |
49 | d2cea1e2 | Giorgos Verigakis | """GRNet Pithos API client"""
|
50 | a91e0293 | Giorgos Verigakis | |
51 | a91e0293 | Giorgos Verigakis | def purge_container(self, container): |
52 | a91e0293 | Giorgos Verigakis | self.assert_account()
|
53 | a91e0293 | Giorgos Verigakis | |
54 | a91e0293 | Giorgos Verigakis | path = '/%s/%s' % (self.account, container) |
55 | a91e0293 | Giorgos Verigakis | params = {'until': int(time())} |
56 | a91e0293 | Giorgos Verigakis | self.delete(path, params=params, success=204) |
57 | a91e0293 | Giorgos Verigakis | |
58 | 53129af9 | Giorgos Verigakis | def put_block(self, data, hash): |
59 | 6a0b1658 | Giorgos Verigakis | path = '/%s/%s' % (self.account, self.container) |
60 | 6a0b1658 | Giorgos Verigakis | params = {'update': ''} |
61 | 53129af9 | Giorgos Verigakis | headers = {'Content-Type': 'application/octet-stream', |
62 | 6a0b1658 | Giorgos Verigakis | 'Content-Length': str(len(data))} |
63 | 6a0b1658 | Giorgos Verigakis | r = self.post(path, params=params, data=data, headers=headers,
|
64 | 6a0b1658 | Giorgos Verigakis | success=202)
|
65 | 6a0b1658 | Giorgos Verigakis | assert r.text.strip() == hash, 'Local hash does not match server' |
66 | d2cea1e2 | Giorgos Verigakis | |
67 | 188f23b9 | Giorgos Verigakis | def create_object(self, object, f, size=None, hash_cb=None, |
68 | 188f23b9 | Giorgos Verigakis | upload_cb=None):
|
69 | 6a0b1658 | Giorgos Verigakis | """Create an object by uploading only the missing blocks
|
70 | 6a0b1658 | Giorgos Verigakis |
|
71 | 6a0b1658 | Giorgos Verigakis | hash_cb is a generator function taking the total number of blocks to
|
72 | 6a0b1658 | Giorgos Verigakis | be hashed as an argument. Its next() will be called every time a block
|
73 | 6a0b1658 | Giorgos Verigakis | is hashed.
|
74 | 6a0b1658 | Giorgos Verigakis |
|
75 | 6a0b1658 | Giorgos Verigakis | upload_cb is a generator function with the same properties that is
|
76 | 6a0b1658 | Giorgos Verigakis | called every time a block is uploaded.
|
77 | 6a0b1658 | Giorgos Verigakis | """
|
78 | 6a0b1658 | Giorgos Verigakis | self.assert_container()
|
79 | 6a0b1658 | Giorgos Verigakis | |
80 | 6a0b1658 | Giorgos Verigakis | meta = self.get_container_meta(self.container) |
81 | d2cea1e2 | Giorgos Verigakis | blocksize = int(meta['block-size']) |
82 | d2cea1e2 | Giorgos Verigakis | blockhash = meta['block-hash']
|
83 | d2cea1e2 | Giorgos Verigakis | |
84 | 188f23b9 | Giorgos Verigakis | file_size = size if size is not None else os.fstat(f.fileno()).st_size |
85 | 6a0b1658 | Giorgos Verigakis | nblocks = 1 + (file_size - 1) // blocksize |
86 | 0c6d7489 | Giorgos Verigakis | hashes = [] |
87 | 0c6d7489 | Giorgos Verigakis | map = {} |
88 | 0c6d7489 | Giorgos Verigakis | |
89 | 6a0b1658 | Giorgos Verigakis | size = 0
|
90 | 6a0b1658 | Giorgos Verigakis | |
91 | 6a0b1658 | Giorgos Verigakis | if hash_cb:
|
92 | 6a0b1658 | Giorgos Verigakis | hash_gen = hash_cb(nblocks) |
93 | 6a0b1658 | Giorgos Verigakis | hash_gen.next() |
94 | 0c6d7489 | Giorgos Verigakis | |
95 | 6a0b1658 | Giorgos Verigakis | for i in range(nblocks): |
96 | 6a0b1658 | Giorgos Verigakis | block = f.read(blocksize) |
97 | 6a0b1658 | Giorgos Verigakis | bytes = len(block)
|
98 | 6a0b1658 | Giorgos Verigakis | hash = pithos_hash(block, blockhash) |
99 | 0c6d7489 | Giorgos Verigakis | hashes.append(hash)
|
100 | 0c6d7489 | Giorgos Verigakis | map[hash] = (size, bytes) |
101 | 53129af9 | Giorgos Verigakis | size += bytes
|
102 | 6a0b1658 | Giorgos Verigakis | if hash_cb:
|
103 | 6a0b1658 | Giorgos Verigakis | hash_gen.next() |
104 | 6a0b1658 | Giorgos Verigakis | |
105 | 6a0b1658 | Giorgos Verigakis | assert size == file_size
|
106 | d2cea1e2 | Giorgos Verigakis | |
107 | 6a0b1658 | Giorgos Verigakis | path = '/%s/%s/%s' % (self.account, self.container, object) |
108 | 2bcb595a | Giorgos Verigakis | params = dict(format='json', hashmap='') |
109 | 0c6d7489 | Giorgos Verigakis | hashmap = dict(bytes=size, hashes=hashes)
|
110 | 466c1ef7 | Giorgos Verigakis | headers = {'Content-Type': 'application/octet-stream'} |
111 | 466c1ef7 | Giorgos Verigakis | r = self.put(path, params=params, headers=headers, json=hashmap,
|
112 | 466c1ef7 | Giorgos Verigakis | success=(201, 409)) |
113 | 466c1ef7 | Giorgos Verigakis | |
114 | 6a0b1658 | Giorgos Verigakis | if r.status_code == 201: |
115 | d2cea1e2 | Giorgos Verigakis | return
|
116 | d2cea1e2 | Giorgos Verigakis | |
117 | 6a0b1658 | Giorgos Verigakis | missing = r.json |
118 | d2cea1e2 | Giorgos Verigakis | |
119 | 6a0b1658 | Giorgos Verigakis | if upload_cb:
|
120 | 6a0b1658 | Giorgos Verigakis | upload_gen = upload_cb(len(missing))
|
121 | 6a0b1658 | Giorgos Verigakis | upload_gen.next() |
122 | 0c6d7489 | Giorgos Verigakis | |
123 | 53129af9 | Giorgos Verigakis | for hash in missing: |
124 | 0c6d7489 | Giorgos Verigakis | offset, bytes = map[hash] |
125 | 53129af9 | Giorgos Verigakis | f.seek(offset) |
126 | 53129af9 | Giorgos Verigakis | data = f.read(bytes)
|
127 | 53129af9 | Giorgos Verigakis | self.put_block(data, hash) |
128 | 6a0b1658 | Giorgos Verigakis | if upload_cb:
|
129 | 6a0b1658 | Giorgos Verigakis | upload_gen.next() |
130 | 0c6d7489 | Giorgos Verigakis | |
131 | 6a0b1658 | Giorgos Verigakis | self.put(path, params=params, json=hashmap, success=201) |