root / kamaki / clients / pithos / __init__.py @ c6cec5e1
History | View | Annotate | Download (36.5 kB)
1 | fce31e83 | Stavros Sachtouris | # Copyright 2011-2013 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | d2cea1e2 | Giorgos Verigakis | #
|
3 | d2cea1e2 | Giorgos Verigakis | # Redistribution and use in source and binary forms, with or
|
4 | d2cea1e2 | Giorgos Verigakis | # without modification, are permitted provided that the following
|
5 | d2cea1e2 | Giorgos Verigakis | # conditions are met:
|
6 | d2cea1e2 | Giorgos Verigakis | #
|
7 | d2cea1e2 | Giorgos Verigakis | # 1. Redistributions of source code must retain the above
|
8 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
9 | d2cea1e2 | Giorgos Verigakis | # disclaimer.
|
10 | d2cea1e2 | Giorgos Verigakis | #
|
11 | d2cea1e2 | Giorgos Verigakis | # 2. Redistributions in binary form must reproduce the above
|
12 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
13 | d2cea1e2 | Giorgos Verigakis | # disclaimer in the documentation and/or other materials
|
14 | d2cea1e2 | Giorgos Verigakis | # provided with the distribution.
|
15 | d2cea1e2 | Giorgos Verigakis | #
|
16 | d2cea1e2 | Giorgos Verigakis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | d2cea1e2 | Giorgos Verigakis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | d2cea1e2 | Giorgos Verigakis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | d2cea1e2 | Giorgos Verigakis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | d2cea1e2 | Giorgos Verigakis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | d2cea1e2 | Giorgos Verigakis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | d2cea1e2 | Giorgos Verigakis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | d2cea1e2 | Giorgos Verigakis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | d2cea1e2 | Giorgos Verigakis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | d2cea1e2 | Giorgos Verigakis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | d2cea1e2 | Giorgos Verigakis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | d2cea1e2 | Giorgos Verigakis | # POSSIBILITY OF SUCH DAMAGE.
|
28 | d2cea1e2 | Giorgos Verigakis | #
|
29 | d2cea1e2 | Giorgos Verigakis | # The views and conclusions contained in the software and
|
30 | d2cea1e2 | Giorgos Verigakis | # documentation are those of the authors and should not be
|
31 | d2cea1e2 | Giorgos Verigakis | # interpreted as representing official policies, either expressed
|
32 | d2cea1e2 | Giorgos Verigakis | # or implied, of GRNET S.A.
|
33 | d2cea1e2 | Giorgos Verigakis | |
34 | f27ed9a0 | Stavros Sachtouris | from threading import enumerate as activethreads |
35 | 435008b6 | Stavros Sachtouris | |
36 | 3dabe5d2 | Stavros Sachtouris | from os import fstat |
37 | 64ab4c13 | Stavros Sachtouris | from hashlib import new as newhashlib |
38 | e02728f9 | Stavros Sachtouris | from time import time |
39 | a91e0293 | Giorgos Verigakis | |
40 | 64ab4c13 | Stavros Sachtouris | from binascii import hexlify |
41 | 6a0b1658 | Giorgos Verigakis | |
42 | 6069b53b | Stavros Sachtouris | from kamaki.clients import SilentEvent, sendlog |
43 | 55faa0bc | Stavros Sachtouris | from kamaki.clients.pithos.rest_api import PithosRestClient |
44 | c270fe96 | Stavros Sachtouris | from kamaki.clients.storage import ClientError |
45 | c270fe96 | Stavros Sachtouris | from kamaki.clients.utils import path4url, filter_in |
46 | 1785ad41 | Stavros Sachtouris | from StringIO import StringIO |
47 | 6a0b1658 | Giorgos Verigakis | |
48 | 3dabe5d2 | Stavros Sachtouris | |
49 | 4375e020 | Stavros Sachtouris | def _pithos_hash(block, blockhash): |
50 | 64ab4c13 | Stavros Sachtouris | h = newhashlib(blockhash) |
51 | 6a0b1658 | Giorgos Verigakis | h.update(block.rstrip('\x00'))
|
52 | 6a0b1658 | Giorgos Verigakis | return h.hexdigest()
|
53 | 6a0b1658 | Giorgos Verigakis | |
54 | 3dabe5d2 | Stavros Sachtouris | |
55 | 642f1bbd | Stavros Sachtouris | def _range_up(start, end, a_range): |
56 | 642f1bbd | Stavros Sachtouris | if a_range:
|
57 | 642f1bbd | Stavros Sachtouris | (rstart, rend) = a_range.split('-')
|
58 | 642f1bbd | Stavros Sachtouris | (rstart, rend) = (int(rstart), int(rend)) |
59 | 642f1bbd | Stavros Sachtouris | if rstart > end or rend < start: |
60 | 3dabe5d2 | Stavros Sachtouris | return (0, 0) |
61 | 642f1bbd | Stavros Sachtouris | if rstart > start:
|
62 | 642f1bbd | Stavros Sachtouris | start = rstart |
63 | 642f1bbd | Stavros Sachtouris | if rend < end:
|
64 | 642f1bbd | Stavros Sachtouris | end = rend |
65 | 642f1bbd | Stavros Sachtouris | return (start, end)
|
66 | 642f1bbd | Stavros Sachtouris | |
67 | 3dabe5d2 | Stavros Sachtouris | |
68 | 55faa0bc | Stavros Sachtouris | class PithosClient(PithosRestClient): |
69 | 76e7661e | Stavros Sachtouris | """Synnefo Pithos+ API client"""
|
70 | a91e0293 | Giorgos Verigakis | |
71 | 3dabe5d2 | Stavros Sachtouris | def __init__(self, base_url, token, account=None, container=None): |
72 | 3dabe5d2 | Stavros Sachtouris | super(PithosClient, self).__init__(base_url, token, account, container) |
73 | 435008b6 | Stavros Sachtouris | |
74 | 2a7292f1 | Stavros Sachtouris | def purge_container(self, container=None): |
75 | 4375e020 | Stavros Sachtouris | """Delete an empty container and destroy associated blocks
|
76 | 4375e020 | Stavros Sachtouris | """
|
77 | 2a7292f1 | Stavros Sachtouris | cnt_back_up = self.container
|
78 | 2a7292f1 | Stavros Sachtouris | try:
|
79 | 2a7292f1 | Stavros Sachtouris | self.container = container or cnt_back_up |
80 | c2b5da2f | Stavros Sachtouris | self.container_delete(until=unicode(time())) |
81 | 2a7292f1 | Stavros Sachtouris | finally:
|
82 | 2a7292f1 | Stavros Sachtouris | self.container = cnt_back_up
|
83 | 3dabe5d2 | Stavros Sachtouris | |
84 | 24ff0a35 | Stavros Sachtouris | def upload_object_unchunked( |
85 | 24ff0a35 | Stavros Sachtouris | self, obj, f,
|
86 | 24ff0a35 | Stavros Sachtouris | withHashFile=False,
|
87 | 24ff0a35 | Stavros Sachtouris | size=None,
|
88 | 24ff0a35 | Stavros Sachtouris | etag=None,
|
89 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
90 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
91 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
92 | 24ff0a35 | Stavros Sachtouris | sharing=None,
|
93 | 24ff0a35 | Stavros Sachtouris | public=None):
|
94 | 4375e020 | Stavros Sachtouris | """
|
95 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
96 | 4375e020 | Stavros Sachtouris |
|
97 | 4375e020 | Stavros Sachtouris | :param f: open file descriptor
|
98 | 4375e020 | Stavros Sachtouris |
|
99 | 4375e020 | Stavros Sachtouris | :param withHashFile: (bool)
|
100 | 4375e020 | Stavros Sachtouris |
|
101 | 4375e020 | Stavros Sachtouris | :param size: (int) size of data to upload
|
102 | 4375e020 | Stavros Sachtouris |
|
103 | 4375e020 | Stavros Sachtouris | :param etag: (str)
|
104 | 4375e020 | Stavros Sachtouris |
|
105 | 4375e020 | Stavros Sachtouris | :param content_encoding: (str)
|
106 | 4375e020 | Stavros Sachtouris |
|
107 | 4375e020 | Stavros Sachtouris | :param content_disposition: (str)
|
108 | 4375e020 | Stavros Sachtouris |
|
109 | 4375e020 | Stavros Sachtouris | :param content_type: (str)
|
110 | 4375e020 | Stavros Sachtouris |
|
111 | 4375e020 | Stavros Sachtouris | :param sharing: {'read':[user and/or grp names],
|
112 | 4375e020 | Stavros Sachtouris | 'write':[usr and/or grp names]}
|
113 | 4375e020 | Stavros Sachtouris |
|
114 | 4375e020 | Stavros Sachtouris | :param public: (bool)
|
115 | 4375e020 | Stavros Sachtouris | """
|
116 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
117 | 65a45524 | Stavros Sachtouris | |
118 | 65a45524 | Stavros Sachtouris | if withHashFile:
|
119 | 65a45524 | Stavros Sachtouris | data = f.read() |
120 | 65a45524 | Stavros Sachtouris | try:
|
121 | 65a45524 | Stavros Sachtouris | import json |
122 | 65a45524 | Stavros Sachtouris | data = json.dumps(json.loads(data)) |
123 | 65a45524 | Stavros Sachtouris | except ValueError: |
124 | 24ff0a35 | Stavros Sachtouris | raise ClientError('"%s" is not json-formated' % f.name, 1) |
125 | 65a45524 | Stavros Sachtouris | except SyntaxError: |
126 | 24ff0a35 | Stavros Sachtouris | msg = '"%s" is not a valid hashmap file' % f.name
|
127 | 24ff0a35 | Stavros Sachtouris | raise ClientError(msg, 1) |
128 | 65a45524 | Stavros Sachtouris | f = StringIO(data) |
129 | 2a7292f1 | Stavros Sachtouris | else:
|
130 | 975270f2 | Stavros Sachtouris | data = f.read(size) if size else f.read() |
131 | c2b5da2f | Stavros Sachtouris | self.object_put(
|
132 | 24ff0a35 | Stavros Sachtouris | obj, |
133 | 3dabe5d2 | Stavros Sachtouris | data=data, |
134 | 3dabe5d2 | Stavros Sachtouris | etag=etag, |
135 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
136 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
137 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
138 | 3dabe5d2 | Stavros Sachtouris | permissions=sharing, |
139 | 3dabe5d2 | Stavros Sachtouris | public=public, |
140 | 3dabe5d2 | Stavros Sachtouris | success=201)
|
141 | 3dabe5d2 | Stavros Sachtouris | |
142 | 24ff0a35 | Stavros Sachtouris | def create_object_by_manifestation( |
143 | 24ff0a35 | Stavros Sachtouris | self, obj,
|
144 | 24ff0a35 | Stavros Sachtouris | etag=None,
|
145 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
146 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
147 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
148 | 24ff0a35 | Stavros Sachtouris | sharing=None,
|
149 | 24ff0a35 | Stavros Sachtouris | public=None):
|
150 | 4375e020 | Stavros Sachtouris | """
|
151 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
152 | 4375e020 | Stavros Sachtouris |
|
153 | 4375e020 | Stavros Sachtouris | :param etag: (str)
|
154 | 4375e020 | Stavros Sachtouris |
|
155 | 4375e020 | Stavros Sachtouris | :param content_encoding: (str)
|
156 | 4375e020 | Stavros Sachtouris |
|
157 | 4375e020 | Stavros Sachtouris | :param content_disposition: (str)
|
158 | 4375e020 | Stavros Sachtouris |
|
159 | 4375e020 | Stavros Sachtouris | :param content_type: (str)
|
160 | 4375e020 | Stavros Sachtouris |
|
161 | 4375e020 | Stavros Sachtouris | :param sharing: {'read':[user and/or grp names],
|
162 | 4375e020 | Stavros Sachtouris | 'write':[usr and/or grp names]}
|
163 | 4375e020 | Stavros Sachtouris |
|
164 | 4375e020 | Stavros Sachtouris | :param public: (bool)
|
165 | 4375e020 | Stavros Sachtouris | """
|
166 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
167 | c2b5da2f | Stavros Sachtouris | self.object_put(
|
168 | 24ff0a35 | Stavros Sachtouris | obj, |
169 | 3dabe5d2 | Stavros Sachtouris | content_length=0,
|
170 | 3dabe5d2 | Stavros Sachtouris | etag=etag, |
171 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
172 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
173 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
174 | 3dabe5d2 | Stavros Sachtouris | permissions=sharing, |
175 | 3dabe5d2 | Stavros Sachtouris | public=public, |
176 | 3dabe5d2 | Stavros Sachtouris | manifest='%s/%s' % (self.container, obj)) |
177 | 3dabe5d2 | Stavros Sachtouris | |
178 | 4375e020 | Stavros Sachtouris | # upload_* auxiliary methods
|
179 | 1d329d27 | Stavros Sachtouris | def _put_block_async(self, data, hash, upload_gen=None): |
180 | 4375e020 | Stavros Sachtouris | event = SilentEvent(method=self._put_block, data=data, hash=hash) |
181 | 4375e020 | Stavros Sachtouris | event.start() |
182 | 4375e020 | Stavros Sachtouris | return event
|
183 | 4375e020 | Stavros Sachtouris | |
184 | 4375e020 | Stavros Sachtouris | def _put_block(self, data, hash): |
185 | 24ff0a35 | Stavros Sachtouris | r = self.container_post(
|
186 | 24ff0a35 | Stavros Sachtouris | update=True,
|
187 | 4375e020 | Stavros Sachtouris | content_type='application/octet-stream',
|
188 | 4375e020 | Stavros Sachtouris | content_length=len(data),
|
189 | 4375e020 | Stavros Sachtouris | data=data, |
190 | 4375e020 | Stavros Sachtouris | format='json')
|
191 | 4375e020 | Stavros Sachtouris | assert r.json[0] == hash, 'Local hash does not match server' |
192 | 4375e020 | Stavros Sachtouris | |
193 | 64ab4c13 | Stavros Sachtouris | def _get_file_block_info(self, fileobj, size=None): |
194 | 2f749e6e | Stavros Sachtouris | meta = self.get_container_info()
|
195 | 435008b6 | Stavros Sachtouris | blocksize = int(meta['x-container-block-size']) |
196 | 435008b6 | Stavros Sachtouris | blockhash = meta['x-container-block-hash']
|
197 | 64ab4c13 | Stavros Sachtouris | size = size if size is not None else fstat(fileobj.fileno()).st_size |
198 | 435008b6 | Stavros Sachtouris | nblocks = 1 + (size - 1) // blocksize |
199 | 64ab4c13 | Stavros Sachtouris | return (blocksize, blockhash, size, nblocks)
|
200 | 64ab4c13 | Stavros Sachtouris | |
201 | b349b84b | Stavros Sachtouris | def _create_or_get_missing_hashes( |
202 | 24ff0a35 | Stavros Sachtouris | self, obj, json,
|
203 | 24ff0a35 | Stavros Sachtouris | size=None,
|
204 | 24ff0a35 | Stavros Sachtouris | format='json',
|
205 | 24ff0a35 | Stavros Sachtouris | hashmap=True,
|
206 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
207 | b349b84b | Stavros Sachtouris | if_etag_match=None,
|
208 | b349b84b | Stavros Sachtouris | if_etag_not_match=None,
|
209 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
210 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
211 | 24ff0a35 | Stavros Sachtouris | permissions=None,
|
212 | 24ff0a35 | Stavros Sachtouris | public=None,
|
213 | 24ff0a35 | Stavros Sachtouris | success=(201, 409)): |
214 | 24ff0a35 | Stavros Sachtouris | r = self.object_put(
|
215 | 24ff0a35 | Stavros Sachtouris | obj, |
216 | 3dabe5d2 | Stavros Sachtouris | format='json',
|
217 | 3dabe5d2 | Stavros Sachtouris | hashmap=True,
|
218 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
219 | 3dabe5d2 | Stavros Sachtouris | json=json, |
220 | b349b84b | Stavros Sachtouris | if_etag_match=if_etag_match, |
221 | b349b84b | Stavros Sachtouris | if_etag_not_match=if_etag_not_match, |
222 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
223 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
224 | 3dabe5d2 | Stavros Sachtouris | permissions=permissions, |
225 | 3dabe5d2 | Stavros Sachtouris | public=public, |
226 | 64ab4c13 | Stavros Sachtouris | success=success) |
227 | c2b5da2f | Stavros Sachtouris | return None if r.status_code == 201 else r.json |
228 | 435008b6 | Stavros Sachtouris | |
229 | b349b84b | Stavros Sachtouris | def _calculate_blocks_for_upload( |
230 | 2005b18e | Stavros Sachtouris | self, blocksize, blockhash, size, nblocks, hashes, hmap, fileobj,
|
231 | 2005b18e | Stavros Sachtouris | hash_cb=None):
|
232 | 3dabe5d2 | Stavros Sachtouris | offset = 0
|
233 | 435008b6 | Stavros Sachtouris | if hash_cb:
|
234 | 435008b6 | Stavros Sachtouris | hash_gen = hash_cb(nblocks) |
235 | 435008b6 | Stavros Sachtouris | hash_gen.next() |
236 | 435008b6 | Stavros Sachtouris | |
237 | 435008b6 | Stavros Sachtouris | for i in range(nblocks): |
238 | 64ab4c13 | Stavros Sachtouris | block = fileobj.read(min(blocksize, size - offset))
|
239 | 435008b6 | Stavros Sachtouris | bytes = len(block)
|
240 | 4375e020 | Stavros Sachtouris | hash = _pithos_hash(block, blockhash) |
241 | 435008b6 | Stavros Sachtouris | hashes.append(hash)
|
242 | 5b263ba2 | Stavros Sachtouris | hmap[hash] = (offset, bytes) |
243 | 435008b6 | Stavros Sachtouris | offset += bytes
|
244 | 435008b6 | Stavros Sachtouris | if hash_cb:
|
245 | 435008b6 | Stavros Sachtouris | hash_gen.next() |
246 | fce31e83 | Stavros Sachtouris | msg = 'Failed to calculate uploaded blocks:'
|
247 | bcbf3cf0 | Stavros Sachtouris | ' Offset and object size do not match'
|
248 | fce31e83 | Stavros Sachtouris | assert offset == size, msg
|
249 | 435008b6 | Stavros Sachtouris | |
250 | 3e7d1e0e | Stavros Sachtouris | def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None): |
251 | 3e7d1e0e | Stavros Sachtouris | """upload missing blocks asynchronously"""
|
252 | 435008b6 | Stavros Sachtouris | |
253 | cad39033 | Stavros Sachtouris | self._init_thread_limit()
|
254 | e9abe82b | Stavros Sachtouris | |
255 | 435008b6 | Stavros Sachtouris | flying = [] |
256 | 7644c38e | Stavros Sachtouris | failures = [] |
257 | 435008b6 | Stavros Sachtouris | for hash in missing: |
258 | 5b263ba2 | Stavros Sachtouris | offset, bytes = hmap[hash]
|
259 | 64ab4c13 | Stavros Sachtouris | fileobj.seek(offset) |
260 | 64ab4c13 | Stavros Sachtouris | data = fileobj.read(bytes)
|
261 | 1d329d27 | Stavros Sachtouris | r = self._put_block_async(data, hash, upload_gen) |
262 | 435008b6 | Stavros Sachtouris | flying.append(r) |
263 | 745d938b | Stavros Sachtouris | unfinished = self._watch_thread_limit(flying)
|
264 | 745d938b | Stavros Sachtouris | for thread in set(flying).difference(unfinished): |
265 | 7644c38e | Stavros Sachtouris | if thread.exception:
|
266 | 7644c38e | Stavros Sachtouris | failures.append(thread) |
267 | 24ff0a35 | Stavros Sachtouris | if isinstance( |
268 | 2005b18e | Stavros Sachtouris | thread.exception, |
269 | 2005b18e | Stavros Sachtouris | ClientError) and thread.exception.status == 502: |
270 | 2005b18e | Stavros Sachtouris | self.POOLSIZE = self._thread_limit |
271 | 7644c38e | Stavros Sachtouris | elif thread.isAlive():
|
272 | 745d938b | Stavros Sachtouris | flying.append(thread) |
273 | 3e7d1e0e | Stavros Sachtouris | elif upload_gen:
|
274 | 3e7d1e0e | Stavros Sachtouris | try:
|
275 | 3e7d1e0e | Stavros Sachtouris | upload_gen.next() |
276 | 3e7d1e0e | Stavros Sachtouris | except:
|
277 | 3e7d1e0e | Stavros Sachtouris | pass
|
278 | e02728f9 | Stavros Sachtouris | flying = unfinished |
279 | e02728f9 | Stavros Sachtouris | |
280 | e02728f9 | Stavros Sachtouris | for thread in flying: |
281 | e02728f9 | Stavros Sachtouris | thread.join() |
282 | 7644c38e | Stavros Sachtouris | if thread.exception:
|
283 | 7644c38e | Stavros Sachtouris | failures.append(thread) |
284 | 3e7d1e0e | Stavros Sachtouris | elif upload_gen:
|
285 | 3e7d1e0e | Stavros Sachtouris | try:
|
286 | 3e7d1e0e | Stavros Sachtouris | upload_gen.next() |
287 | 3e7d1e0e | Stavros Sachtouris | except:
|
288 | 3e7d1e0e | Stavros Sachtouris | pass
|
289 | 56f0908a | Stavros Sachtouris | |
290 | 7644c38e | Stavros Sachtouris | return [failure.kwargs['hash'] for failure in failures] |
291 | 3dabe5d2 | Stavros Sachtouris | |
292 | 24ff0a35 | Stavros Sachtouris | def upload_object( |
293 | 24ff0a35 | Stavros Sachtouris | self, obj, f,
|
294 | 24ff0a35 | Stavros Sachtouris | size=None,
|
295 | 24ff0a35 | Stavros Sachtouris | hash_cb=None,
|
296 | 24ff0a35 | Stavros Sachtouris | upload_cb=None,
|
297 | 24ff0a35 | Stavros Sachtouris | etag=None,
|
298 | e9ac514e | Stavros Sachtouris | if_etag_match=None,
|
299 | 524d9cdd | Stavros Sachtouris | if_not_exist=None,
|
300 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
301 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
302 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
303 | 24ff0a35 | Stavros Sachtouris | sharing=None,
|
304 | 24ff0a35 | Stavros Sachtouris | public=None):
|
305 | 4375e020 | Stavros Sachtouris | """Upload an object using multiple connections (threads)
|
306 | 4375e020 | Stavros Sachtouris |
|
307 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
308 | 4375e020 | Stavros Sachtouris |
|
309 | 4375e020 | Stavros Sachtouris | :param f: open file descriptor (rb)
|
310 | 4375e020 | Stavros Sachtouris |
|
311 | 4375e020 | Stavros Sachtouris | :param hash_cb: optional progress.bar object for calculating hashes
|
312 | 4375e020 | Stavros Sachtouris |
|
313 | 4375e020 | Stavros Sachtouris | :param upload_cb: optional progress.bar object for uploading
|
314 | 4375e020 | Stavros Sachtouris |
|
315 | 4375e020 | Stavros Sachtouris | :param etag: (str)
|
316 | 4375e020 | Stavros Sachtouris |
|
317 | e9ac514e | Stavros Sachtouris | :param if_etag_match: (str) Push that value to if-match header at file
|
318 | e9ac514e | Stavros Sachtouris | creation
|
319 | e9ac514e | Stavros Sachtouris |
|
320 | 524d9cdd | Stavros Sachtouris | :param if_not_exist: (bool) If true, the file will be uploaded ONLY if
|
321 | 524d9cdd | Stavros Sachtouris | it does not exist remotely, otherwise the operation will fail.
|
322 | 524d9cdd | Stavros Sachtouris | Involves the case of an object with the same path is created while
|
323 | 524d9cdd | Stavros Sachtouris | the object is being uploaded.
|
324 | 524d9cdd | Stavros Sachtouris |
|
325 | 4375e020 | Stavros Sachtouris | :param content_encoding: (str)
|
326 | 4375e020 | Stavros Sachtouris |
|
327 | 4375e020 | Stavros Sachtouris | :param content_disposition: (str)
|
328 | 4375e020 | Stavros Sachtouris |
|
329 | 4375e020 | Stavros Sachtouris | :param content_type: (str)
|
330 | 4375e020 | Stavros Sachtouris |
|
331 | 4375e020 | Stavros Sachtouris | :param sharing: {'read':[user and/or grp names],
|
332 | 4375e020 | Stavros Sachtouris | 'write':[usr and/or grp names]}
|
333 | 4375e020 | Stavros Sachtouris |
|
334 | 4375e020 | Stavros Sachtouris | :param public: (bool)
|
335 | 4375e020 | Stavros Sachtouris | """
|
336 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
337 | 56f0908a | Stavros Sachtouris | |
338 | 64ab4c13 | Stavros Sachtouris | #init
|
339 | 3dabe5d2 | Stavros Sachtouris | block_info = (blocksize, blockhash, size, nblocks) =\ |
340 | 3dabe5d2 | Stavros Sachtouris | self._get_file_block_info(f, size)
|
341 | 64ab4c13 | Stavros Sachtouris | (hashes, hmap, offset) = ([], {}, 0)
|
342 | bcbf3cf0 | Stavros Sachtouris | if not content_type: |
343 | 3dabe5d2 | Stavros Sachtouris | content_type = 'application/octet-stream'
|
344 | 64ab4c13 | Stavros Sachtouris | |
345 | b349b84b | Stavros Sachtouris | self._calculate_blocks_for_upload(
|
346 | 24ff0a35 | Stavros Sachtouris | *block_info, |
347 | 3dabe5d2 | Stavros Sachtouris | hashes=hashes, |
348 | 3dabe5d2 | Stavros Sachtouris | hmap=hmap, |
349 | 3dabe5d2 | Stavros Sachtouris | fileobj=f, |
350 | 64ab4c13 | Stavros Sachtouris | hash_cb=hash_cb) |
351 | 64ab4c13 | Stavros Sachtouris | |
352 | 64ab4c13 | Stavros Sachtouris | hashmap = dict(bytes=size, hashes=hashes)
|
353 | c6cec5e1 | Stavros Sachtouris | missing = self._create_or_get_missing_hashes(
|
354 | 24ff0a35 | Stavros Sachtouris | obj, hashmap, |
355 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
356 | 3dabe5d2 | Stavros Sachtouris | size=size, |
357 | b349b84b | Stavros Sachtouris | if_etag_match=if_etag_match, |
358 | b349b84b | Stavros Sachtouris | if_etag_not_match='*' if if_not_exist else None, |
359 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
360 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
361 | 3dabe5d2 | Stavros Sachtouris | permissions=sharing, |
362 | 3dabe5d2 | Stavros Sachtouris | public=public) |
363 | 64ab4c13 | Stavros Sachtouris | |
364 | 64ab4c13 | Stavros Sachtouris | if missing is None: |
365 | 64ab4c13 | Stavros Sachtouris | return
|
366 | 1d329d27 | Stavros Sachtouris | |
367 | 3e7d1e0e | Stavros Sachtouris | if upload_cb:
|
368 | 3e7d1e0e | Stavros Sachtouris | upload_gen = upload_cb(len(missing))
|
369 | 745d938b | Stavros Sachtouris | for i in range(len(missing), len(hashmap['hashes']) + 1): |
370 | 745d938b | Stavros Sachtouris | try:
|
371 | 745d938b | Stavros Sachtouris | upload_gen.next() |
372 | 745d938b | Stavros Sachtouris | except:
|
373 | 745d938b | Stavros Sachtouris | upload_gen = None
|
374 | 3e7d1e0e | Stavros Sachtouris | else:
|
375 | 3e7d1e0e | Stavros Sachtouris | upload_gen = None
|
376 | 3e7d1e0e | Stavros Sachtouris | |
377 | 7eda693f | Stavros Sachtouris | retries = 7
|
378 | f27ed9a0 | Stavros Sachtouris | try:
|
379 | 7644c38e | Stavros Sachtouris | while retries:
|
380 | 7eda693f | Stavros Sachtouris | sendlog.info('%s blocks missing' % len(missing)) |
381 | 7644c38e | Stavros Sachtouris | num_of_blocks = len(missing)
|
382 | 7644c38e | Stavros Sachtouris | missing = self._upload_missing_blocks(
|
383 | 7644c38e | Stavros Sachtouris | missing, |
384 | 7644c38e | Stavros Sachtouris | hmap, |
385 | 7644c38e | Stavros Sachtouris | f, |
386 | 3e7d1e0e | Stavros Sachtouris | upload_gen) |
387 | 16c895db | Stavros Sachtouris | if missing:
|
388 | 16c895db | Stavros Sachtouris | if num_of_blocks == len(missing): |
389 | 16c895db | Stavros Sachtouris | retries -= 1
|
390 | 16c895db | Stavros Sachtouris | else:
|
391 | 16c895db | Stavros Sachtouris | num_of_blocks = len(missing)
|
392 | 7644c38e | Stavros Sachtouris | else:
|
393 | 7644c38e | Stavros Sachtouris | break
|
394 | 7644c38e | Stavros Sachtouris | if missing:
|
395 | 7644c38e | Stavros Sachtouris | raise ClientError(
|
396 | 7644c38e | Stavros Sachtouris | '%s blocks failed to upload' % len(missing), |
397 | 7644c38e | Stavros Sachtouris | status=800)
|
398 | f27ed9a0 | Stavros Sachtouris | except KeyboardInterrupt: |
399 | 6069b53b | Stavros Sachtouris | sendlog.info('- - - wait for threads to finish')
|
400 | f27ed9a0 | Stavros Sachtouris | for thread in activethreads(): |
401 | f27ed9a0 | Stavros Sachtouris | thread.join() |
402 | f27ed9a0 | Stavros Sachtouris | raise
|
403 | f27ed9a0 | Stavros Sachtouris | |
404 | c2b5da2f | Stavros Sachtouris | self.object_put(
|
405 | f27ed9a0 | Stavros Sachtouris | obj, |
406 | 3dabe5d2 | Stavros Sachtouris | format='json',
|
407 | 3dabe5d2 | Stavros Sachtouris | hashmap=True,
|
408 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
409 | e9ac514e | Stavros Sachtouris | if_etag_match=if_etag_match, |
410 | 524d9cdd | Stavros Sachtouris | if_etag_not_match='*' if if_not_exist else None, |
411 | 524d9cdd | Stavros Sachtouris | etag=etag, |
412 | 3dabe5d2 | Stavros Sachtouris | json=hashmap, |
413 | 50165863 | Stavros Sachtouris | permissions=sharing, |
414 | 50165863 | Stavros Sachtouris | public=public, |
415 | 3dabe5d2 | Stavros Sachtouris | success=201)
|
416 | 3dabe5d2 | Stavros Sachtouris | |
417 | f27ed9a0 | Stavros Sachtouris | # download_* auxiliary methods
|
418 | fbfee225 | Stavros Sachtouris | def _get_remote_blocks_info(self, obj, **restargs): |
419 | 56f0908a | Stavros Sachtouris | #retrieve object hashmap
|
420 | 3dabe5d2 | Stavros Sachtouris | myrange = restargs.pop('data_range', None) |
421 | 319be41b | Stavros Sachtouris | hashmap = self.get_object_hashmap(obj, **restargs)
|
422 | 642f1bbd | Stavros Sachtouris | restargs['data_range'] = myrange
|
423 | 56f0908a | Stavros Sachtouris | blocksize = int(hashmap['block_size']) |
424 | 56f0908a | Stavros Sachtouris | blockhash = hashmap['block_hash']
|
425 | 56f0908a | Stavros Sachtouris | total_size = hashmap['bytes']
|
426 | fbfee225 | Stavros Sachtouris | #assert total_size/blocksize + 1 == len(hashmap['hashes'])
|
427 | 56f0908a | Stavros Sachtouris | map_dict = {} |
428 | fbfee225 | Stavros Sachtouris | for i, h in enumerate(hashmap['hashes']): |
429 | ca7f78c0 | Stavros Sachtouris | # map_dict[h] = i CHAGE
|
430 | ca7f78c0 | Stavros Sachtouris | if h in map_dict: |
431 | ca7f78c0 | Stavros Sachtouris | map_dict[h].append(i) |
432 | ca7f78c0 | Stavros Sachtouris | else:
|
433 | ca7f78c0 | Stavros Sachtouris | map_dict[h] = [i] |
434 | fbfee225 | Stavros Sachtouris | return (blocksize, blockhash, total_size, hashmap['hashes'], map_dict) |
435 | 64ab4c13 | Stavros Sachtouris | |
436 | 24ff0a35 | Stavros Sachtouris | def _dump_blocks_sync( |
437 | 24ff0a35 | Stavros Sachtouris | self, obj, remote_hashes, blocksize, total_size, dst, range, |
438 | 24ff0a35 | Stavros Sachtouris | **args): |
439 | fbfee225 | Stavros Sachtouris | for blockid, blockhash in enumerate(remote_hashes): |
440 | 24ff0a35 | Stavros Sachtouris | if blockhash:
|
441 | 24ff0a35 | Stavros Sachtouris | start = blocksize * blockid |
442 | 24ff0a35 | Stavros Sachtouris | is_last = start + blocksize > total_size |
443 | 24ff0a35 | Stavros Sachtouris | end = (total_size - 1) if is_last else (start + blocksize - 1) |
444 | 24ff0a35 | Stavros Sachtouris | (start, end) = _range_up(start, end, range)
|
445 | 24ff0a35 | Stavros Sachtouris | args['data_range'] = 'bytes=%s-%s' % (start, end) |
446 | 24ff0a35 | Stavros Sachtouris | r = self.object_get(obj, success=(200, 206), **args) |
447 | 24ff0a35 | Stavros Sachtouris | self._cb_next()
|
448 | 24ff0a35 | Stavros Sachtouris | dst.write(r.content) |
449 | 24ff0a35 | Stavros Sachtouris | dst.flush() |
450 | fbfee225 | Stavros Sachtouris | |
451 | 24ff0a35 | Stavros Sachtouris | def _get_block_async(self, obj, **args): |
452 | 24ff0a35 | Stavros Sachtouris | event = SilentEvent(self.object_get, obj, success=(200, 206), **args) |
453 | e02728f9 | Stavros Sachtouris | event.start() |
454 | e02728f9 | Stavros Sachtouris | return event
|
455 | fb0cd49a | Stavros Sachtouris | |
456 | 48c3782c | Stavros Sachtouris | def _hash_from_file(self, fp, start, size, blockhash): |
457 | 48c3782c | Stavros Sachtouris | fp.seek(start) |
458 | 48c3782c | Stavros Sachtouris | block = fp.read(size) |
459 | 48c3782c | Stavros Sachtouris | h = newhashlib(blockhash) |
460 | 48c3782c | Stavros Sachtouris | h.update(block.strip('\x00'))
|
461 | 48c3782c | Stavros Sachtouris | return hexlify(h.digest())
|
462 | 48c3782c | Stavros Sachtouris | |
463 | 28cbc3c2 | Stavros Sachtouris | def _thread2file(self, flying, blockids, local_file, offset=0, **restargs): |
464 | 22fc09fb | Stavros Sachtouris | """write the results of a greenleted rest call to a file
|
465 | f5f2dc53 | Stavros Sachtouris |
|
466 | f5f2dc53 | Stavros Sachtouris | :param offset: the offset of the file up to blocksize
|
467 | f5f2dc53 | Stavros Sachtouris | - e.g. if the range is 10-100, all blocks will be written to
|
468 | f5f2dc53 | Stavros Sachtouris | normal_position - 10
|
469 | f5f2dc53 | Stavros Sachtouris | """
|
470 | 28cbc3c2 | Stavros Sachtouris | for i, (key, g) in enumerate(flying.items()): |
471 | 28cbc3c2 | Stavros Sachtouris | if g.isAlive():
|
472 | 28cbc3c2 | Stavros Sachtouris | continue
|
473 | 28cbc3c2 | Stavros Sachtouris | if g.exception:
|
474 | 28cbc3c2 | Stavros Sachtouris | raise g.exception
|
475 | 28cbc3c2 | Stavros Sachtouris | block = g.value.content |
476 | 28cbc3c2 | Stavros Sachtouris | for block_start in blockids[key]: |
477 | 28cbc3c2 | Stavros Sachtouris | local_file.seek(block_start + offset) |
478 | fbfee225 | Stavros Sachtouris | local_file.write(block) |
479 | fbfee225 | Stavros Sachtouris | self._cb_next()
|
480 | 28cbc3c2 | Stavros Sachtouris | flying.pop(key) |
481 | 28cbc3c2 | Stavros Sachtouris | blockids.pop(key) |
482 | fbfee225 | Stavros Sachtouris | local_file.flush() |
483 | fbfee225 | Stavros Sachtouris | |
484 | 24ff0a35 | Stavros Sachtouris | def _dump_blocks_async( |
485 | 24ff0a35 | Stavros Sachtouris | self, obj, remote_hashes, blocksize, total_size, local_file,
|
486 | 24ff0a35 | Stavros Sachtouris | blockhash=None, resume=False, filerange=None, **restargs): |
487 | 973fbcad | Stavros Sachtouris | file_size = fstat(local_file.fileno()).st_size if resume else 0 |
488 | 28cbc3c2 | Stavros Sachtouris | flying = dict()
|
489 | 28cbc3c2 | Stavros Sachtouris | blockid_dict = dict()
|
490 | 22fc09fb | Stavros Sachtouris | offset = 0
|
491 | 22fc09fb | Stavros Sachtouris | if filerange is not None: |
492 | 22fc09fb | Stavros Sachtouris | rstart = int(filerange.split('-')[0]) |
493 | 3dabe5d2 | Stavros Sachtouris | offset = rstart if blocksize > rstart else rstart % blocksize |
494 | f27ed9a0 | Stavros Sachtouris | |
495 | cad39033 | Stavros Sachtouris | self._init_thread_limit()
|
496 | ca7f78c0 | Stavros Sachtouris | for block_hash, blockids in remote_hashes.items(): |
497 | 28cbc3c2 | Stavros Sachtouris | blockids = [blk * blocksize for blk in blockids] |
498 | 28cbc3c2 | Stavros Sachtouris | unsaved = [blk for blk in blockids if not ( |
499 | 28cbc3c2 | Stavros Sachtouris | blk < file_size and block_hash == self._hash_from_file( |
500 | 28cbc3c2 | Stavros Sachtouris | local_file, blk, blocksize, blockhash))] |
501 | 28cbc3c2 | Stavros Sachtouris | self._cb_next(len(blockids) - len(unsaved)) |
502 | 28cbc3c2 | Stavros Sachtouris | if unsaved:
|
503 | 28cbc3c2 | Stavros Sachtouris | key = unsaved[0]
|
504 | ca7f78c0 | Stavros Sachtouris | self._watch_thread_limit(flying.values())
|
505 | 28cbc3c2 | Stavros Sachtouris | self._thread2file(
|
506 | 28cbc3c2 | Stavros Sachtouris | flying, blockid_dict, local_file, offset, |
507 | ca7f78c0 | Stavros Sachtouris | **restargs) |
508 | 28cbc3c2 | Stavros Sachtouris | end = total_size - 1 if key + blocksize > total_size\ |
509 | 28cbc3c2 | Stavros Sachtouris | else key + blocksize - 1 |
510 | 28cbc3c2 | Stavros Sachtouris | start, end = _range_up(key, end, filerange) |
511 | ca7f78c0 | Stavros Sachtouris | if start == end:
|
512 | ca7f78c0 | Stavros Sachtouris | self._cb_next()
|
513 | ca7f78c0 | Stavros Sachtouris | continue
|
514 | ca7f78c0 | Stavros Sachtouris | restargs['async_headers'] = {
|
515 | ca7f78c0 | Stavros Sachtouris | 'Range': 'bytes=%s-%s' % (start, end)} |
516 | 28cbc3c2 | Stavros Sachtouris | flying[key] = self._get_block_async(obj, **restargs)
|
517 | 28cbc3c2 | Stavros Sachtouris | blockid_dict[key] = unsaved |
518 | fbfee225 | Stavros Sachtouris | |
519 | e02728f9 | Stavros Sachtouris | for thread in flying.values(): |
520 | e02728f9 | Stavros Sachtouris | thread.join() |
521 | 28cbc3c2 | Stavros Sachtouris | self._thread2file(flying, blockid_dict, local_file, offset, **restargs)
|
522 | fbfee225 | Stavros Sachtouris | |
523 | 24ff0a35 | Stavros Sachtouris | def download_object( |
524 | 24ff0a35 | Stavros Sachtouris | self, obj, dst,
|
525 | 24ff0a35 | Stavros Sachtouris | download_cb=None,
|
526 | 24ff0a35 | Stavros Sachtouris | version=None,
|
527 | 24ff0a35 | Stavros Sachtouris | resume=False,
|
528 | 24ff0a35 | Stavros Sachtouris | range_str=None,
|
529 | 24ff0a35 | Stavros Sachtouris | if_match=None,
|
530 | 24ff0a35 | Stavros Sachtouris | if_none_match=None,
|
531 | 24ff0a35 | Stavros Sachtouris | if_modified_since=None,
|
532 | 24ff0a35 | Stavros Sachtouris | if_unmodified_since=None):
|
533 | f5f2dc53 | Stavros Sachtouris | """Download an object (multiple connections, random blocks)
|
534 | 4375e020 | Stavros Sachtouris |
|
535 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
536 | 4375e020 | Stavros Sachtouris |
|
537 | 4375e020 | Stavros Sachtouris | :param dst: open file descriptor (wb+)
|
538 | 4375e020 | Stavros Sachtouris |
|
539 | 4375e020 | Stavros Sachtouris | :param download_cb: optional progress.bar object for downloading
|
540 | 4375e020 | Stavros Sachtouris |
|
541 | 4375e020 | Stavros Sachtouris | :param version: (str) file version
|
542 | 4375e020 | Stavros Sachtouris |
|
543 | 4375e020 | Stavros Sachtouris | :param resume: (bool) if set, preserve already downloaded file parts
|
544 | 4375e020 | Stavros Sachtouris |
|
545 | f5f2dc53 | Stavros Sachtouris | :param range_str: (str) from, to are file positions (int) in bytes
|
546 | 4375e020 | Stavros Sachtouris |
|
547 | 4375e020 | Stavros Sachtouris | :param if_match: (str)
|
548 | 4375e020 | Stavros Sachtouris |
|
549 | 4375e020 | Stavros Sachtouris | :param if_none_match: (str)
|
550 | 4375e020 | Stavros Sachtouris |
|
551 | 4375e020 | Stavros Sachtouris | :param if_modified_since: (str) formated date
|
552 | 4375e020 | Stavros Sachtouris |
|
553 | f5f2dc53 | Stavros Sachtouris | :param if_unmodified_since: (str) formated date"""
|
554 | 24ff0a35 | Stavros Sachtouris | restargs = dict(
|
555 | 24ff0a35 | Stavros Sachtouris | version=version, |
556 | 24ff0a35 | Stavros Sachtouris | data_range=None if range_str is None else 'bytes=%s' % range_str, |
557 | fbfee225 | Stavros Sachtouris | if_match=if_match, |
558 | fbfee225 | Stavros Sachtouris | if_none_match=if_none_match, |
559 | fbfee225 | Stavros Sachtouris | if_modified_since=if_modified_since, |
560 | fbfee225 | Stavros Sachtouris | if_unmodified_since=if_unmodified_since) |
561 | fbfee225 | Stavros Sachtouris | |
562 | 24ff0a35 | Stavros Sachtouris | ( |
563 | 24ff0a35 | Stavros Sachtouris | blocksize, |
564 | fbfee225 | Stavros Sachtouris | blockhash, |
565 | fbfee225 | Stavros Sachtouris | total_size, |
566 | 3dabe5d2 | Stavros Sachtouris | hash_list, |
567 | fbfee225 | Stavros Sachtouris | remote_hashes) = self._get_remote_blocks_info(obj, **restargs)
|
568 | fbfee225 | Stavros Sachtouris | assert total_size >= 0 |
569 | fbfee225 | Stavros Sachtouris | |
570 | fbfee225 | Stavros Sachtouris | if download_cb:
|
571 | ca7f78c0 | Stavros Sachtouris | self.progress_bar_gen = download_cb(len(hash_list)) |
572 | fbfee225 | Stavros Sachtouris | self._cb_next()
|
573 | fbfee225 | Stavros Sachtouris | |
574 | fbfee225 | Stavros Sachtouris | if dst.isatty():
|
575 | 24ff0a35 | Stavros Sachtouris | self._dump_blocks_sync(
|
576 | 24ff0a35 | Stavros Sachtouris | obj, |
577 | 3dabe5d2 | Stavros Sachtouris | hash_list, |
578 | 3dabe5d2 | Stavros Sachtouris | blocksize, |
579 | 3dabe5d2 | Stavros Sachtouris | total_size, |
580 | 3dabe5d2 | Stavros Sachtouris | dst, |
581 | 24ff0a35 | Stavros Sachtouris | range_str, |
582 | 3dabe5d2 | Stavros Sachtouris | **restargs) |
583 | 699d3bb1 | Stavros Sachtouris | else:
|
584 | 24ff0a35 | Stavros Sachtouris | self._dump_blocks_async(
|
585 | 24ff0a35 | Stavros Sachtouris | obj, |
586 | 3dabe5d2 | Stavros Sachtouris | remote_hashes, |
587 | 3dabe5d2 | Stavros Sachtouris | blocksize, |
588 | 3dabe5d2 | Stavros Sachtouris | total_size, |
589 | 3dabe5d2 | Stavros Sachtouris | dst, |
590 | 3dabe5d2 | Stavros Sachtouris | blockhash, |
591 | 3dabe5d2 | Stavros Sachtouris | resume, |
592 | 24ff0a35 | Stavros Sachtouris | range_str, |
593 | 3dabe5d2 | Stavros Sachtouris | **restargs) |
594 | 24ff0a35 | Stavros Sachtouris | if not range_str: |
595 | 22fc09fb | Stavros Sachtouris | dst.truncate(total_size) |
596 | 5b263ba2 | Stavros Sachtouris | |
597 | fbfee225 | Stavros Sachtouris | self._complete_cb()
|
598 | fbfee225 | Stavros Sachtouris | |
599 | fbfee225 | Stavros Sachtouris | #Command Progress Bar method
|
600 | 28cbc3c2 | Stavros Sachtouris | def _cb_next(self, step=1): |
601 | fbfee225 | Stavros Sachtouris | if hasattr(self, 'progress_bar_gen'): |
602 | fbfee225 | Stavros Sachtouris | try:
|
603 | e5c76b1a | Stavros Sachtouris | for i in xrange(step): |
604 | e5c76b1a | Stavros Sachtouris | self.progress_bar_gen.next()
|
605 | fbfee225 | Stavros Sachtouris | except:
|
606 | fbfee225 | Stavros Sachtouris | pass
|
607 | 3dabe5d2 | Stavros Sachtouris | |
608 | fbfee225 | Stavros Sachtouris | def _complete_cb(self): |
609 | fbfee225 | Stavros Sachtouris | while True: |
610 | fbfee225 | Stavros Sachtouris | try:
|
611 | e5c76b1a | Stavros Sachtouris | self.progress_bar_gen.next()
|
612 | fbfee225 | Stavros Sachtouris | except:
|
613 | fbfee225 | Stavros Sachtouris | break
|
614 | b1713259 | Stavros Sachtouris | |
615 | 24ff0a35 | Stavros Sachtouris | def get_object_hashmap( |
616 | 24ff0a35 | Stavros Sachtouris | self, obj,
|
617 | 24ff0a35 | Stavros Sachtouris | version=None,
|
618 | 24ff0a35 | Stavros Sachtouris | if_match=None,
|
619 | 24ff0a35 | Stavros Sachtouris | if_none_match=None,
|
620 | 24ff0a35 | Stavros Sachtouris | if_modified_since=None,
|
621 | 24ff0a35 | Stavros Sachtouris | if_unmodified_since=None,
|
622 | 24ff0a35 | Stavros Sachtouris | data_range=None):
|
623 | 4375e020 | Stavros Sachtouris | """
|
624 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
625 | 4375e020 | Stavros Sachtouris |
|
626 | 4375e020 | Stavros Sachtouris | :param if_match: (str)
|
627 | 4375e020 | Stavros Sachtouris |
|
628 | 4375e020 | Stavros Sachtouris | :param if_none_match: (str)
|
629 | 4375e020 | Stavros Sachtouris |
|
630 | 4375e020 | Stavros Sachtouris | :param if_modified_since: (str) formated date
|
631 | 4375e020 | Stavros Sachtouris |
|
632 | 4375e020 | Stavros Sachtouris | :param if_unmodified_since: (str) formated date
|
633 | 4375e020 | Stavros Sachtouris |
|
634 | 4375e020 | Stavros Sachtouris | :param data_range: (str) from-to where from and to are integers
|
635 | 4375e020 | Stavros Sachtouris | denoting file positions in bytes
|
636 | 4375e020 | Stavros Sachtouris |
|
637 | 4375e020 | Stavros Sachtouris | :returns: (list)
|
638 | 4375e020 | Stavros Sachtouris | """
|
639 | d804de82 | Stavros Sachtouris | try:
|
640 | 24ff0a35 | Stavros Sachtouris | r = self.object_get(
|
641 | 24ff0a35 | Stavros Sachtouris | obj, |
642 | 3dabe5d2 | Stavros Sachtouris | hashmap=True,
|
643 | 3dabe5d2 | Stavros Sachtouris | version=version, |
644 | 3dabe5d2 | Stavros Sachtouris | if_etag_match=if_match, |
645 | 3dabe5d2 | Stavros Sachtouris | if_etag_not_match=if_none_match, |
646 | 3dabe5d2 | Stavros Sachtouris | if_modified_since=if_modified_since, |
647 | 3dabe5d2 | Stavros Sachtouris | if_unmodified_since=if_unmodified_since, |
648 | 3dabe5d2 | Stavros Sachtouris | data_range=data_range) |
649 | d804de82 | Stavros Sachtouris | except ClientError as err: |
650 | d804de82 | Stavros Sachtouris | if err.status == 304 or err.status == 412: |
651 | d804de82 | Stavros Sachtouris | return {}
|
652 | d804de82 | Stavros Sachtouris | raise
|
653 | 64ab4c13 | Stavros Sachtouris | return r.json
|
654 | 56f0908a | Stavros Sachtouris | |
655 | 3a9e54b0 | Stavros Sachtouris | def set_account_group(self, group, usernames): |
656 | 4375e020 | Stavros Sachtouris | """
|
657 | 4375e020 | Stavros Sachtouris | :param group: (str)
|
658 | 4375e020 | Stavros Sachtouris |
|
659 | 4375e020 | Stavros Sachtouris | :param usernames: (list)
|
660 | 4375e020 | Stavros Sachtouris | """
|
661 | c2b5da2f | Stavros Sachtouris | self.account_post(update=True, groups={group: usernames}) |
662 | eb903ba7 | Stavros Sachtouris | |
663 | c2867610 | Stavros Sachtouris | def del_account_group(self, group): |
664 | 4375e020 | Stavros Sachtouris | """
|
665 | 4375e020 | Stavros Sachtouris | :param group: (str)
|
666 | 4375e020 | Stavros Sachtouris | """
|
667 | c2b5da2f | Stavros Sachtouris | self.account_post(update=True, groups={group: []}) |
668 | c2867610 | Stavros Sachtouris | |
669 | 8af4cc0b | Stavros Sachtouris | def get_account_info(self, until=None): |
670 | 4375e020 | Stavros Sachtouris | """
|
671 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
672 | 4375e020 | Stavros Sachtouris |
|
673 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
674 | 4375e020 | Stavros Sachtouris | """
|
675 | 8af4cc0b | Stavros Sachtouris | r = self.account_head(until=until)
|
676 | 6657ec8c | Stavros Sachtouris | if r.status_code == 401: |
677 | 24851aa5 | Stavros Sachtouris | raise ClientError("No authorization", status=401) |
678 | 64ab4c13 | Stavros Sachtouris | return r.headers
|
679 | 6657ec8c | Stavros Sachtouris | |
680 | d1856abf | Stavros Sachtouris | def get_account_quota(self): |
681 | 4375e020 | Stavros Sachtouris | """
|
682 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
683 | 4375e020 | Stavros Sachtouris | """
|
684 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
685 | 24ff0a35 | Stavros Sachtouris | self.get_account_info(),
|
686 | 3dabe5d2 | Stavros Sachtouris | 'X-Account-Policy-Quota',
|
687 | 3dabe5d2 | Stavros Sachtouris | exactMatch=True)
|
688 | d1856abf | Stavros Sachtouris | |
689 | d1856abf | Stavros Sachtouris | def get_account_versioning(self): |
690 | 4375e020 | Stavros Sachtouris | """
|
691 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
692 | 4375e020 | Stavros Sachtouris | """
|
693 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
694 | 24ff0a35 | Stavros Sachtouris | self.get_account_info(),
|
695 | 3dabe5d2 | Stavros Sachtouris | 'X-Account-Policy-Versioning',
|
696 | 3dabe5d2 | Stavros Sachtouris | exactMatch=True)
|
697 | e6b39366 | Stavros Sachtouris | |
698 | 8af4cc0b | Stavros Sachtouris | def get_account_meta(self, until=None): |
699 | 4375e020 | Stavros Sachtouris | """
|
700 | 4375e020 | Stavros Sachtouris | :meta until: (str) formated date
|
701 | 4375e020 | Stavros Sachtouris |
|
702 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
703 | 4375e020 | Stavros Sachtouris | """
|
704 | 3dabe5d2 | Stavros Sachtouris | return filter_in(self.get_account_info(until=until), 'X-Account-Meta-') |
705 | af3b2b36 | Stavros Sachtouris | |
706 | c2867610 | Stavros Sachtouris | def get_account_group(self): |
707 | 4375e020 | Stavros Sachtouris | """
|
708 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
709 | 4375e020 | Stavros Sachtouris | """
|
710 | c2867610 | Stavros Sachtouris | return filter_in(self.get_account_info(), 'X-Account-Group-') |
711 | c2867610 | Stavros Sachtouris | |
712 | af3b2b36 | Stavros Sachtouris | def set_account_meta(self, metapairs): |
713 | 4375e020 | Stavros Sachtouris | """
|
714 | 4375e020 | Stavros Sachtouris | :param metapairs: (dict) {key1:val1, key2:val2, ...}
|
715 | 4375e020 | Stavros Sachtouris | """
|
716 | af3b2b36 | Stavros Sachtouris | assert(type(metapairs) is dict) |
717 | c2b5da2f | Stavros Sachtouris | self.account_post(update=True, metadata=metapairs) |
718 | af3b2b36 | Stavros Sachtouris | |
719 | 379cd4bb | Stavros Sachtouris | def del_account_meta(self, metakey): |
720 | 4375e020 | Stavros Sachtouris | """
|
721 | 4375e020 | Stavros Sachtouris | :param metakey: (str) metadatum key
|
722 | 4375e020 | Stavros Sachtouris | """
|
723 | c2b5da2f | Stavros Sachtouris | self.account_post(update=True, metadata={metakey: ''}) |
724 | 379cd4bb | Stavros Sachtouris | |
725 | 3ed6dbde | Stavros Sachtouris | """
|
726 | d1856abf | Stavros Sachtouris | def set_account_quota(self, quota):
|
727 | 3ed6dbde | Stavros Sachtouris | ""
|
728 | 4375e020 | Stavros Sachtouris | :param quota: (int)
|
729 | 3ed6dbde | Stavros Sachtouris | ""
|
730 | c2b5da2f | Stavros Sachtouris | self.account_post(update=True, quota=quota)
|
731 | 3ed6dbde | Stavros Sachtouris | """
|
732 | d1856abf | Stavros Sachtouris | |
733 | d1856abf | Stavros Sachtouris | def set_account_versioning(self, versioning): |
734 | 4375e020 | Stavros Sachtouris | """
|
735 | 4375e020 | Stavros Sachtouris | "param versioning: (str)
|
736 | 4375e020 | Stavros Sachtouris | """
|
737 | c2b5da2f | Stavros Sachtouris | self.account_post(update=True, versioning=versioning) |
738 | d1856abf | Stavros Sachtouris | |
739 | 4fd88feb | Stavros Sachtouris | def list_containers(self): |
740 | 4375e020 | Stavros Sachtouris | """
|
741 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
742 | 4375e020 | Stavros Sachtouris | """
|
743 | 4fd88feb | Stavros Sachtouris | r = self.account_get()
|
744 | 64ab4c13 | Stavros Sachtouris | return r.json
|
745 | b758e547 | Stavros Sachtouris | |
746 | a298f2ab | Stavros Sachtouris | def del_container(self, until=None, delimiter=None): |
747 | 4375e020 | Stavros Sachtouris | """
|
748 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
749 | 4375e020 | Stavros Sachtouris |
|
750 | 4c1882ab | Stavros Sachtouris | :param delimiter: (str) with / empty container
|
751 | 4375e020 | Stavros Sachtouris |
|
752 | 4375e020 | Stavros Sachtouris | :raises ClientError: 404 Container does not exist
|
753 | 4375e020 | Stavros Sachtouris |
|
754 | 4375e020 | Stavros Sachtouris | :raises ClientError: 409 Container is not empty
|
755 | 4375e020 | Stavros Sachtouris | """
|
756 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
757 | 2005b18e | Stavros Sachtouris | r = self.container_delete(
|
758 | 2005b18e | Stavros Sachtouris | until=until, |
759 | 3dabe5d2 | Stavros Sachtouris | delimiter=delimiter, |
760 | 3dabe5d2 | Stavros Sachtouris | success=(204, 404, 409)) |
761 | a298f2ab | Stavros Sachtouris | if r.status_code == 404: |
762 | 24ff0a35 | Stavros Sachtouris | raise ClientError(
|
763 | 24ff0a35 | Stavros Sachtouris | 'Container "%s" does not exist' % self.container, |
764 | 3dabe5d2 | Stavros Sachtouris | r.status_code) |
765 | a298f2ab | Stavros Sachtouris | elif r.status_code == 409: |
766 | 24ff0a35 | Stavros Sachtouris | raise ClientError(
|
767 | 24ff0a35 | Stavros Sachtouris | 'Container "%s" is not empty' % self.container, |
768 | 3dabe5d2 | Stavros Sachtouris | r.status_code) |
769 | a298f2ab | Stavros Sachtouris | |
770 | bae347d3 | Stavros Sachtouris | def get_container_versioning(self, container=None): |
771 | 4375e020 | Stavros Sachtouris | """
|
772 | 4375e020 | Stavros Sachtouris | :param container: (str)
|
773 | 4375e020 | Stavros Sachtouris |
|
774 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
775 | 4375e020 | Stavros Sachtouris | """
|
776 | bae347d3 | Stavros Sachtouris | cnt_back_up = self.container
|
777 | bae347d3 | Stavros Sachtouris | try:
|
778 | bae347d3 | Stavros Sachtouris | self.container = container or cnt_back_up |
779 | bae347d3 | Stavros Sachtouris | return filter_in(
|
780 | bae347d3 | Stavros Sachtouris | self.get_container_info(),
|
781 | bae347d3 | Stavros Sachtouris | 'X-Container-Policy-Versioning')
|
782 | bae347d3 | Stavros Sachtouris | finally:
|
783 | bae347d3 | Stavros Sachtouris | self.container = cnt_back_up
|
784 | d1856abf | Stavros Sachtouris | |
785 | 3ed6dbde | Stavros Sachtouris | def get_container_limit(self, container=None): |
786 | 4375e020 | Stavros Sachtouris | """
|
787 | 4375e020 | Stavros Sachtouris | :param container: (str)
|
788 | 4375e020 | Stavros Sachtouris |
|
789 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
790 | 4375e020 | Stavros Sachtouris | """
|
791 | 1879e310 | Stavros Sachtouris | cnt_back_up = self.container
|
792 | 1879e310 | Stavros Sachtouris | try:
|
793 | 1879e310 | Stavros Sachtouris | self.container = container or cnt_back_up |
794 | 1879e310 | Stavros Sachtouris | return filter_in(
|
795 | 1879e310 | Stavros Sachtouris | self.get_container_info(),
|
796 | 1879e310 | Stavros Sachtouris | 'X-Container-Policy-Quota')
|
797 | 1879e310 | Stavros Sachtouris | finally:
|
798 | 1879e310 | Stavros Sachtouris | self.container = cnt_back_up
|
799 | e6b39366 | Stavros Sachtouris | |
800 | 3dabe5d2 | Stavros Sachtouris | def get_container_info(self, until=None): |
801 | 4375e020 | Stavros Sachtouris | """
|
802 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
803 | 4375e020 | Stavros Sachtouris |
|
804 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
805 | f91bc6b1 | Stavros Sachtouris |
|
806 | f91bc6b1 | Stavros Sachtouris | :raises ClientError: 404 Container not found
|
807 | 4375e020 | Stavros Sachtouris | """
|
808 | f91bc6b1 | Stavros Sachtouris | try:
|
809 | f91bc6b1 | Stavros Sachtouris | r = self.container_head(until=until)
|
810 | f91bc6b1 | Stavros Sachtouris | except ClientError as err: |
811 | f91bc6b1 | Stavros Sachtouris | err.details.append('for container %s' % self.container) |
812 | f91bc6b1 | Stavros Sachtouris | raise err
|
813 | 8af4cc0b | Stavros Sachtouris | return r.headers
|
814 | 8af4cc0b | Stavros Sachtouris | |
815 | 3dabe5d2 | Stavros Sachtouris | def get_container_meta(self, until=None): |
816 | 4375e020 | Stavros Sachtouris | """
|
817 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
818 | 4375e020 | Stavros Sachtouris |
|
819 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
820 | 4375e020 | Stavros Sachtouris | """
|
821 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
822 | 24ff0a35 | Stavros Sachtouris | self.get_container_info(until=until),
|
823 | 3dabe5d2 | Stavros Sachtouris | 'X-Container-Meta')
|
824 | e6b39366 | Stavros Sachtouris | |
825 | 3dabe5d2 | Stavros Sachtouris | def get_container_object_meta(self, until=None): |
826 | 4375e020 | Stavros Sachtouris | """
|
827 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
828 | 4375e020 | Stavros Sachtouris |
|
829 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
830 | 4375e020 | Stavros Sachtouris | """
|
831 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
832 | 24ff0a35 | Stavros Sachtouris | self.get_container_info(until=until),
|
833 | 3dabe5d2 | Stavros Sachtouris | 'X-Container-Object-Meta')
|
834 | e6b39366 | Stavros Sachtouris | |
835 | af3b2b36 | Stavros Sachtouris | def set_container_meta(self, metapairs): |
836 | 4375e020 | Stavros Sachtouris | """
|
837 | 4375e020 | Stavros Sachtouris | :param metapairs: (dict) {key1:val1, key2:val2, ...}
|
838 | 4375e020 | Stavros Sachtouris | """
|
839 | af3b2b36 | Stavros Sachtouris | assert(type(metapairs) is dict) |
840 | c2b5da2f | Stavros Sachtouris | self.container_post(update=True, metadata=metapairs) |
841 | 3dabe5d2 | Stavros Sachtouris | |
842 | 3e544e5b | Stavros Sachtouris | def del_container_meta(self, metakey): |
843 | 4375e020 | Stavros Sachtouris | """
|
844 | 4375e020 | Stavros Sachtouris | :param metakey: (str) metadatum key
|
845 | 4375e020 | Stavros Sachtouris | """
|
846 | c2b5da2f | Stavros Sachtouris | self.container_post(update=True, metadata={metakey: ''}) |
847 | e6b39366 | Stavros Sachtouris | |
848 | 326a79b9 | Stavros Sachtouris | def set_container_limit(self, limit): |
849 | 4375e020 | Stavros Sachtouris | """
|
850 | 326a79b9 | Stavros Sachtouris | :param limit: (int)
|
851 | 4375e020 | Stavros Sachtouris | """
|
852 | 326a79b9 | Stavros Sachtouris | self.container_post(update=True, quota=limit) |
853 | d1856abf | Stavros Sachtouris | |
854 | d1856abf | Stavros Sachtouris | def set_container_versioning(self, versioning): |
855 | 4375e020 | Stavros Sachtouris | """
|
856 | 4375e020 | Stavros Sachtouris | :param versioning: (str)
|
857 | 4375e020 | Stavros Sachtouris | """
|
858 | c2b5da2f | Stavros Sachtouris | self.container_post(update=True, versioning=versioning) |
859 | d1856abf | Stavros Sachtouris | |
860 | a298f2ab | Stavros Sachtouris | def del_object(self, obj, until=None, delimiter=None): |
861 | 4375e020 | Stavros Sachtouris | """
|
862 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
863 | 4375e020 | Stavros Sachtouris |
|
864 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
865 | 4375e020 | Stavros Sachtouris |
|
866 | 4375e020 | Stavros Sachtouris | :param delimiter: (str)
|
867 | 4375e020 | Stavros Sachtouris | """
|
868 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
869 | c2b5da2f | Stavros Sachtouris | self.object_delete(obj, until=until, delimiter=delimiter)
|
870 | a298f2ab | Stavros Sachtouris | |
871 | 4375e020 | Stavros Sachtouris | def set_object_meta(self, obj, metapairs): |
872 | 4375e020 | Stavros Sachtouris | """
|
873 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
874 | 4375e020 | Stavros Sachtouris |
|
875 | 4375e020 | Stavros Sachtouris | :param metapairs: (dict) {key1:val1, key2:val2, ...}
|
876 | 4375e020 | Stavros Sachtouris | """
|
877 | af3b2b36 | Stavros Sachtouris | assert(type(metapairs) is dict) |
878 | c2b5da2f | Stavros Sachtouris | self.object_post(obj, update=True, metadata=metapairs) |
879 | 87688514 | Stavros Sachtouris | |
880 | bc223d91 | Stavros Sachtouris | def del_object_meta(self, obj, metakey): |
881 | 4375e020 | Stavros Sachtouris | """
|
882 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
883 | bc223d91 | Stavros Sachtouris |
|
884 | bc223d91 | Stavros Sachtouris | :param metakey: (str) metadatum key
|
885 | 4375e020 | Stavros Sachtouris | """
|
886 | c2b5da2f | Stavros Sachtouris | self.object_post(obj, update=True, metadata={metakey: ''}) |
887 | 6de1f262 | Stavros Sachtouris | |
888 | bc223d91 | Stavros Sachtouris | def publish_object(self, obj): |
889 | bc223d91 | Stavros Sachtouris | """
|
890 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
891 | 5260a313 | Stavros Sachtouris |
|
892 | 5260a313 | Stavros Sachtouris | :returns: (str) access url
|
893 | bc223d91 | Stavros Sachtouris | """
|
894 | c2b5da2f | Stavros Sachtouris | self.object_post(obj, update=True, public=True) |
895 | 5260a313 | Stavros Sachtouris | info = self.get_object_info(obj)
|
896 | 93542587 | Stavros Sachtouris | pref, sep, rest = self.base_url.partition('//') |
897 | 93542587 | Stavros Sachtouris | base = rest.split('/')[0] |
898 | f847a0cc | Stavros Sachtouris | return '%s%s%s/%s' % (pref, sep, base, info['x-object-public']) |
899 | 87688514 | Stavros Sachtouris | |
900 | bc223d91 | Stavros Sachtouris | def unpublish_object(self, obj): |
901 | bc223d91 | Stavros Sachtouris | """
|
902 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
903 | bc223d91 | Stavros Sachtouris | """
|
904 | c2b5da2f | Stavros Sachtouris | self.object_post(obj, update=True, public=False) |
905 | 28470086 | Stavros Sachtouris | |
906 | 8af4cc0b | Stavros Sachtouris | def get_object_info(self, obj, version=None): |
907 | bc223d91 | Stavros Sachtouris | """
|
908 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
909 | bc223d91 | Stavros Sachtouris |
|
910 | bc223d91 | Stavros Sachtouris | :param version: (str)
|
911 | bc223d91 | Stavros Sachtouris |
|
912 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
913 | bc223d91 | Stavros Sachtouris | """
|
914 | ca092af4 | Stavros Sachtouris | try:
|
915 | ca092af4 | Stavros Sachtouris | r = self.object_head(obj, version=version)
|
916 | ca092af4 | Stavros Sachtouris | return r.headers
|
917 | ca092af4 | Stavros Sachtouris | except ClientError as ce: |
918 | ca092af4 | Stavros Sachtouris | if ce.status == 404: |
919 | 723e9d47 | Stavros Sachtouris | raise ClientError('Object %s not found' % obj, status=404) |
920 | ca092af4 | Stavros Sachtouris | raise
|
921 | 8af4cc0b | Stavros Sachtouris | |
922 | 8af4cc0b | Stavros Sachtouris | def get_object_meta(self, obj, version=None): |
923 | bc223d91 | Stavros Sachtouris | """
|
924 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
925 | bc223d91 | Stavros Sachtouris |
|
926 | bc223d91 | Stavros Sachtouris | :param version: (str)
|
927 | bc223d91 | Stavros Sachtouris |
|
928 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
929 | bc223d91 | Stavros Sachtouris | """
|
930 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
931 | 24ff0a35 | Stavros Sachtouris | self.get_object_info(obj, version=version),
|
932 | 3dabe5d2 | Stavros Sachtouris | 'X-Object-Meta')
|
933 | 8af4cc0b | Stavros Sachtouris | |
934 | bc223d91 | Stavros Sachtouris | def get_object_sharing(self, obj): |
935 | bc223d91 | Stavros Sachtouris | """
|
936 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
937 | bc223d91 | Stavros Sachtouris |
|
938 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
939 | bc223d91 | Stavros Sachtouris | """
|
940 | 24ff0a35 | Stavros Sachtouris | r = filter_in( |
941 | 24ff0a35 | Stavros Sachtouris | self.get_object_info(obj),
|
942 | 3dabe5d2 | Stavros Sachtouris | 'X-Object-Sharing',
|
943 | 3dabe5d2 | Stavros Sachtouris | exactMatch=True)
|
944 | f70616fc | Stavros Sachtouris | reply = {} |
945 | f70616fc | Stavros Sachtouris | if len(r) > 0: |
946 | f70616fc | Stavros Sachtouris | perms = r['x-object-sharing'].split(';') |
947 | f70616fc | Stavros Sachtouris | for perm in perms: |
948 | f70616fc | Stavros Sachtouris | try:
|
949 | f70616fc | Stavros Sachtouris | perm.index('=')
|
950 | f70616fc | Stavros Sachtouris | except ValueError: |
951 | f70616fc | Stavros Sachtouris | raise ClientError('Incorrect reply format') |
952 | f70616fc | Stavros Sachtouris | (key, val) = perm.strip().split('=')
|
953 | f70616fc | Stavros Sachtouris | reply[key] = val |
954 | f70616fc | Stavros Sachtouris | return reply
|
955 | f49084df | Stavros Sachtouris | |
956 | 24ff0a35 | Stavros Sachtouris | def set_object_sharing( |
957 | 2005b18e | Stavros Sachtouris | self, obj,
|
958 | 2005b18e | Stavros Sachtouris | read_permition=False, write_permition=False): |
959 | 28470086 | Stavros Sachtouris | """Give read/write permisions to an object.
|
960 | bc223d91 | Stavros Sachtouris |
|
961 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
962 | bc223d91 | Stavros Sachtouris |
|
963 | bc223d91 | Stavros Sachtouris | :param read_permition: (list - bool) users and user groups that get
|
964 | bc223d91 | Stavros Sachtouris | read permition for this object - False means all previous read
|
965 | bc223d91 | Stavros Sachtouris | permissions will be removed
|
966 | bc223d91 | Stavros Sachtouris |
|
967 | bc223d91 | Stavros Sachtouris | :param write_perimition: (list - bool) of users and user groups to get
|
968 | bc223d91 | Stavros Sachtouris | write permition for this object - False means all previous write
|
969 | 4067cdaf | Stavros Sachtouris | permissions will be removed
|
970 | 28470086 | Stavros Sachtouris | """
|
971 | 4067cdaf | Stavros Sachtouris | |
972 | 221820b8 | Stavros Sachtouris | perms = dict(read=read_permition or '', write=write_permition or '') |
973 | c2b5da2f | Stavros Sachtouris | self.object_post(obj, update=True, permissions=perms) |
974 | 28470086 | Stavros Sachtouris | |
975 | bc223d91 | Stavros Sachtouris | def del_object_sharing(self, obj): |
976 | bc223d91 | Stavros Sachtouris | """
|
977 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
978 | bc223d91 | Stavros Sachtouris | """
|
979 | bc223d91 | Stavros Sachtouris | self.set_object_sharing(obj)
|
980 | bc223d91 | Stavros Sachtouris | |
981 | bc223d91 | Stavros Sachtouris | def append_object(self, obj, source_file, upload_cb=None): |
982 | bc223d91 | Stavros Sachtouris | """
|
983 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
984 | bc223d91 | Stavros Sachtouris |
|
985 | bc223d91 | Stavros Sachtouris | :param source_file: open file descriptor
|
986 | f49084df | Stavros Sachtouris |
|
987 | bc223d91 | Stavros Sachtouris | :param upload_db: progress.bar for uploading
|
988 | bcabbc35 | Stavros Sachtouris | """
|
989 | 4067cdaf | Stavros Sachtouris | |
990 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
991 | 2f749e6e | Stavros Sachtouris | meta = self.get_container_info()
|
992 | ab474306 | Stavros Sachtouris | blocksize = int(meta['x-container-block-size']) |
993 | 64ab4c13 | Stavros Sachtouris | filesize = fstat(source_file.fileno()).st_size |
994 | 3dabe5d2 | Stavros Sachtouris | nblocks = 1 + (filesize - 1) // blocksize |
995 | ab474306 | Stavros Sachtouris | offset = 0
|
996 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
997 | bcabbc35 | Stavros Sachtouris | upload_gen = upload_cb(nblocks) |
998 | ca092af4 | Stavros Sachtouris | upload_gen.next() |
999 | ab474306 | Stavros Sachtouris | for i in range(nblocks): |
1000 | ab474306 | Stavros Sachtouris | block = source_file.read(min(blocksize, filesize - offset))
|
1001 | ab474306 | Stavros Sachtouris | offset += len(block)
|
1002 | c2b5da2f | Stavros Sachtouris | self.object_post(
|
1003 | 24ff0a35 | Stavros Sachtouris | obj, |
1004 | 3dabe5d2 | Stavros Sachtouris | update=True,
|
1005 | 3dabe5d2 | Stavros Sachtouris | content_range='bytes */*',
|
1006 | 3dabe5d2 | Stavros Sachtouris | content_type='application/octet-stream',
|
1007 | 3dabe5d2 | Stavros Sachtouris | content_length=len(block),
|
1008 | 3dabe5d2 | Stavros Sachtouris | data=block) |
1009 | 3dabe5d2 | Stavros Sachtouris | |
1010 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
1011 | bcabbc35 | Stavros Sachtouris | upload_gen.next() |
1012 | 561116a6 | Stavros Sachtouris | |
1013 | bc223d91 | Stavros Sachtouris | def truncate_object(self, obj, upto_bytes): |
1014 | bc223d91 | Stavros Sachtouris | """
|
1015 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
1016 | bc223d91 | Stavros Sachtouris |
|
1017 | bc223d91 | Stavros Sachtouris | :param upto_bytes: max number of bytes to leave on file
|
1018 | bc223d91 | Stavros Sachtouris | """
|
1019 | c2b5da2f | Stavros Sachtouris | self.object_post(
|
1020 | 24ff0a35 | Stavros Sachtouris | obj, |
1021 | 3dabe5d2 | Stavros Sachtouris | update=True,
|
1022 | 3dabe5d2 | Stavros Sachtouris | content_range='bytes 0-%s/*' % upto_bytes,
|
1023 | 3dabe5d2 | Stavros Sachtouris | content_type='application/octet-stream',
|
1024 | 3dabe5d2 | Stavros Sachtouris | object_bytes=upto_bytes, |
1025 | bc223d91 | Stavros Sachtouris | source_object=path4url(self.container, obj))
|
1026 | ee62607e | Stavros Sachtouris | |
1027 | 2005b18e | Stavros Sachtouris | def overwrite_object(self, obj, start, end, source_file, upload_cb=None): |
1028 | bc223d91 | Stavros Sachtouris | """Overwrite a part of an object from local source file
|
1029 | bc223d91 | Stavros Sachtouris |
|
1030 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
1031 | bc223d91 | Stavros Sachtouris |
|
1032 | bc223d91 | Stavros Sachtouris | :param start: (int) position in bytes to start overwriting from
|
1033 | bc223d91 | Stavros Sachtouris |
|
1034 | bc223d91 | Stavros Sachtouris | :param end: (int) position in bytes to stop overwriting at
|
1035 | bc223d91 | Stavros Sachtouris |
|
1036 | bc223d91 | Stavros Sachtouris | :param source_file: open file descriptor
|
1037 | bc223d91 | Stavros Sachtouris |
|
1038 | bc223d91 | Stavros Sachtouris | :param upload_db: progress.bar for uploading
|
1039 | ee62607e | Stavros Sachtouris | """
|
1040 | 4067cdaf | Stavros Sachtouris | |
1041 | ca092af4 | Stavros Sachtouris | r = self.get_object_info(obj)
|
1042 | ca092af4 | Stavros Sachtouris | rf_size = int(r['content-length']) |
1043 | ca092af4 | Stavros Sachtouris | if rf_size < int(start): |
1044 | ca092af4 | Stavros Sachtouris | raise ClientError(
|
1045 | ca092af4 | Stavros Sachtouris | 'Range start exceeds file size',
|
1046 | ca092af4 | Stavros Sachtouris | status=416)
|
1047 | ca092af4 | Stavros Sachtouris | elif rf_size < int(end): |
1048 | ca092af4 | Stavros Sachtouris | raise ClientError(
|
1049 | ca092af4 | Stavros Sachtouris | 'Range end exceeds file size',
|
1050 | ca092af4 | Stavros Sachtouris | status=416)
|
1051 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
1052 | 2f749e6e | Stavros Sachtouris | meta = self.get_container_info()
|
1053 | ee62607e | Stavros Sachtouris | blocksize = int(meta['x-container-block-size']) |
1054 | 64ab4c13 | Stavros Sachtouris | filesize = fstat(source_file.fileno()).st_size |
1055 | ee62607e | Stavros Sachtouris | datasize = int(end) - int(start) + 1 |
1056 | 3dabe5d2 | Stavros Sachtouris | nblocks = 1 + (datasize - 1) // blocksize |
1057 | ee62607e | Stavros Sachtouris | offset = 0
|
1058 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
1059 | bcabbc35 | Stavros Sachtouris | upload_gen = upload_cb(nblocks) |
1060 | ca092af4 | Stavros Sachtouris | upload_gen.next() |
1061 | ee62607e | Stavros Sachtouris | for i in range(nblocks): |
1062 | 24ff0a35 | Stavros Sachtouris | read_size = min(blocksize, filesize - offset, datasize - offset)
|
1063 | 24ff0a35 | Stavros Sachtouris | block = source_file.read(read_size) |
1064 | c2b5da2f | Stavros Sachtouris | self.object_post(
|
1065 | 24ff0a35 | Stavros Sachtouris | obj, |
1066 | 3dabe5d2 | Stavros Sachtouris | update=True,
|
1067 | 3dabe5d2 | Stavros Sachtouris | content_type='application/octet-stream',
|
1068 | 3dabe5d2 | Stavros Sachtouris | content_length=len(block),
|
1069 | ca092af4 | Stavros Sachtouris | content_range='bytes %s-%s/*' % (
|
1070 | ca092af4 | Stavros Sachtouris | start + offset, |
1071 | ca092af4 | Stavros Sachtouris | start + offset + len(block) - 1), |
1072 | 3dabe5d2 | Stavros Sachtouris | data=block) |
1073 | ca092af4 | Stavros Sachtouris | offset += len(block)
|
1074 | 3dabe5d2 | Stavros Sachtouris | |
1075 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
1076 | bcabbc35 | Stavros Sachtouris | upload_gen.next() |
1077 | 7d420701 | Stavros Sachtouris | |
1078 | 24ff0a35 | Stavros Sachtouris | def copy_object( |
1079 | 2005b18e | Stavros Sachtouris | self, src_container, src_object, dst_container,
|
1080 | 33487500 | Stavros Sachtouris | dst_object=None,
|
1081 | 2005b18e | Stavros Sachtouris | source_version=None,
|
1082 | 3a066af4 | Stavros Sachtouris | source_account=None,
|
1083 | 2005b18e | Stavros Sachtouris | public=False,
|
1084 | 2005b18e | Stavros Sachtouris | content_type=None,
|
1085 | 2005b18e | Stavros Sachtouris | delimiter=None):
|
1086 | bc223d91 | Stavros Sachtouris | """
|
1087 | bc223d91 | Stavros Sachtouris | :param src_container: (str) source container
|
1088 | bc223d91 | Stavros Sachtouris |
|
1089 | bc223d91 | Stavros Sachtouris | :param src_object: (str) source object path
|
1090 | bc223d91 | Stavros Sachtouris |
|
1091 | bc223d91 | Stavros Sachtouris | :param dst_container: (str) destination container
|
1092 | bc223d91 | Stavros Sachtouris |
|
1093 | bc223d91 | Stavros Sachtouris | :param dst_object: (str) destination object path
|
1094 | bc223d91 | Stavros Sachtouris |
|
1095 | bc223d91 | Stavros Sachtouris | :param source_version: (str) source object version
|
1096 | bc223d91 | Stavros Sachtouris |
|
1097 | 3a066af4 | Stavros Sachtouris | :param source_account: (str) account to copy from
|
1098 | 3a066af4 | Stavros Sachtouris |
|
1099 | bc223d91 | Stavros Sachtouris | :param public: (bool)
|
1100 | bc223d91 | Stavros Sachtouris |
|
1101 | bc223d91 | Stavros Sachtouris | :param content_type: (str)
|
1102 | bc223d91 | Stavros Sachtouris |
|
1103 | bc223d91 | Stavros Sachtouris | :param delimiter: (str)
|
1104 | bc223d91 | Stavros Sachtouris | """
|
1105 | 277ca4ed | Dionysis Zindros | self._assert_account()
|
1106 | 7d420701 | Stavros Sachtouris | self.container = dst_container
|
1107 | 7d420701 | Stavros Sachtouris | src_path = path4url(src_container, src_object) |
1108 | c2b5da2f | Stavros Sachtouris | self.object_put(
|
1109 | 33487500 | Stavros Sachtouris | dst_object or src_object,
|
1110 | 3dabe5d2 | Stavros Sachtouris | success=201,
|
1111 | 3dabe5d2 | Stavros Sachtouris | copy_from=src_path, |
1112 | 3dabe5d2 | Stavros Sachtouris | content_length=0,
|
1113 | 3dabe5d2 | Stavros Sachtouris | source_version=source_version, |
1114 | 3a066af4 | Stavros Sachtouris | source_account=source_account, |
1115 | 3dabe5d2 | Stavros Sachtouris | public=public, |
1116 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
1117 | 7d420701 | Stavros Sachtouris | delimiter=delimiter) |
1118 | a5e0629d | Stavros Sachtouris | |
1119 | 24ff0a35 | Stavros Sachtouris | def move_object( |
1120 | 24ff0a35 | Stavros Sachtouris | self, src_container, src_object, dst_container,
|
1121 | 24ff0a35 | Stavros Sachtouris | dst_object=False,
|
1122 | 4f266635 | Stavros Sachtouris | source_account=None,
|
1123 | 24ff0a35 | Stavros Sachtouris | source_version=None,
|
1124 | 24ff0a35 | Stavros Sachtouris | public=False,
|
1125 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
1126 | 24ff0a35 | Stavros Sachtouris | delimiter=None):
|
1127 | bc223d91 | Stavros Sachtouris | """
|
1128 | bc223d91 | Stavros Sachtouris | :param src_container: (str) source container
|
1129 | bc223d91 | Stavros Sachtouris |
|
1130 | bc223d91 | Stavros Sachtouris | :param src_object: (str) source object path
|
1131 | bc223d91 | Stavros Sachtouris |
|
1132 | bc223d91 | Stavros Sachtouris | :param dst_container: (str) destination container
|
1133 | bc223d91 | Stavros Sachtouris |
|
1134 | bc223d91 | Stavros Sachtouris | :param dst_object: (str) destination object path
|
1135 | bc223d91 | Stavros Sachtouris |
|
1136 | 4f266635 | Stavros Sachtouris | :param source_account: (str) account to move from
|
1137 | 4f266635 | Stavros Sachtouris |
|
1138 | bc223d91 | Stavros Sachtouris | :param source_version: (str) source object version
|
1139 | bc223d91 | Stavros Sachtouris |
|
1140 | bc223d91 | Stavros Sachtouris | :param public: (bool)
|
1141 | bc223d91 | Stavros Sachtouris |
|
1142 | bc223d91 | Stavros Sachtouris | :param content_type: (str)
|
1143 | bc223d91 | Stavros Sachtouris |
|
1144 | bc223d91 | Stavros Sachtouris | :param delimiter: (str)
|
1145 | bc223d91 | Stavros Sachtouris | """
|
1146 | 277ca4ed | Dionysis Zindros | self._assert_account()
|
1147 | a5e0629d | Stavros Sachtouris | self.container = dst_container
|
1148 | a5e0629d | Stavros Sachtouris | dst_object = dst_object or src_object
|
1149 | a5e0629d | Stavros Sachtouris | src_path = path4url(src_container, src_object) |
1150 | c2b5da2f | Stavros Sachtouris | self.object_put(
|
1151 | 24ff0a35 | Stavros Sachtouris | dst_object, |
1152 | 3dabe5d2 | Stavros Sachtouris | success=201,
|
1153 | 3dabe5d2 | Stavros Sachtouris | move_from=src_path, |
1154 | 3dabe5d2 | Stavros Sachtouris | content_length=0,
|
1155 | 4f266635 | Stavros Sachtouris | source_account=source_account, |
1156 | 3dabe5d2 | Stavros Sachtouris | source_version=source_version, |
1157 | 3dabe5d2 | Stavros Sachtouris | public=public, |
1158 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
1159 | a23f6ffe | Stavros Sachtouris | delimiter=delimiter) |
1160 | a23f6ffe | Stavros Sachtouris | |
1161 | a23f6ffe | Stavros Sachtouris | def get_sharing_accounts(self, limit=None, marker=None, *args, **kwargs): |
1162 | bc223d91 | Stavros Sachtouris | """Get accounts that share with self.account
|
1163 | bc223d91 | Stavros Sachtouris |
|
1164 | bc223d91 | Stavros Sachtouris | :param limit: (str)
|
1165 | bc223d91 | Stavros Sachtouris |
|
1166 | bc223d91 | Stavros Sachtouris | :param marker: (str)
|
1167 | bc223d91 | Stavros Sachtouris |
|
1168 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
1169 | bc223d91 | Stavros Sachtouris | """
|
1170 | 277ca4ed | Dionysis Zindros | self._assert_account()
|
1171 | a23f6ffe | Stavros Sachtouris | |
1172 | 3dabe5d2 | Stavros Sachtouris | self.set_param('format', 'json') |
1173 | 3dabe5d2 | Stavros Sachtouris | self.set_param('limit', limit, iff=limit is not None) |
1174 | 3dabe5d2 | Stavros Sachtouris | self.set_param('marker', marker, iff=marker is not None) |
1175 | a23f6ffe | Stavros Sachtouris | |
1176 | a23f6ffe | Stavros Sachtouris | path = ''
|
1177 | a23f6ffe | Stavros Sachtouris | success = kwargs.pop('success', (200, 204)) |
1178 | 3dabe5d2 | Stavros Sachtouris | r = self.get(path, *args, success=success, **kwargs)
|
1179 | 38dc5d2f | Stavros Sachtouris | return r.json
|
1180 | 38dc5d2f | Stavros Sachtouris | |
1181 | bc223d91 | Stavros Sachtouris | def get_object_versionlist(self, obj): |
1182 | bc223d91 | Stavros Sachtouris | """
|
1183 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
1184 | bc223d91 | Stavros Sachtouris |
|
1185 | bc223d91 | Stavros Sachtouris | :returns: (list)
|
1186 | bc223d91 | Stavros Sachtouris | """
|
1187 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
1188 | bc223d91 | Stavros Sachtouris | r = self.object_get(obj, format='json', version='list') |
1189 | 38dc5d2f | Stavros Sachtouris | return r.json['versions'] |