root / kamaki / clients / pithos.py @ edd209d3
History | View | Annotate | Download (35.8 kB)
1 | fce31e83 | Stavros Sachtouris | # Copyright 2011-2013 GRNET S.A. All rights reserved.
|
---|---|---|---|
2 | d2cea1e2 | Giorgos Verigakis | #
|
3 | d2cea1e2 | Giorgos Verigakis | # Redistribution and use in source and binary forms, with or
|
4 | d2cea1e2 | Giorgos Verigakis | # without modification, are permitted provided that the following
|
5 | d2cea1e2 | Giorgos Verigakis | # conditions are met:
|
6 | d2cea1e2 | Giorgos Verigakis | #
|
7 | d2cea1e2 | Giorgos Verigakis | # 1. Redistributions of source code must retain the above
|
8 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
9 | d2cea1e2 | Giorgos Verigakis | # disclaimer.
|
10 | d2cea1e2 | Giorgos Verigakis | #
|
11 | d2cea1e2 | Giorgos Verigakis | # 2. Redistributions in binary form must reproduce the above
|
12 | d2cea1e2 | Giorgos Verigakis | # copyright notice, this list of conditions and the following
|
13 | d2cea1e2 | Giorgos Verigakis | # disclaimer in the documentation and/or other materials
|
14 | d2cea1e2 | Giorgos Verigakis | # provided with the distribution.
|
15 | d2cea1e2 | Giorgos Verigakis | #
|
16 | d2cea1e2 | Giorgos Verigakis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 | d2cea1e2 | Giorgos Verigakis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 | d2cea1e2 | Giorgos Verigakis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 | d2cea1e2 | Giorgos Verigakis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 | d2cea1e2 | Giorgos Verigakis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 | d2cea1e2 | Giorgos Verigakis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 | d2cea1e2 | Giorgos Verigakis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 | d2cea1e2 | Giorgos Verigakis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 | d2cea1e2 | Giorgos Verigakis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 | d2cea1e2 | Giorgos Verigakis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 | d2cea1e2 | Giorgos Verigakis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 | d2cea1e2 | Giorgos Verigakis | # POSSIBILITY OF SUCH DAMAGE.
|
28 | d2cea1e2 | Giorgos Verigakis | #
|
29 | d2cea1e2 | Giorgos Verigakis | # The views and conclusions contained in the software and
|
30 | d2cea1e2 | Giorgos Verigakis | # documentation are those of the authors and should not be
|
31 | d2cea1e2 | Giorgos Verigakis | # interpreted as representing official policies, either expressed
|
32 | d2cea1e2 | Giorgos Verigakis | # or implied, of GRNET S.A.
|
33 | d2cea1e2 | Giorgos Verigakis | |
34 | f27ed9a0 | Stavros Sachtouris | from threading import enumerate as activethreads |
35 | 435008b6 | Stavros Sachtouris | |
36 | 3dabe5d2 | Stavros Sachtouris | from os import fstat |
37 | 64ab4c13 | Stavros Sachtouris | from hashlib import new as newhashlib |
38 | e02728f9 | Stavros Sachtouris | from time import time |
39 | a91e0293 | Giorgos Verigakis | |
40 | 64ab4c13 | Stavros Sachtouris | from binascii import hexlify |
41 | 6a0b1658 | Giorgos Verigakis | |
42 | 6069b53b | Stavros Sachtouris | from kamaki.clients import SilentEvent, sendlog |
43 | c270fe96 | Stavros Sachtouris | from kamaki.clients.pithos_rest_api import PithosRestAPI |
44 | c270fe96 | Stavros Sachtouris | from kamaki.clients.storage import ClientError |
45 | c270fe96 | Stavros Sachtouris | from kamaki.clients.utils import path4url, filter_in |
46 | 1785ad41 | Stavros Sachtouris | from StringIO import StringIO |
47 | 6a0b1658 | Giorgos Verigakis | |
48 | 3dabe5d2 | Stavros Sachtouris | |
49 | 4375e020 | Stavros Sachtouris | def _pithos_hash(block, blockhash): |
50 | 64ab4c13 | Stavros Sachtouris | h = newhashlib(blockhash) |
51 | 6a0b1658 | Giorgos Verigakis | h.update(block.rstrip('\x00'))
|
52 | 6a0b1658 | Giorgos Verigakis | return h.hexdigest()
|
53 | 6a0b1658 | Giorgos Verigakis | |
54 | 3dabe5d2 | Stavros Sachtouris | |
55 | 642f1bbd | Stavros Sachtouris | def _range_up(start, end, a_range): |
56 | 642f1bbd | Stavros Sachtouris | if a_range:
|
57 | 642f1bbd | Stavros Sachtouris | (rstart, rend) = a_range.split('-')
|
58 | 642f1bbd | Stavros Sachtouris | (rstart, rend) = (int(rstart), int(rend)) |
59 | 642f1bbd | Stavros Sachtouris | if rstart > end or rend < start: |
60 | 3dabe5d2 | Stavros Sachtouris | return (0, 0) |
61 | 642f1bbd | Stavros Sachtouris | if rstart > start:
|
62 | 642f1bbd | Stavros Sachtouris | start = rstart |
63 | 642f1bbd | Stavros Sachtouris | if rend < end:
|
64 | 642f1bbd | Stavros Sachtouris | end = rend |
65 | 642f1bbd | Stavros Sachtouris | return (start, end)
|
66 | 642f1bbd | Stavros Sachtouris | |
67 | 3dabe5d2 | Stavros Sachtouris | |
68 | 64ab4c13 | Stavros Sachtouris | class PithosClient(PithosRestAPI): |
69 | d2cea1e2 | Giorgos Verigakis | """GRNet Pithos API client"""
|
70 | a91e0293 | Giorgos Verigakis | |
71 | e02728f9 | Stavros Sachtouris | _thread_exceptions = [] |
72 | e02728f9 | Stavros Sachtouris | |
73 | 3dabe5d2 | Stavros Sachtouris | def __init__(self, base_url, token, account=None, container=None): |
74 | 3dabe5d2 | Stavros Sachtouris | super(PithosClient, self).__init__(base_url, token, account, container) |
75 | 435008b6 | Stavros Sachtouris | |
76 | 2a7292f1 | Stavros Sachtouris | def purge_container(self, container=None): |
77 | 4375e020 | Stavros Sachtouris | """Delete an empty container and destroy associated blocks
|
78 | 4375e020 | Stavros Sachtouris | """
|
79 | 2a7292f1 | Stavros Sachtouris | cnt_back_up = self.container
|
80 | 2a7292f1 | Stavros Sachtouris | try:
|
81 | 2a7292f1 | Stavros Sachtouris | self.container = container or cnt_back_up |
82 | 2a7292f1 | Stavros Sachtouris | r = self.container_delete(until=unicode(time())) |
83 | 2a7292f1 | Stavros Sachtouris | finally:
|
84 | 2a7292f1 | Stavros Sachtouris | self.container = cnt_back_up
|
85 | 6ce9fc72 | Stavros Sachtouris | r.release() |
86 | 3dabe5d2 | Stavros Sachtouris | |
87 | 24ff0a35 | Stavros Sachtouris | def upload_object_unchunked( |
88 | 24ff0a35 | Stavros Sachtouris | self, obj, f,
|
89 | 24ff0a35 | Stavros Sachtouris | withHashFile=False,
|
90 | 24ff0a35 | Stavros Sachtouris | size=None,
|
91 | 24ff0a35 | Stavros Sachtouris | etag=None,
|
92 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
93 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
94 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
95 | 24ff0a35 | Stavros Sachtouris | sharing=None,
|
96 | 24ff0a35 | Stavros Sachtouris | public=None):
|
97 | 4375e020 | Stavros Sachtouris | """
|
98 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
99 | 4375e020 | Stavros Sachtouris |
|
100 | 4375e020 | Stavros Sachtouris | :param f: open file descriptor
|
101 | 4375e020 | Stavros Sachtouris |
|
102 | 4375e020 | Stavros Sachtouris | :param withHashFile: (bool)
|
103 | 4375e020 | Stavros Sachtouris |
|
104 | 4375e020 | Stavros Sachtouris | :param size: (int) size of data to upload
|
105 | 4375e020 | Stavros Sachtouris |
|
106 | 4375e020 | Stavros Sachtouris | :param etag: (str)
|
107 | 4375e020 | Stavros Sachtouris |
|
108 | 4375e020 | Stavros Sachtouris | :param content_encoding: (str)
|
109 | 4375e020 | Stavros Sachtouris |
|
110 | 4375e020 | Stavros Sachtouris | :param content_disposition: (str)
|
111 | 4375e020 | Stavros Sachtouris |
|
112 | 4375e020 | Stavros Sachtouris | :param content_type: (str)
|
113 | 4375e020 | Stavros Sachtouris |
|
114 | 4375e020 | Stavros Sachtouris | :param sharing: {'read':[user and/or grp names],
|
115 | 4375e020 | Stavros Sachtouris | 'write':[usr and/or grp names]}
|
116 | 4375e020 | Stavros Sachtouris |
|
117 | 4375e020 | Stavros Sachtouris | :param public: (bool)
|
118 | 4375e020 | Stavros Sachtouris | """
|
119 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
120 | 65a45524 | Stavros Sachtouris | |
121 | 65a45524 | Stavros Sachtouris | if withHashFile:
|
122 | 65a45524 | Stavros Sachtouris | data = f.read() |
123 | 65a45524 | Stavros Sachtouris | try:
|
124 | 65a45524 | Stavros Sachtouris | import json |
125 | 65a45524 | Stavros Sachtouris | data = json.dumps(json.loads(data)) |
126 | 65a45524 | Stavros Sachtouris | except ValueError: |
127 | 24ff0a35 | Stavros Sachtouris | raise ClientError('"%s" is not json-formated' % f.name, 1) |
128 | 65a45524 | Stavros Sachtouris | except SyntaxError: |
129 | 24ff0a35 | Stavros Sachtouris | msg = '"%s" is not a valid hashmap file' % f.name
|
130 | 24ff0a35 | Stavros Sachtouris | raise ClientError(msg, 1) |
131 | 65a45524 | Stavros Sachtouris | f = StringIO(data) |
132 | 2a7292f1 | Stavros Sachtouris | else:
|
133 | 2a7292f1 | Stavros Sachtouris | data = f.read(size) if size is not None else f.read() |
134 | 24ff0a35 | Stavros Sachtouris | r = self.object_put(
|
135 | 24ff0a35 | Stavros Sachtouris | obj, |
136 | 3dabe5d2 | Stavros Sachtouris | data=data, |
137 | 3dabe5d2 | Stavros Sachtouris | etag=etag, |
138 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
139 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
140 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
141 | 3dabe5d2 | Stavros Sachtouris | permissions=sharing, |
142 | 3dabe5d2 | Stavros Sachtouris | public=public, |
143 | 3dabe5d2 | Stavros Sachtouris | success=201)
|
144 | 6ce9fc72 | Stavros Sachtouris | r.release() |
145 | 3dabe5d2 | Stavros Sachtouris | |
146 | 24ff0a35 | Stavros Sachtouris | def create_object_by_manifestation( |
147 | 24ff0a35 | Stavros Sachtouris | self, obj,
|
148 | 24ff0a35 | Stavros Sachtouris | etag=None,
|
149 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
150 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
151 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
152 | 24ff0a35 | Stavros Sachtouris | sharing=None,
|
153 | 24ff0a35 | Stavros Sachtouris | public=None):
|
154 | 4375e020 | Stavros Sachtouris | """
|
155 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
156 | 4375e020 | Stavros Sachtouris |
|
157 | 4375e020 | Stavros Sachtouris | :param etag: (str)
|
158 | 4375e020 | Stavros Sachtouris |
|
159 | 4375e020 | Stavros Sachtouris | :param content_encoding: (str)
|
160 | 4375e020 | Stavros Sachtouris |
|
161 | 4375e020 | Stavros Sachtouris | :param content_disposition: (str)
|
162 | 4375e020 | Stavros Sachtouris |
|
163 | 4375e020 | Stavros Sachtouris | :param content_type: (str)
|
164 | 4375e020 | Stavros Sachtouris |
|
165 | 4375e020 | Stavros Sachtouris | :param sharing: {'read':[user and/or grp names],
|
166 | 4375e020 | Stavros Sachtouris | 'write':[usr and/or grp names]}
|
167 | 4375e020 | Stavros Sachtouris |
|
168 | 4375e020 | Stavros Sachtouris | :param public: (bool)
|
169 | 4375e020 | Stavros Sachtouris | """
|
170 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
171 | 24ff0a35 | Stavros Sachtouris | r = self.object_put(
|
172 | 24ff0a35 | Stavros Sachtouris | obj, |
173 | 3dabe5d2 | Stavros Sachtouris | content_length=0,
|
174 | 3dabe5d2 | Stavros Sachtouris | etag=etag, |
175 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
176 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
177 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
178 | 3dabe5d2 | Stavros Sachtouris | permissions=sharing, |
179 | 3dabe5d2 | Stavros Sachtouris | public=public, |
180 | 3dabe5d2 | Stavros Sachtouris | manifest='%s/%s' % (self.container, obj)) |
181 | 6ce9fc72 | Stavros Sachtouris | r.release() |
182 | 3dabe5d2 | Stavros Sachtouris | |
183 | 4375e020 | Stavros Sachtouris | # upload_* auxiliary methods
|
184 | 1d329d27 | Stavros Sachtouris | def _put_block_async(self, data, hash, upload_gen=None): |
185 | 4375e020 | Stavros Sachtouris | event = SilentEvent(method=self._put_block, data=data, hash=hash) |
186 | 4375e020 | Stavros Sachtouris | event.start() |
187 | 4375e020 | Stavros Sachtouris | return event
|
188 | 4375e020 | Stavros Sachtouris | |
189 | 4375e020 | Stavros Sachtouris | def _put_block(self, data, hash): |
190 | 24ff0a35 | Stavros Sachtouris | r = self.container_post(
|
191 | 24ff0a35 | Stavros Sachtouris | update=True,
|
192 | 4375e020 | Stavros Sachtouris | content_type='application/octet-stream',
|
193 | 4375e020 | Stavros Sachtouris | content_length=len(data),
|
194 | 4375e020 | Stavros Sachtouris | data=data, |
195 | 4375e020 | Stavros Sachtouris | format='json')
|
196 | 4375e020 | Stavros Sachtouris | assert r.json[0] == hash, 'Local hash does not match server' |
197 | 4375e020 | Stavros Sachtouris | |
198 | 64ab4c13 | Stavros Sachtouris | def _get_file_block_info(self, fileobj, size=None): |
199 | 2f749e6e | Stavros Sachtouris | meta = self.get_container_info()
|
200 | 435008b6 | Stavros Sachtouris | blocksize = int(meta['x-container-block-size']) |
201 | 435008b6 | Stavros Sachtouris | blockhash = meta['x-container-block-hash']
|
202 | 64ab4c13 | Stavros Sachtouris | size = size if size is not None else fstat(fileobj.fileno()).st_size |
203 | 435008b6 | Stavros Sachtouris | nblocks = 1 + (size - 1) // blocksize |
204 | 64ab4c13 | Stavros Sachtouris | return (blocksize, blockhash, size, nblocks)
|
205 | 64ab4c13 | Stavros Sachtouris | |
206 | 24ff0a35 | Stavros Sachtouris | def _get_missing_hashes( |
207 | 24ff0a35 | Stavros Sachtouris | self, obj, json,
|
208 | 24ff0a35 | Stavros Sachtouris | size=None,
|
209 | 24ff0a35 | Stavros Sachtouris | format='json',
|
210 | 24ff0a35 | Stavros Sachtouris | hashmap=True,
|
211 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
212 | 24ff0a35 | Stavros Sachtouris | etag=None,
|
213 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
214 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
215 | 24ff0a35 | Stavros Sachtouris | permissions=None,
|
216 | 24ff0a35 | Stavros Sachtouris | public=None,
|
217 | 24ff0a35 | Stavros Sachtouris | success=(201, 409)): |
218 | 24ff0a35 | Stavros Sachtouris | r = self.object_put(
|
219 | 24ff0a35 | Stavros Sachtouris | obj, |
220 | 3dabe5d2 | Stavros Sachtouris | format='json',
|
221 | 3dabe5d2 | Stavros Sachtouris | hashmap=True,
|
222 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
223 | 3dabe5d2 | Stavros Sachtouris | json=json, |
224 | 3dabe5d2 | Stavros Sachtouris | etag=etag, |
225 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
226 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
227 | 3dabe5d2 | Stavros Sachtouris | permissions=permissions, |
228 | 3dabe5d2 | Stavros Sachtouris | public=public, |
229 | 64ab4c13 | Stavros Sachtouris | success=success) |
230 | 64ab4c13 | Stavros Sachtouris | if r.status_code == 201: |
231 | 6ce9fc72 | Stavros Sachtouris | r.release() |
232 | 64ab4c13 | Stavros Sachtouris | return None |
233 | 64ab4c13 | Stavros Sachtouris | return r.json
|
234 | 435008b6 | Stavros Sachtouris | |
235 | bcbf3cf0 | Stavros Sachtouris | def _culculate_blocks_for_upload( |
236 | 2005b18e | Stavros Sachtouris | self, blocksize, blockhash, size, nblocks, hashes, hmap, fileobj,
|
237 | 2005b18e | Stavros Sachtouris | hash_cb=None):
|
238 | 3dabe5d2 | Stavros Sachtouris | offset = 0
|
239 | 435008b6 | Stavros Sachtouris | if hash_cb:
|
240 | 435008b6 | Stavros Sachtouris | hash_gen = hash_cb(nblocks) |
241 | 435008b6 | Stavros Sachtouris | hash_gen.next() |
242 | 435008b6 | Stavros Sachtouris | |
243 | 435008b6 | Stavros Sachtouris | for i in range(nblocks): |
244 | 64ab4c13 | Stavros Sachtouris | block = fileobj.read(min(blocksize, size - offset))
|
245 | 435008b6 | Stavros Sachtouris | bytes = len(block)
|
246 | 4375e020 | Stavros Sachtouris | hash = _pithos_hash(block, blockhash) |
247 | 435008b6 | Stavros Sachtouris | hashes.append(hash)
|
248 | 5b263ba2 | Stavros Sachtouris | hmap[hash] = (offset, bytes) |
249 | 435008b6 | Stavros Sachtouris | offset += bytes
|
250 | 435008b6 | Stavros Sachtouris | if hash_cb:
|
251 | 435008b6 | Stavros Sachtouris | hash_gen.next() |
252 | fce31e83 | Stavros Sachtouris | msg = 'Failed to calculate uploaded blocks:'
|
253 | bcbf3cf0 | Stavros Sachtouris | ' Offset and object size do not match'
|
254 | fce31e83 | Stavros Sachtouris | assert offset == size, msg
|
255 | 435008b6 | Stavros Sachtouris | |
256 | 3e7d1e0e | Stavros Sachtouris | def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None): |
257 | 3e7d1e0e | Stavros Sachtouris | """upload missing blocks asynchronously"""
|
258 | 435008b6 | Stavros Sachtouris | |
259 | cad39033 | Stavros Sachtouris | self._init_thread_limit()
|
260 | e9abe82b | Stavros Sachtouris | |
261 | 435008b6 | Stavros Sachtouris | flying = [] |
262 | 7644c38e | Stavros Sachtouris | failures = [] |
263 | 435008b6 | Stavros Sachtouris | for hash in missing: |
264 | 5b263ba2 | Stavros Sachtouris | offset, bytes = hmap[hash]
|
265 | 64ab4c13 | Stavros Sachtouris | fileobj.seek(offset) |
266 | 64ab4c13 | Stavros Sachtouris | data = fileobj.read(bytes)
|
267 | 1d329d27 | Stavros Sachtouris | r = self._put_block_async(data, hash, upload_gen) |
268 | 435008b6 | Stavros Sachtouris | flying.append(r) |
269 | 745d938b | Stavros Sachtouris | unfinished = self._watch_thread_limit(flying)
|
270 | 745d938b | Stavros Sachtouris | for thread in set(flying).difference(unfinished): |
271 | 7644c38e | Stavros Sachtouris | if thread.exception:
|
272 | 7644c38e | Stavros Sachtouris | failures.append(thread) |
273 | 24ff0a35 | Stavros Sachtouris | if isinstance( |
274 | 2005b18e | Stavros Sachtouris | thread.exception, |
275 | 2005b18e | Stavros Sachtouris | ClientError) and thread.exception.status == 502: |
276 | 2005b18e | Stavros Sachtouris | self.POOLSIZE = self._thread_limit |
277 | 7644c38e | Stavros Sachtouris | elif thread.isAlive():
|
278 | 745d938b | Stavros Sachtouris | flying.append(thread) |
279 | 3e7d1e0e | Stavros Sachtouris | elif upload_gen:
|
280 | 3e7d1e0e | Stavros Sachtouris | try:
|
281 | 3e7d1e0e | Stavros Sachtouris | upload_gen.next() |
282 | 3e7d1e0e | Stavros Sachtouris | except:
|
283 | 3e7d1e0e | Stavros Sachtouris | pass
|
284 | e02728f9 | Stavros Sachtouris | flying = unfinished |
285 | e02728f9 | Stavros Sachtouris | |
286 | e02728f9 | Stavros Sachtouris | for thread in flying: |
287 | e02728f9 | Stavros Sachtouris | thread.join() |
288 | 7644c38e | Stavros Sachtouris | if thread.exception:
|
289 | 7644c38e | Stavros Sachtouris | failures.append(thread) |
290 | 3e7d1e0e | Stavros Sachtouris | elif upload_gen:
|
291 | 3e7d1e0e | Stavros Sachtouris | try:
|
292 | 3e7d1e0e | Stavros Sachtouris | upload_gen.next() |
293 | 3e7d1e0e | Stavros Sachtouris | except:
|
294 | 3e7d1e0e | Stavros Sachtouris | pass
|
295 | 56f0908a | Stavros Sachtouris | |
296 | 7644c38e | Stavros Sachtouris | return [failure.kwargs['hash'] for failure in failures] |
297 | 3dabe5d2 | Stavros Sachtouris | |
298 | 24ff0a35 | Stavros Sachtouris | def upload_object( |
299 | 24ff0a35 | Stavros Sachtouris | self, obj, f,
|
300 | 24ff0a35 | Stavros Sachtouris | size=None,
|
301 | 24ff0a35 | Stavros Sachtouris | hash_cb=None,
|
302 | 24ff0a35 | Stavros Sachtouris | upload_cb=None,
|
303 | 24ff0a35 | Stavros Sachtouris | etag=None,
|
304 | 24ff0a35 | Stavros Sachtouris | content_encoding=None,
|
305 | 24ff0a35 | Stavros Sachtouris | content_disposition=None,
|
306 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
307 | 24ff0a35 | Stavros Sachtouris | sharing=None,
|
308 | 24ff0a35 | Stavros Sachtouris | public=None):
|
309 | 4375e020 | Stavros Sachtouris | """Upload an object using multiple connections (threads)
|
310 | 4375e020 | Stavros Sachtouris |
|
311 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
312 | 4375e020 | Stavros Sachtouris |
|
313 | 4375e020 | Stavros Sachtouris | :param f: open file descriptor (rb)
|
314 | 4375e020 | Stavros Sachtouris |
|
315 | 4375e020 | Stavros Sachtouris | :param hash_cb: optional progress.bar object for calculating hashes
|
316 | 4375e020 | Stavros Sachtouris |
|
317 | 4375e020 | Stavros Sachtouris | :param upload_cb: optional progress.bar object for uploading
|
318 | 4375e020 | Stavros Sachtouris |
|
319 | 4375e020 | Stavros Sachtouris | :param etag: (str)
|
320 | 4375e020 | Stavros Sachtouris |
|
321 | 4375e020 | Stavros Sachtouris | :param content_encoding: (str)
|
322 | 4375e020 | Stavros Sachtouris |
|
323 | 4375e020 | Stavros Sachtouris | :param content_disposition: (str)
|
324 | 4375e020 | Stavros Sachtouris |
|
325 | 4375e020 | Stavros Sachtouris | :param content_type: (str)
|
326 | 4375e020 | Stavros Sachtouris |
|
327 | 4375e020 | Stavros Sachtouris | :param sharing: {'read':[user and/or grp names],
|
328 | 4375e020 | Stavros Sachtouris | 'write':[usr and/or grp names]}
|
329 | 4375e020 | Stavros Sachtouris |
|
330 | 4375e020 | Stavros Sachtouris | :param public: (bool)
|
331 | 4375e020 | Stavros Sachtouris | """
|
332 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
333 | 56f0908a | Stavros Sachtouris | |
334 | 64ab4c13 | Stavros Sachtouris | #init
|
335 | 3dabe5d2 | Stavros Sachtouris | block_info = (blocksize, blockhash, size, nblocks) =\ |
336 | 3dabe5d2 | Stavros Sachtouris | self._get_file_block_info(f, size)
|
337 | 64ab4c13 | Stavros Sachtouris | (hashes, hmap, offset) = ([], {}, 0)
|
338 | bcbf3cf0 | Stavros Sachtouris | if not content_type: |
339 | 3dabe5d2 | Stavros Sachtouris | content_type = 'application/octet-stream'
|
340 | 64ab4c13 | Stavros Sachtouris | |
341 | bcbf3cf0 | Stavros Sachtouris | self._culculate_blocks_for_upload(
|
342 | 24ff0a35 | Stavros Sachtouris | *block_info, |
343 | 3dabe5d2 | Stavros Sachtouris | hashes=hashes, |
344 | 3dabe5d2 | Stavros Sachtouris | hmap=hmap, |
345 | 3dabe5d2 | Stavros Sachtouris | fileobj=f, |
346 | 64ab4c13 | Stavros Sachtouris | hash_cb=hash_cb) |
347 | 64ab4c13 | Stavros Sachtouris | |
348 | 64ab4c13 | Stavros Sachtouris | hashmap = dict(bytes=size, hashes=hashes)
|
349 | 24ff0a35 | Stavros Sachtouris | missing = self._get_missing_hashes(
|
350 | 24ff0a35 | Stavros Sachtouris | obj, hashmap, |
351 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
352 | 3dabe5d2 | Stavros Sachtouris | size=size, |
353 | 3dabe5d2 | Stavros Sachtouris | etag=etag, |
354 | 3dabe5d2 | Stavros Sachtouris | content_encoding=content_encoding, |
355 | 3dabe5d2 | Stavros Sachtouris | content_disposition=content_disposition, |
356 | 3dabe5d2 | Stavros Sachtouris | permissions=sharing, |
357 | 3dabe5d2 | Stavros Sachtouris | public=public) |
358 | 64ab4c13 | Stavros Sachtouris | |
359 | 64ab4c13 | Stavros Sachtouris | if missing is None: |
360 | 64ab4c13 | Stavros Sachtouris | return
|
361 | 1d329d27 | Stavros Sachtouris | |
362 | 3e7d1e0e | Stavros Sachtouris | if upload_cb:
|
363 | 3e7d1e0e | Stavros Sachtouris | upload_gen = upload_cb(len(missing))
|
364 | 745d938b | Stavros Sachtouris | for i in range(len(missing), len(hashmap['hashes']) + 1): |
365 | 745d938b | Stavros Sachtouris | try:
|
366 | 745d938b | Stavros Sachtouris | upload_gen.next() |
367 | 745d938b | Stavros Sachtouris | except:
|
368 | 745d938b | Stavros Sachtouris | upload_gen = None
|
369 | 3e7d1e0e | Stavros Sachtouris | else:
|
370 | 3e7d1e0e | Stavros Sachtouris | upload_gen = None
|
371 | 3e7d1e0e | Stavros Sachtouris | |
372 | 7eda693f | Stavros Sachtouris | retries = 7
|
373 | f27ed9a0 | Stavros Sachtouris | try:
|
374 | 7644c38e | Stavros Sachtouris | while retries:
|
375 | 7eda693f | Stavros Sachtouris | sendlog.info('%s blocks missing' % len(missing)) |
376 | 7644c38e | Stavros Sachtouris | num_of_blocks = len(missing)
|
377 | 7644c38e | Stavros Sachtouris | missing = self._upload_missing_blocks(
|
378 | 7644c38e | Stavros Sachtouris | missing, |
379 | 7644c38e | Stavros Sachtouris | hmap, |
380 | 7644c38e | Stavros Sachtouris | f, |
381 | 3e7d1e0e | Stavros Sachtouris | upload_gen) |
382 | 16c895db | Stavros Sachtouris | if missing:
|
383 | 16c895db | Stavros Sachtouris | if num_of_blocks == len(missing): |
384 | 16c895db | Stavros Sachtouris | retries -= 1
|
385 | 16c895db | Stavros Sachtouris | else:
|
386 | 16c895db | Stavros Sachtouris | num_of_blocks = len(missing)
|
387 | 7644c38e | Stavros Sachtouris | else:
|
388 | 7644c38e | Stavros Sachtouris | break
|
389 | 7644c38e | Stavros Sachtouris | if missing:
|
390 | 7644c38e | Stavros Sachtouris | raise ClientError(
|
391 | 7644c38e | Stavros Sachtouris | '%s blocks failed to upload' % len(missing), |
392 | 7644c38e | Stavros Sachtouris | status=800)
|
393 | f27ed9a0 | Stavros Sachtouris | except KeyboardInterrupt: |
394 | 6069b53b | Stavros Sachtouris | sendlog.info('- - - wait for threads to finish')
|
395 | f27ed9a0 | Stavros Sachtouris | for thread in activethreads(): |
396 | f27ed9a0 | Stavros Sachtouris | thread.join() |
397 | f27ed9a0 | Stavros Sachtouris | raise
|
398 | f27ed9a0 | Stavros Sachtouris | |
399 | f27ed9a0 | Stavros Sachtouris | r = self.object_put(
|
400 | f27ed9a0 | Stavros Sachtouris | obj, |
401 | 3dabe5d2 | Stavros Sachtouris | format='json',
|
402 | 3dabe5d2 | Stavros Sachtouris | hashmap=True,
|
403 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
404 | 3dabe5d2 | Stavros Sachtouris | json=hashmap, |
405 | 3dabe5d2 | Stavros Sachtouris | success=201)
|
406 | 6ce9fc72 | Stavros Sachtouris | r.release() |
407 | 3dabe5d2 | Stavros Sachtouris | |
408 | f27ed9a0 | Stavros Sachtouris | # download_* auxiliary methods
|
409 | fbfee225 | Stavros Sachtouris | def _get_remote_blocks_info(self, obj, **restargs): |
410 | 56f0908a | Stavros Sachtouris | #retrieve object hashmap
|
411 | 3dabe5d2 | Stavros Sachtouris | myrange = restargs.pop('data_range', None) |
412 | 319be41b | Stavros Sachtouris | hashmap = self.get_object_hashmap(obj, **restargs)
|
413 | 642f1bbd | Stavros Sachtouris | restargs['data_range'] = myrange
|
414 | 56f0908a | Stavros Sachtouris | blocksize = int(hashmap['block_size']) |
415 | 56f0908a | Stavros Sachtouris | blockhash = hashmap['block_hash']
|
416 | 56f0908a | Stavros Sachtouris | total_size = hashmap['bytes']
|
417 | fbfee225 | Stavros Sachtouris | #assert total_size/blocksize + 1 == len(hashmap['hashes'])
|
418 | 56f0908a | Stavros Sachtouris | map_dict = {} |
419 | fbfee225 | Stavros Sachtouris | for i, h in enumerate(hashmap['hashes']): |
420 | fbfee225 | Stavros Sachtouris | map_dict[h] = i |
421 | fbfee225 | Stavros Sachtouris | return (blocksize, blockhash, total_size, hashmap['hashes'], map_dict) |
422 | 64ab4c13 | Stavros Sachtouris | |
423 | 24ff0a35 | Stavros Sachtouris | def _dump_blocks_sync( |
424 | 24ff0a35 | Stavros Sachtouris | self, obj, remote_hashes, blocksize, total_size, dst, range, |
425 | 24ff0a35 | Stavros Sachtouris | **args): |
426 | fbfee225 | Stavros Sachtouris | for blockid, blockhash in enumerate(remote_hashes): |
427 | 24ff0a35 | Stavros Sachtouris | if blockhash:
|
428 | 24ff0a35 | Stavros Sachtouris | start = blocksize * blockid |
429 | 24ff0a35 | Stavros Sachtouris | is_last = start + blocksize > total_size |
430 | 24ff0a35 | Stavros Sachtouris | end = (total_size - 1) if is_last else (start + blocksize - 1) |
431 | 24ff0a35 | Stavros Sachtouris | (start, end) = _range_up(start, end, range)
|
432 | 24ff0a35 | Stavros Sachtouris | args['data_range'] = 'bytes=%s-%s' % (start, end) |
433 | 24ff0a35 | Stavros Sachtouris | r = self.object_get(obj, success=(200, 206), **args) |
434 | 24ff0a35 | Stavros Sachtouris | self._cb_next()
|
435 | 24ff0a35 | Stavros Sachtouris | dst.write(r.content) |
436 | 24ff0a35 | Stavros Sachtouris | dst.flush() |
437 | fbfee225 | Stavros Sachtouris | |
438 | 24ff0a35 | Stavros Sachtouris | def _get_block_async(self, obj, **args): |
439 | 24ff0a35 | Stavros Sachtouris | event = SilentEvent(self.object_get, obj, success=(200, 206), **args) |
440 | e02728f9 | Stavros Sachtouris | event.start() |
441 | e02728f9 | Stavros Sachtouris | return event
|
442 | fb0cd49a | Stavros Sachtouris | |
443 | 48c3782c | Stavros Sachtouris | def _hash_from_file(self, fp, start, size, blockhash): |
444 | 48c3782c | Stavros Sachtouris | fp.seek(start) |
445 | 48c3782c | Stavros Sachtouris | block = fp.read(size) |
446 | 48c3782c | Stavros Sachtouris | h = newhashlib(blockhash) |
447 | 48c3782c | Stavros Sachtouris | h.update(block.strip('\x00'))
|
448 | 48c3782c | Stavros Sachtouris | return hexlify(h.digest())
|
449 | 48c3782c | Stavros Sachtouris | |
450 | 24ff0a35 | Stavros Sachtouris | def _thread2file(self, flying, local_file, offset=0, **restargs): |
451 | 22fc09fb | Stavros Sachtouris | """write the results of a greenleted rest call to a file
|
452 | f5f2dc53 | Stavros Sachtouris |
|
453 | f5f2dc53 | Stavros Sachtouris | :param offset: the offset of the file up to blocksize
|
454 | f5f2dc53 | Stavros Sachtouris | - e.g. if the range is 10-100, all blocks will be written to
|
455 | f5f2dc53 | Stavros Sachtouris | normal_position - 10
|
456 | f5f2dc53 | Stavros Sachtouris | """
|
457 | fbfee225 | Stavros Sachtouris | finished = [] |
458 | f27ed9a0 | Stavros Sachtouris | for i, (start, g) in enumerate(flying.items()): |
459 | e02728f9 | Stavros Sachtouris | if not g.isAlive(): |
460 | fbfee225 | Stavros Sachtouris | if g.exception:
|
461 | fbfee225 | Stavros Sachtouris | raise g.exception
|
462 | 1785ad41 | Stavros Sachtouris | block = g.value.content |
463 | 22fc09fb | Stavros Sachtouris | local_file.seek(start - offset) |
464 | fbfee225 | Stavros Sachtouris | local_file.write(block) |
465 | fbfee225 | Stavros Sachtouris | self._cb_next()
|
466 | e02728f9 | Stavros Sachtouris | finished.append(flying.pop(start)) |
467 | fbfee225 | Stavros Sachtouris | local_file.flush() |
468 | fbfee225 | Stavros Sachtouris | return finished
|
469 | fbfee225 | Stavros Sachtouris | |
470 | 24ff0a35 | Stavros Sachtouris | def _dump_blocks_async( |
471 | 24ff0a35 | Stavros Sachtouris | self, obj, remote_hashes, blocksize, total_size, local_file,
|
472 | 24ff0a35 | Stavros Sachtouris | blockhash=None, resume=False, filerange=None, **restargs): |
473 | 973fbcad | Stavros Sachtouris | file_size = fstat(local_file.fileno()).st_size if resume else 0 |
474 | e02728f9 | Stavros Sachtouris | flying = {} |
475 | e02728f9 | Stavros Sachtouris | finished = [] |
476 | 22fc09fb | Stavros Sachtouris | offset = 0
|
477 | 22fc09fb | Stavros Sachtouris | if filerange is not None: |
478 | 22fc09fb | Stavros Sachtouris | rstart = int(filerange.split('-')[0]) |
479 | 3dabe5d2 | Stavros Sachtouris | offset = rstart if blocksize > rstart else rstart % blocksize |
480 | f27ed9a0 | Stavros Sachtouris | |
481 | cad39033 | Stavros Sachtouris | self._init_thread_limit()
|
482 | fbfee225 | Stavros Sachtouris | for block_hash, blockid in remote_hashes.items(): |
483 | 3dabe5d2 | Stavros Sachtouris | start = blocksize * blockid |
484 | 24ff0a35 | Stavros Sachtouris | if start < file_size and block_hash == self._hash_from_file( |
485 | 24ff0a35 | Stavros Sachtouris | local_file, start, blocksize, blockhash): |
486 | 3dabe5d2 | Stavros Sachtouris | self._cb_next()
|
487 | 3dabe5d2 | Stavros Sachtouris | continue
|
488 | cad39033 | Stavros Sachtouris | self._watch_thread_limit(flying.values())
|
489 | f27ed9a0 | Stavros Sachtouris | finished += self._thread2file(
|
490 | f27ed9a0 | Stavros Sachtouris | flying, |
491 | f27ed9a0 | Stavros Sachtouris | local_file, |
492 | f27ed9a0 | Stavros Sachtouris | offset, |
493 | f27ed9a0 | Stavros Sachtouris | **restargs) |
494 | 3dabe5d2 | Stavros Sachtouris | end = total_size - 1 if start + blocksize > total_size\ |
495 | 3dabe5d2 | Stavros Sachtouris | else start + blocksize - 1 |
496 | 22fc09fb | Stavros Sachtouris | (start, end) = _range_up(start, end, filerange) |
497 | 22fc09fb | Stavros Sachtouris | if start == end:
|
498 | 22fc09fb | Stavros Sachtouris | self._cb_next()
|
499 | 22fc09fb | Stavros Sachtouris | continue
|
500 | 3dabe5d2 | Stavros Sachtouris | restargs['async_headers'] = {'Range': 'bytes=%s-%s' % (start, end)} |
501 | e02728f9 | Stavros Sachtouris | flying[start] = self._get_block_async(obj, **restargs)
|
502 | fbfee225 | Stavros Sachtouris | |
503 | e02728f9 | Stavros Sachtouris | for thread in flying.values(): |
504 | e02728f9 | Stavros Sachtouris | thread.join() |
505 | e02728f9 | Stavros Sachtouris | finished += self._thread2file(flying, local_file, offset, **restargs)
|
506 | fbfee225 | Stavros Sachtouris | |
507 | 24ff0a35 | Stavros Sachtouris | def download_object( |
508 | 24ff0a35 | Stavros Sachtouris | self, obj, dst,
|
509 | 24ff0a35 | Stavros Sachtouris | download_cb=None,
|
510 | 24ff0a35 | Stavros Sachtouris | version=None,
|
511 | 24ff0a35 | Stavros Sachtouris | resume=False,
|
512 | 24ff0a35 | Stavros Sachtouris | range_str=None,
|
513 | 24ff0a35 | Stavros Sachtouris | if_match=None,
|
514 | 24ff0a35 | Stavros Sachtouris | if_none_match=None,
|
515 | 24ff0a35 | Stavros Sachtouris | if_modified_since=None,
|
516 | 24ff0a35 | Stavros Sachtouris | if_unmodified_since=None):
|
517 | f5f2dc53 | Stavros Sachtouris | """Download an object (multiple connections, random blocks)
|
518 | 4375e020 | Stavros Sachtouris |
|
519 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
520 | 4375e020 | Stavros Sachtouris |
|
521 | 4375e020 | Stavros Sachtouris | :param dst: open file descriptor (wb+)
|
522 | 4375e020 | Stavros Sachtouris |
|
523 | 4375e020 | Stavros Sachtouris | :param download_cb: optional progress.bar object for downloading
|
524 | 4375e020 | Stavros Sachtouris |
|
525 | 4375e020 | Stavros Sachtouris | :param version: (str) file version
|
526 | 4375e020 | Stavros Sachtouris |
|
527 | 4375e020 | Stavros Sachtouris | :param resume: (bool) if set, preserve already downloaded file parts
|
528 | 4375e020 | Stavros Sachtouris |
|
529 | f5f2dc53 | Stavros Sachtouris | :param range_str: (str) from, to are file positions (int) in bytes
|
530 | 4375e020 | Stavros Sachtouris |
|
531 | 4375e020 | Stavros Sachtouris | :param if_match: (str)
|
532 | 4375e020 | Stavros Sachtouris |
|
533 | 4375e020 | Stavros Sachtouris | :param if_none_match: (str)
|
534 | 4375e020 | Stavros Sachtouris |
|
535 | 4375e020 | Stavros Sachtouris | :param if_modified_since: (str) formated date
|
536 | 4375e020 | Stavros Sachtouris |
|
537 | f5f2dc53 | Stavros Sachtouris | :param if_unmodified_since: (str) formated date"""
|
538 | 24ff0a35 | Stavros Sachtouris | restargs = dict(
|
539 | 24ff0a35 | Stavros Sachtouris | version=version, |
540 | 24ff0a35 | Stavros Sachtouris | data_range=None if range_str is None else 'bytes=%s' % range_str, |
541 | fbfee225 | Stavros Sachtouris | if_match=if_match, |
542 | fbfee225 | Stavros Sachtouris | if_none_match=if_none_match, |
543 | fbfee225 | Stavros Sachtouris | if_modified_since=if_modified_since, |
544 | fbfee225 | Stavros Sachtouris | if_unmodified_since=if_unmodified_since) |
545 | fbfee225 | Stavros Sachtouris | |
546 | 24ff0a35 | Stavros Sachtouris | ( |
547 | 24ff0a35 | Stavros Sachtouris | blocksize, |
548 | fbfee225 | Stavros Sachtouris | blockhash, |
549 | fbfee225 | Stavros Sachtouris | total_size, |
550 | 3dabe5d2 | Stavros Sachtouris | hash_list, |
551 | fbfee225 | Stavros Sachtouris | remote_hashes) = self._get_remote_blocks_info(obj, **restargs)
|
552 | fbfee225 | Stavros Sachtouris | assert total_size >= 0 |
553 | fbfee225 | Stavros Sachtouris | |
554 | fbfee225 | Stavros Sachtouris | if download_cb:
|
555 | 20c13fcc | Stavros Sachtouris | self.progress_bar_gen = download_cb(len(remote_hashes)) |
556 | fbfee225 | Stavros Sachtouris | self._cb_next()
|
557 | fbfee225 | Stavros Sachtouris | |
558 | fbfee225 | Stavros Sachtouris | if dst.isatty():
|
559 | 24ff0a35 | Stavros Sachtouris | self._dump_blocks_sync(
|
560 | 24ff0a35 | Stavros Sachtouris | obj, |
561 | 3dabe5d2 | Stavros Sachtouris | hash_list, |
562 | 3dabe5d2 | Stavros Sachtouris | blocksize, |
563 | 3dabe5d2 | Stavros Sachtouris | total_size, |
564 | 3dabe5d2 | Stavros Sachtouris | dst, |
565 | 24ff0a35 | Stavros Sachtouris | range_str, |
566 | 3dabe5d2 | Stavros Sachtouris | **restargs) |
567 | 699d3bb1 | Stavros Sachtouris | else:
|
568 | 24ff0a35 | Stavros Sachtouris | self._dump_blocks_async(
|
569 | 24ff0a35 | Stavros Sachtouris | obj, |
570 | 3dabe5d2 | Stavros Sachtouris | remote_hashes, |
571 | 3dabe5d2 | Stavros Sachtouris | blocksize, |
572 | 3dabe5d2 | Stavros Sachtouris | total_size, |
573 | 3dabe5d2 | Stavros Sachtouris | dst, |
574 | 3dabe5d2 | Stavros Sachtouris | blockhash, |
575 | 3dabe5d2 | Stavros Sachtouris | resume, |
576 | 24ff0a35 | Stavros Sachtouris | range_str, |
577 | 3dabe5d2 | Stavros Sachtouris | **restargs) |
578 | 24ff0a35 | Stavros Sachtouris | if not range_str: |
579 | 22fc09fb | Stavros Sachtouris | dst.truncate(total_size) |
580 | 5b263ba2 | Stavros Sachtouris | |
581 | fbfee225 | Stavros Sachtouris | self._complete_cb()
|
582 | fbfee225 | Stavros Sachtouris | |
583 | fbfee225 | Stavros Sachtouris | #Command Progress Bar method
|
584 | fbfee225 | Stavros Sachtouris | def _cb_next(self): |
585 | fbfee225 | Stavros Sachtouris | if hasattr(self, 'progress_bar_gen'): |
586 | fbfee225 | Stavros Sachtouris | try:
|
587 | fbfee225 | Stavros Sachtouris | self.progress_bar_gen.next()
|
588 | fbfee225 | Stavros Sachtouris | except:
|
589 | fbfee225 | Stavros Sachtouris | pass
|
590 | 3dabe5d2 | Stavros Sachtouris | |
591 | fbfee225 | Stavros Sachtouris | def _complete_cb(self): |
592 | fbfee225 | Stavros Sachtouris | while True: |
593 | fbfee225 | Stavros Sachtouris | try:
|
594 | fbfee225 | Stavros Sachtouris | self.progress_bar_gen.next()
|
595 | fbfee225 | Stavros Sachtouris | except:
|
596 | fbfee225 | Stavros Sachtouris | break
|
597 | b1713259 | Stavros Sachtouris | |
598 | 24ff0a35 | Stavros Sachtouris | def get_object_hashmap( |
599 | 24ff0a35 | Stavros Sachtouris | self, obj,
|
600 | 24ff0a35 | Stavros Sachtouris | version=None,
|
601 | 24ff0a35 | Stavros Sachtouris | if_match=None,
|
602 | 24ff0a35 | Stavros Sachtouris | if_none_match=None,
|
603 | 24ff0a35 | Stavros Sachtouris | if_modified_since=None,
|
604 | 24ff0a35 | Stavros Sachtouris | if_unmodified_since=None,
|
605 | 24ff0a35 | Stavros Sachtouris | data_range=None):
|
606 | 4375e020 | Stavros Sachtouris | """
|
607 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
608 | 4375e020 | Stavros Sachtouris |
|
609 | 4375e020 | Stavros Sachtouris | :param if_match: (str)
|
610 | 4375e020 | Stavros Sachtouris |
|
611 | 4375e020 | Stavros Sachtouris | :param if_none_match: (str)
|
612 | 4375e020 | Stavros Sachtouris |
|
613 | 4375e020 | Stavros Sachtouris | :param if_modified_since: (str) formated date
|
614 | 4375e020 | Stavros Sachtouris |
|
615 | 4375e020 | Stavros Sachtouris | :param if_unmodified_since: (str) formated date
|
616 | 4375e020 | Stavros Sachtouris |
|
617 | 4375e020 | Stavros Sachtouris | :param data_range: (str) from-to where from and to are integers
|
618 | 4375e020 | Stavros Sachtouris | denoting file positions in bytes
|
619 | 4375e020 | Stavros Sachtouris |
|
620 | 4375e020 | Stavros Sachtouris | :returns: (list)
|
621 | 4375e020 | Stavros Sachtouris | """
|
622 | d804de82 | Stavros Sachtouris | try:
|
623 | 24ff0a35 | Stavros Sachtouris | r = self.object_get(
|
624 | 24ff0a35 | Stavros Sachtouris | obj, |
625 | 3dabe5d2 | Stavros Sachtouris | hashmap=True,
|
626 | 3dabe5d2 | Stavros Sachtouris | version=version, |
627 | 3dabe5d2 | Stavros Sachtouris | if_etag_match=if_match, |
628 | 3dabe5d2 | Stavros Sachtouris | if_etag_not_match=if_none_match, |
629 | 3dabe5d2 | Stavros Sachtouris | if_modified_since=if_modified_since, |
630 | 3dabe5d2 | Stavros Sachtouris | if_unmodified_since=if_unmodified_since, |
631 | 3dabe5d2 | Stavros Sachtouris | data_range=data_range) |
632 | d804de82 | Stavros Sachtouris | except ClientError as err: |
633 | d804de82 | Stavros Sachtouris | if err.status == 304 or err.status == 412: |
634 | d804de82 | Stavros Sachtouris | return {}
|
635 | d804de82 | Stavros Sachtouris | raise
|
636 | 64ab4c13 | Stavros Sachtouris | return r.json
|
637 | 56f0908a | Stavros Sachtouris | |
638 | 3a9e54b0 | Stavros Sachtouris | def set_account_group(self, group, usernames): |
639 | 4375e020 | Stavros Sachtouris | """
|
640 | 4375e020 | Stavros Sachtouris | :param group: (str)
|
641 | 4375e020 | Stavros Sachtouris |
|
642 | 4375e020 | Stavros Sachtouris | :param usernames: (list)
|
643 | 4375e020 | Stavros Sachtouris | """
|
644 | 3dabe5d2 | Stavros Sachtouris | r = self.account_post(update=True, groups={group: usernames}) |
645 | 6ce9fc72 | Stavros Sachtouris | r.release() |
646 | eb903ba7 | Stavros Sachtouris | |
647 | c2867610 | Stavros Sachtouris | def del_account_group(self, group): |
648 | 4375e020 | Stavros Sachtouris | """
|
649 | 4375e020 | Stavros Sachtouris | :param group: (str)
|
650 | 4375e020 | Stavros Sachtouris | """
|
651 | 3dabe5d2 | Stavros Sachtouris | r = self.account_post(update=True, groups={group: []}) |
652 | 6ce9fc72 | Stavros Sachtouris | r.release() |
653 | c2867610 | Stavros Sachtouris | |
654 | 8af4cc0b | Stavros Sachtouris | def get_account_info(self, until=None): |
655 | 4375e020 | Stavros Sachtouris | """
|
656 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
657 | 4375e020 | Stavros Sachtouris |
|
658 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
659 | 4375e020 | Stavros Sachtouris | """
|
660 | 8af4cc0b | Stavros Sachtouris | r = self.account_head(until=until)
|
661 | 6657ec8c | Stavros Sachtouris | if r.status_code == 401: |
662 | 24851aa5 | Stavros Sachtouris | raise ClientError("No authorization", status=401) |
663 | 64ab4c13 | Stavros Sachtouris | return r.headers
|
664 | 6657ec8c | Stavros Sachtouris | |
665 | d1856abf | Stavros Sachtouris | def get_account_quota(self): |
666 | 4375e020 | Stavros Sachtouris | """
|
667 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
668 | 4375e020 | Stavros Sachtouris | """
|
669 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
670 | 24ff0a35 | Stavros Sachtouris | self.get_account_info(),
|
671 | 3dabe5d2 | Stavros Sachtouris | 'X-Account-Policy-Quota',
|
672 | 3dabe5d2 | Stavros Sachtouris | exactMatch=True)
|
673 | d1856abf | Stavros Sachtouris | |
674 | d1856abf | Stavros Sachtouris | def get_account_versioning(self): |
675 | 4375e020 | Stavros Sachtouris | """
|
676 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
677 | 4375e020 | Stavros Sachtouris | """
|
678 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
679 | 24ff0a35 | Stavros Sachtouris | self.get_account_info(),
|
680 | 3dabe5d2 | Stavros Sachtouris | 'X-Account-Policy-Versioning',
|
681 | 3dabe5d2 | Stavros Sachtouris | exactMatch=True)
|
682 | e6b39366 | Stavros Sachtouris | |
683 | 8af4cc0b | Stavros Sachtouris | def get_account_meta(self, until=None): |
684 | 4375e020 | Stavros Sachtouris | """
|
685 | 4375e020 | Stavros Sachtouris | :meta until: (str) formated date
|
686 | 4375e020 | Stavros Sachtouris |
|
687 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
688 | 4375e020 | Stavros Sachtouris | """
|
689 | 3dabe5d2 | Stavros Sachtouris | return filter_in(self.get_account_info(until=until), 'X-Account-Meta-') |
690 | af3b2b36 | Stavros Sachtouris | |
691 | c2867610 | Stavros Sachtouris | def get_account_group(self): |
692 | 4375e020 | Stavros Sachtouris | """
|
693 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
694 | 4375e020 | Stavros Sachtouris | """
|
695 | c2867610 | Stavros Sachtouris | return filter_in(self.get_account_info(), 'X-Account-Group-') |
696 | c2867610 | Stavros Sachtouris | |
697 | af3b2b36 | Stavros Sachtouris | def set_account_meta(self, metapairs): |
698 | 4375e020 | Stavros Sachtouris | """
|
699 | 4375e020 | Stavros Sachtouris | :param metapairs: (dict) {key1:val1, key2:val2, ...}
|
700 | 4375e020 | Stavros Sachtouris | """
|
701 | af3b2b36 | Stavros Sachtouris | assert(type(metapairs) is dict) |
702 | 6ce9fc72 | Stavros Sachtouris | r = self.account_post(update=True, metadata=metapairs) |
703 | 6ce9fc72 | Stavros Sachtouris | r.release() |
704 | af3b2b36 | Stavros Sachtouris | |
705 | 379cd4bb | Stavros Sachtouris | def del_account_meta(self, metakey): |
706 | 4375e020 | Stavros Sachtouris | """
|
707 | 4375e020 | Stavros Sachtouris | :param metakey: (str) metadatum key
|
708 | 4375e020 | Stavros Sachtouris | """
|
709 | 3dabe5d2 | Stavros Sachtouris | r = self.account_post(update=True, metadata={metakey: ''}) |
710 | 6ce9fc72 | Stavros Sachtouris | r.release() |
711 | 379cd4bb | Stavros Sachtouris | |
712 | d1856abf | Stavros Sachtouris | def set_account_quota(self, quota): |
713 | 4375e020 | Stavros Sachtouris | """
|
714 | 4375e020 | Stavros Sachtouris | :param quota: (int)
|
715 | 4375e020 | Stavros Sachtouris | """
|
716 | 6ce9fc72 | Stavros Sachtouris | r = self.account_post(update=True, quota=quota) |
717 | 6ce9fc72 | Stavros Sachtouris | r.release() |
718 | d1856abf | Stavros Sachtouris | |
719 | d1856abf | Stavros Sachtouris | def set_account_versioning(self, versioning): |
720 | 4375e020 | Stavros Sachtouris | """
|
721 | 4375e020 | Stavros Sachtouris | "param versioning: (str)
|
722 | 4375e020 | Stavros Sachtouris | """
|
723 | 3dabe5d2 | Stavros Sachtouris | r = self.account_post(update=True, versioning=versioning) |
724 | 6ce9fc72 | Stavros Sachtouris | r.release() |
725 | d1856abf | Stavros Sachtouris | |
726 | 4fd88feb | Stavros Sachtouris | def list_containers(self): |
727 | 4375e020 | Stavros Sachtouris | """
|
728 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
729 | 4375e020 | Stavros Sachtouris | """
|
730 | 4fd88feb | Stavros Sachtouris | r = self.account_get()
|
731 | 64ab4c13 | Stavros Sachtouris | return r.json
|
732 | b758e547 | Stavros Sachtouris | |
733 | a298f2ab | Stavros Sachtouris | def del_container(self, until=None, delimiter=None): |
734 | 4375e020 | Stavros Sachtouris | """
|
735 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
736 | 4375e020 | Stavros Sachtouris |
|
737 | 4c1882ab | Stavros Sachtouris | :param delimiter: (str) with / empty container
|
738 | 4375e020 | Stavros Sachtouris |
|
739 | 4375e020 | Stavros Sachtouris | :raises ClientError: 404 Container does not exist
|
740 | 4375e020 | Stavros Sachtouris |
|
741 | 4375e020 | Stavros Sachtouris | :raises ClientError: 409 Container is not empty
|
742 | 4375e020 | Stavros Sachtouris | """
|
743 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
744 | 2005b18e | Stavros Sachtouris | r = self.container_delete(
|
745 | 2005b18e | Stavros Sachtouris | until=until, |
746 | 3dabe5d2 | Stavros Sachtouris | delimiter=delimiter, |
747 | 3dabe5d2 | Stavros Sachtouris | success=(204, 404, 409)) |
748 | 6ce9fc72 | Stavros Sachtouris | r.release() |
749 | a298f2ab | Stavros Sachtouris | if r.status_code == 404: |
750 | 24ff0a35 | Stavros Sachtouris | raise ClientError(
|
751 | 24ff0a35 | Stavros Sachtouris | 'Container "%s" does not exist' % self.container, |
752 | 3dabe5d2 | Stavros Sachtouris | r.status_code) |
753 | a298f2ab | Stavros Sachtouris | elif r.status_code == 409: |
754 | 24ff0a35 | Stavros Sachtouris | raise ClientError(
|
755 | 24ff0a35 | Stavros Sachtouris | 'Container "%s" is not empty' % self.container, |
756 | 3dabe5d2 | Stavros Sachtouris | r.status_code) |
757 | a298f2ab | Stavros Sachtouris | |
758 | d1856abf | Stavros Sachtouris | def get_container_versioning(self, container): |
759 | 4375e020 | Stavros Sachtouris | """
|
760 | 4375e020 | Stavros Sachtouris | :param container: (str)
|
761 | 4375e020 | Stavros Sachtouris |
|
762 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
763 | 4375e020 | Stavros Sachtouris | """
|
764 | 2f749e6e | Stavros Sachtouris | self.container = container
|
765 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
766 | 24ff0a35 | Stavros Sachtouris | self.get_container_info(),
|
767 | 3dabe5d2 | Stavros Sachtouris | 'X-Container-Policy-Versioning')
|
768 | d1856abf | Stavros Sachtouris | |
769 | d1856abf | Stavros Sachtouris | def get_container_quota(self, container): |
770 | 4375e020 | Stavros Sachtouris | """
|
771 | 4375e020 | Stavros Sachtouris | :param container: (str)
|
772 | 4375e020 | Stavros Sachtouris |
|
773 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
774 | 4375e020 | Stavros Sachtouris | """
|
775 | 2f749e6e | Stavros Sachtouris | self.container = container
|
776 | 2f749e6e | Stavros Sachtouris | return filter_in(self.get_container_info(), 'X-Container-Policy-Quota') |
777 | e6b39366 | Stavros Sachtouris | |
778 | 3dabe5d2 | Stavros Sachtouris | def get_container_info(self, until=None): |
779 | 4375e020 | Stavros Sachtouris | """
|
780 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
781 | 4375e020 | Stavros Sachtouris |
|
782 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
783 | f91bc6b1 | Stavros Sachtouris |
|
784 | f91bc6b1 | Stavros Sachtouris | :raises ClientError: 404 Container not found
|
785 | 4375e020 | Stavros Sachtouris | """
|
786 | f91bc6b1 | Stavros Sachtouris | try:
|
787 | f91bc6b1 | Stavros Sachtouris | r = self.container_head(until=until)
|
788 | f91bc6b1 | Stavros Sachtouris | except ClientError as err: |
789 | f91bc6b1 | Stavros Sachtouris | err.details.append('for container %s' % self.container) |
790 | f91bc6b1 | Stavros Sachtouris | raise err
|
791 | 8af4cc0b | Stavros Sachtouris | return r.headers
|
792 | 8af4cc0b | Stavros Sachtouris | |
793 | 3dabe5d2 | Stavros Sachtouris | def get_container_meta(self, until=None): |
794 | 4375e020 | Stavros Sachtouris | """
|
795 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
796 | 4375e020 | Stavros Sachtouris |
|
797 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
798 | 4375e020 | Stavros Sachtouris | """
|
799 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
800 | 24ff0a35 | Stavros Sachtouris | self.get_container_info(until=until),
|
801 | 3dabe5d2 | Stavros Sachtouris | 'X-Container-Meta')
|
802 | e6b39366 | Stavros Sachtouris | |
803 | 3dabe5d2 | Stavros Sachtouris | def get_container_object_meta(self, until=None): |
804 | 4375e020 | Stavros Sachtouris | """
|
805 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
806 | 4375e020 | Stavros Sachtouris |
|
807 | 4375e020 | Stavros Sachtouris | :returns: (dict)
|
808 | 4375e020 | Stavros Sachtouris | """
|
809 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
810 | 24ff0a35 | Stavros Sachtouris | self.get_container_info(until=until),
|
811 | 3dabe5d2 | Stavros Sachtouris | 'X-Container-Object-Meta')
|
812 | e6b39366 | Stavros Sachtouris | |
813 | af3b2b36 | Stavros Sachtouris | def set_container_meta(self, metapairs): |
814 | 4375e020 | Stavros Sachtouris | """
|
815 | 4375e020 | Stavros Sachtouris | :param metapairs: (dict) {key1:val1, key2:val2, ...}
|
816 | 4375e020 | Stavros Sachtouris | """
|
817 | af3b2b36 | Stavros Sachtouris | assert(type(metapairs) is dict) |
818 | 6ce9fc72 | Stavros Sachtouris | r = self.container_post(update=True, metadata=metapairs) |
819 | 6ce9fc72 | Stavros Sachtouris | r.release() |
820 | 3dabe5d2 | Stavros Sachtouris | |
821 | 3e544e5b | Stavros Sachtouris | def del_container_meta(self, metakey): |
822 | 4375e020 | Stavros Sachtouris | """
|
823 | 4375e020 | Stavros Sachtouris | :param metakey: (str) metadatum key
|
824 | 4375e020 | Stavros Sachtouris | """
|
825 | 3dabe5d2 | Stavros Sachtouris | r = self.container_post(update=True, metadata={metakey: ''}) |
826 | 6ce9fc72 | Stavros Sachtouris | r.release() |
827 | e6b39366 | Stavros Sachtouris | |
828 | d1856abf | Stavros Sachtouris | def set_container_quota(self, quota): |
829 | 4375e020 | Stavros Sachtouris | """
|
830 | 4375e020 | Stavros Sachtouris | :param quota: (int)
|
831 | 4375e020 | Stavros Sachtouris | """
|
832 | 6ce9fc72 | Stavros Sachtouris | r = self.container_post(update=True, quota=quota) |
833 | 6ce9fc72 | Stavros Sachtouris | r.release() |
834 | d1856abf | Stavros Sachtouris | |
835 | d1856abf | Stavros Sachtouris | def set_container_versioning(self, versioning): |
836 | 4375e020 | Stavros Sachtouris | """
|
837 | 4375e020 | Stavros Sachtouris | :param versioning: (str)
|
838 | 4375e020 | Stavros Sachtouris | """
|
839 | 6ce9fc72 | Stavros Sachtouris | r = self.container_post(update=True, versioning=versioning) |
840 | 6ce9fc72 | Stavros Sachtouris | r.release() |
841 | d1856abf | Stavros Sachtouris | |
842 | a298f2ab | Stavros Sachtouris | def del_object(self, obj, until=None, delimiter=None): |
843 | 4375e020 | Stavros Sachtouris | """
|
844 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
845 | 4375e020 | Stavros Sachtouris |
|
846 | 4375e020 | Stavros Sachtouris | :param until: (str) formated date
|
847 | 4375e020 | Stavros Sachtouris |
|
848 | 4375e020 | Stavros Sachtouris | :param delimiter: (str)
|
849 | 4375e020 | Stavros Sachtouris | """
|
850 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
851 | 6ce9fc72 | Stavros Sachtouris | r = self.object_delete(obj, until=until, delimiter=delimiter)
|
852 | 6ce9fc72 | Stavros Sachtouris | r.release() |
853 | a298f2ab | Stavros Sachtouris | |
854 | 4375e020 | Stavros Sachtouris | def set_object_meta(self, obj, metapairs): |
855 | 4375e020 | Stavros Sachtouris | """
|
856 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
857 | 4375e020 | Stavros Sachtouris |
|
858 | 4375e020 | Stavros Sachtouris | :param metapairs: (dict) {key1:val1, key2:val2, ...}
|
859 | 4375e020 | Stavros Sachtouris | """
|
860 | af3b2b36 | Stavros Sachtouris | assert(type(metapairs) is dict) |
861 | 4375e020 | Stavros Sachtouris | r = self.object_post(obj, update=True, metadata=metapairs) |
862 | 6ce9fc72 | Stavros Sachtouris | r.release() |
863 | 87688514 | Stavros Sachtouris | |
864 | bc223d91 | Stavros Sachtouris | def del_object_meta(self, obj, metakey): |
865 | 4375e020 | Stavros Sachtouris | """
|
866 | 4375e020 | Stavros Sachtouris | :param obj: (str) remote object path
|
867 | bc223d91 | Stavros Sachtouris |
|
868 | bc223d91 | Stavros Sachtouris | :param metakey: (str) metadatum key
|
869 | 4375e020 | Stavros Sachtouris | """
|
870 | 5fb18128 | Stavros Sachtouris | r = self.object_post(obj, update=True, metadata={metakey: ''}) |
871 | 6ce9fc72 | Stavros Sachtouris | r.release() |
872 | 6de1f262 | Stavros Sachtouris | |
873 | bc223d91 | Stavros Sachtouris | def publish_object(self, obj): |
874 | bc223d91 | Stavros Sachtouris | """
|
875 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
876 | 5260a313 | Stavros Sachtouris |
|
877 | 5260a313 | Stavros Sachtouris | :returns: (str) access url
|
878 | bc223d91 | Stavros Sachtouris | """
|
879 | bc223d91 | Stavros Sachtouris | r = self.object_post(obj, update=True, public=True) |
880 | 6ce9fc72 | Stavros Sachtouris | r.release() |
881 | 5260a313 | Stavros Sachtouris | info = self.get_object_info(obj)
|
882 | 93542587 | Stavros Sachtouris | pref, sep, rest = self.base_url.partition('//') |
883 | 93542587 | Stavros Sachtouris | base = rest.split('/')[0] |
884 | 24ff0a35 | Stavros Sachtouris | newurl = path4url( |
885 | 24ff0a35 | Stavros Sachtouris | '%s%s%s' % (pref, sep, base),
|
886 | 93542587 | Stavros Sachtouris | info['x-object-public'])
|
887 | 5260a313 | Stavros Sachtouris | return newurl[1:] |
888 | 87688514 | Stavros Sachtouris | |
889 | bc223d91 | Stavros Sachtouris | def unpublish_object(self, obj): |
890 | bc223d91 | Stavros Sachtouris | """
|
891 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
892 | bc223d91 | Stavros Sachtouris | """
|
893 | bc223d91 | Stavros Sachtouris | r = self.object_post(obj, update=True, public=False) |
894 | 6ce9fc72 | Stavros Sachtouris | r.release() |
895 | 28470086 | Stavros Sachtouris | |
896 | 8af4cc0b | Stavros Sachtouris | def get_object_info(self, obj, version=None): |
897 | bc223d91 | Stavros Sachtouris | """
|
898 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
899 | bc223d91 | Stavros Sachtouris |
|
900 | bc223d91 | Stavros Sachtouris | :param version: (str)
|
901 | bc223d91 | Stavros Sachtouris |
|
902 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
903 | bc223d91 | Stavros Sachtouris | """
|
904 | ca092af4 | Stavros Sachtouris | try:
|
905 | ca092af4 | Stavros Sachtouris | r = self.object_head(obj, version=version)
|
906 | ca092af4 | Stavros Sachtouris | return r.headers
|
907 | ca092af4 | Stavros Sachtouris | except ClientError as ce: |
908 | ca092af4 | Stavros Sachtouris | if ce.status == 404: |
909 | 723e9d47 | Stavros Sachtouris | raise ClientError('Object %s not found' % obj, status=404) |
910 | ca092af4 | Stavros Sachtouris | raise
|
911 | 8af4cc0b | Stavros Sachtouris | |
912 | 8af4cc0b | Stavros Sachtouris | def get_object_meta(self, obj, version=None): |
913 | bc223d91 | Stavros Sachtouris | """
|
914 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
915 | bc223d91 | Stavros Sachtouris |
|
916 | bc223d91 | Stavros Sachtouris | :param version: (str)
|
917 | bc223d91 | Stavros Sachtouris |
|
918 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
919 | bc223d91 | Stavros Sachtouris | """
|
920 | 24ff0a35 | Stavros Sachtouris | return filter_in(
|
921 | 24ff0a35 | Stavros Sachtouris | self.get_object_info(obj, version=version),
|
922 | 3dabe5d2 | Stavros Sachtouris | 'X-Object-Meta')
|
923 | 8af4cc0b | Stavros Sachtouris | |
924 | bc223d91 | Stavros Sachtouris | def get_object_sharing(self, obj): |
925 | bc223d91 | Stavros Sachtouris | """
|
926 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
927 | bc223d91 | Stavros Sachtouris |
|
928 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
929 | bc223d91 | Stavros Sachtouris | """
|
930 | 24ff0a35 | Stavros Sachtouris | r = filter_in( |
931 | 24ff0a35 | Stavros Sachtouris | self.get_object_info(obj),
|
932 | 3dabe5d2 | Stavros Sachtouris | 'X-Object-Sharing',
|
933 | 3dabe5d2 | Stavros Sachtouris | exactMatch=True)
|
934 | f70616fc | Stavros Sachtouris | reply = {} |
935 | f70616fc | Stavros Sachtouris | if len(r) > 0: |
936 | f70616fc | Stavros Sachtouris | perms = r['x-object-sharing'].split(';') |
937 | f70616fc | Stavros Sachtouris | for perm in perms: |
938 | f70616fc | Stavros Sachtouris | try:
|
939 | f70616fc | Stavros Sachtouris | perm.index('=')
|
940 | f70616fc | Stavros Sachtouris | except ValueError: |
941 | f70616fc | Stavros Sachtouris | raise ClientError('Incorrect reply format') |
942 | f70616fc | Stavros Sachtouris | (key, val) = perm.strip().split('=')
|
943 | f70616fc | Stavros Sachtouris | reply[key] = val |
944 | f70616fc | Stavros Sachtouris | return reply
|
945 | f49084df | Stavros Sachtouris | |
946 | 24ff0a35 | Stavros Sachtouris | def set_object_sharing( |
947 | 2005b18e | Stavros Sachtouris | self, obj,
|
948 | 2005b18e | Stavros Sachtouris | read_permition=False, write_permition=False): |
949 | 28470086 | Stavros Sachtouris | """Give read/write permisions to an object.
|
950 | bc223d91 | Stavros Sachtouris |
|
951 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
952 | bc223d91 | Stavros Sachtouris |
|
953 | bc223d91 | Stavros Sachtouris | :param read_permition: (list - bool) users and user groups that get
|
954 | bc223d91 | Stavros Sachtouris | read permition for this object - False means all previous read
|
955 | bc223d91 | Stavros Sachtouris | permissions will be removed
|
956 | bc223d91 | Stavros Sachtouris |
|
957 | bc223d91 | Stavros Sachtouris | :param write_perimition: (list - bool) of users and user groups to get
|
958 | bc223d91 | Stavros Sachtouris | write permition for this object - False means all previous write
|
959 | 4067cdaf | Stavros Sachtouris | permissions will be removed
|
960 | 28470086 | Stavros Sachtouris | """
|
961 | 4067cdaf | Stavros Sachtouris | |
962 | 24ff0a35 | Stavros Sachtouris | perms = dict(
|
963 | 24ff0a35 | Stavros Sachtouris | read='' if not read_permition else read_permition, |
964 | 3dabe5d2 | Stavros Sachtouris | write='' if not write_permition else write_permition) |
965 | bc223d91 | Stavros Sachtouris | r = self.object_post(obj, update=True, permissions=perms) |
966 | 6ce9fc72 | Stavros Sachtouris | r.release() |
967 | 28470086 | Stavros Sachtouris | |
968 | bc223d91 | Stavros Sachtouris | def del_object_sharing(self, obj): |
969 | bc223d91 | Stavros Sachtouris | """
|
970 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
971 | bc223d91 | Stavros Sachtouris | """
|
972 | bc223d91 | Stavros Sachtouris | self.set_object_sharing(obj)
|
973 | bc223d91 | Stavros Sachtouris | |
974 | bc223d91 | Stavros Sachtouris | def append_object(self, obj, source_file, upload_cb=None): |
975 | bc223d91 | Stavros Sachtouris | """
|
976 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
977 | bc223d91 | Stavros Sachtouris |
|
978 | bc223d91 | Stavros Sachtouris | :param source_file: open file descriptor
|
979 | f49084df | Stavros Sachtouris |
|
980 | bc223d91 | Stavros Sachtouris | :param upload_db: progress.bar for uploading
|
981 | bcabbc35 | Stavros Sachtouris | """
|
982 | 4067cdaf | Stavros Sachtouris | |
983 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
984 | 2f749e6e | Stavros Sachtouris | meta = self.get_container_info()
|
985 | ab474306 | Stavros Sachtouris | blocksize = int(meta['x-container-block-size']) |
986 | 64ab4c13 | Stavros Sachtouris | filesize = fstat(source_file.fileno()).st_size |
987 | 3dabe5d2 | Stavros Sachtouris | nblocks = 1 + (filesize - 1) // blocksize |
988 | ab474306 | Stavros Sachtouris | offset = 0
|
989 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
990 | bcabbc35 | Stavros Sachtouris | upload_gen = upload_cb(nblocks) |
991 | ca092af4 | Stavros Sachtouris | upload_gen.next() |
992 | ab474306 | Stavros Sachtouris | for i in range(nblocks): |
993 | ab474306 | Stavros Sachtouris | block = source_file.read(min(blocksize, filesize - offset))
|
994 | ab474306 | Stavros Sachtouris | offset += len(block)
|
995 | 24ff0a35 | Stavros Sachtouris | r = self.object_post(
|
996 | 24ff0a35 | Stavros Sachtouris | obj, |
997 | 3dabe5d2 | Stavros Sachtouris | update=True,
|
998 | 3dabe5d2 | Stavros Sachtouris | content_range='bytes */*',
|
999 | 3dabe5d2 | Stavros Sachtouris | content_type='application/octet-stream',
|
1000 | 3dabe5d2 | Stavros Sachtouris | content_length=len(block),
|
1001 | 3dabe5d2 | Stavros Sachtouris | data=block) |
1002 | 6ce9fc72 | Stavros Sachtouris | r.release() |
1003 | 3dabe5d2 | Stavros Sachtouris | |
1004 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
1005 | bcabbc35 | Stavros Sachtouris | upload_gen.next() |
1006 | 561116a6 | Stavros Sachtouris | |
1007 | bc223d91 | Stavros Sachtouris | def truncate_object(self, obj, upto_bytes): |
1008 | bc223d91 | Stavros Sachtouris | """
|
1009 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
1010 | bc223d91 | Stavros Sachtouris |
|
1011 | bc223d91 | Stavros Sachtouris | :param upto_bytes: max number of bytes to leave on file
|
1012 | bc223d91 | Stavros Sachtouris | """
|
1013 | 24ff0a35 | Stavros Sachtouris | r = self.object_post(
|
1014 | 24ff0a35 | Stavros Sachtouris | obj, |
1015 | 3dabe5d2 | Stavros Sachtouris | update=True,
|
1016 | 3dabe5d2 | Stavros Sachtouris | content_range='bytes 0-%s/*' % upto_bytes,
|
1017 | 3dabe5d2 | Stavros Sachtouris | content_type='application/octet-stream',
|
1018 | 3dabe5d2 | Stavros Sachtouris | object_bytes=upto_bytes, |
1019 | bc223d91 | Stavros Sachtouris | source_object=path4url(self.container, obj))
|
1020 | 6ce9fc72 | Stavros Sachtouris | r.release() |
1021 | ee62607e | Stavros Sachtouris | |
1022 | 2005b18e | Stavros Sachtouris | def overwrite_object(self, obj, start, end, source_file, upload_cb=None): |
1023 | bc223d91 | Stavros Sachtouris | """Overwrite a part of an object from local source file
|
1024 | bc223d91 | Stavros Sachtouris |
|
1025 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
1026 | bc223d91 | Stavros Sachtouris |
|
1027 | bc223d91 | Stavros Sachtouris | :param start: (int) position in bytes to start overwriting from
|
1028 | bc223d91 | Stavros Sachtouris |
|
1029 | bc223d91 | Stavros Sachtouris | :param end: (int) position in bytes to stop overwriting at
|
1030 | bc223d91 | Stavros Sachtouris |
|
1031 | bc223d91 | Stavros Sachtouris | :param source_file: open file descriptor
|
1032 | bc223d91 | Stavros Sachtouris |
|
1033 | bc223d91 | Stavros Sachtouris | :param upload_db: progress.bar for uploading
|
1034 | ee62607e | Stavros Sachtouris | """
|
1035 | 4067cdaf | Stavros Sachtouris | |
1036 | ca092af4 | Stavros Sachtouris | r = self.get_object_info(obj)
|
1037 | ca092af4 | Stavros Sachtouris | rf_size = int(r['content-length']) |
1038 | ca092af4 | Stavros Sachtouris | if rf_size < int(start): |
1039 | ca092af4 | Stavros Sachtouris | raise ClientError(
|
1040 | ca092af4 | Stavros Sachtouris | 'Range start exceeds file size',
|
1041 | ca092af4 | Stavros Sachtouris | status=416)
|
1042 | ca092af4 | Stavros Sachtouris | elif rf_size < int(end): |
1043 | ca092af4 | Stavros Sachtouris | raise ClientError(
|
1044 | ca092af4 | Stavros Sachtouris | 'Range end exceeds file size',
|
1045 | ca092af4 | Stavros Sachtouris | status=416)
|
1046 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
1047 | 2f749e6e | Stavros Sachtouris | meta = self.get_container_info()
|
1048 | ee62607e | Stavros Sachtouris | blocksize = int(meta['x-container-block-size']) |
1049 | 64ab4c13 | Stavros Sachtouris | filesize = fstat(source_file.fileno()).st_size |
1050 | ee62607e | Stavros Sachtouris | datasize = int(end) - int(start) + 1 |
1051 | 3dabe5d2 | Stavros Sachtouris | nblocks = 1 + (datasize - 1) // blocksize |
1052 | ee62607e | Stavros Sachtouris | offset = 0
|
1053 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
1054 | bcabbc35 | Stavros Sachtouris | upload_gen = upload_cb(nblocks) |
1055 | ca092af4 | Stavros Sachtouris | upload_gen.next() |
1056 | ee62607e | Stavros Sachtouris | for i in range(nblocks): |
1057 | 24ff0a35 | Stavros Sachtouris | read_size = min(blocksize, filesize - offset, datasize - offset)
|
1058 | 24ff0a35 | Stavros Sachtouris | block = source_file.read(read_size) |
1059 | 24ff0a35 | Stavros Sachtouris | r = self.object_post(
|
1060 | 24ff0a35 | Stavros Sachtouris | obj, |
1061 | 3dabe5d2 | Stavros Sachtouris | update=True,
|
1062 | 3dabe5d2 | Stavros Sachtouris | content_type='application/octet-stream',
|
1063 | 3dabe5d2 | Stavros Sachtouris | content_length=len(block),
|
1064 | ca092af4 | Stavros Sachtouris | content_range='bytes %s-%s/*' % (
|
1065 | ca092af4 | Stavros Sachtouris | start + offset, |
1066 | ca092af4 | Stavros Sachtouris | start + offset + len(block) - 1), |
1067 | 3dabe5d2 | Stavros Sachtouris | data=block) |
1068 | ca092af4 | Stavros Sachtouris | offset += len(block)
|
1069 | 6ce9fc72 | Stavros Sachtouris | r.release() |
1070 | 3dabe5d2 | Stavros Sachtouris | |
1071 | ca092af4 | Stavros Sachtouris | if upload_cb:
|
1072 | bcabbc35 | Stavros Sachtouris | upload_gen.next() |
1073 | 7d420701 | Stavros Sachtouris | |
1074 | 24ff0a35 | Stavros Sachtouris | def copy_object( |
1075 | 2005b18e | Stavros Sachtouris | self, src_container, src_object, dst_container,
|
1076 | 33487500 | Stavros Sachtouris | dst_object=None,
|
1077 | 2005b18e | Stavros Sachtouris | source_version=None,
|
1078 | 3a066af4 | Stavros Sachtouris | source_account=None,
|
1079 | 2005b18e | Stavros Sachtouris | public=False,
|
1080 | 2005b18e | Stavros Sachtouris | content_type=None,
|
1081 | 2005b18e | Stavros Sachtouris | delimiter=None):
|
1082 | bc223d91 | Stavros Sachtouris | """
|
1083 | bc223d91 | Stavros Sachtouris | :param src_container: (str) source container
|
1084 | bc223d91 | Stavros Sachtouris |
|
1085 | bc223d91 | Stavros Sachtouris | :param src_object: (str) source object path
|
1086 | bc223d91 | Stavros Sachtouris |
|
1087 | bc223d91 | Stavros Sachtouris | :param dst_container: (str) destination container
|
1088 | bc223d91 | Stavros Sachtouris |
|
1089 | bc223d91 | Stavros Sachtouris | :param dst_object: (str) destination object path
|
1090 | bc223d91 | Stavros Sachtouris |
|
1091 | bc223d91 | Stavros Sachtouris | :param source_version: (str) source object version
|
1092 | bc223d91 | Stavros Sachtouris |
|
1093 | 3a066af4 | Stavros Sachtouris | :param source_account: (str) account to copy from
|
1094 | 3a066af4 | Stavros Sachtouris |
|
1095 | bc223d91 | Stavros Sachtouris | :param public: (bool)
|
1096 | bc223d91 | Stavros Sachtouris |
|
1097 | bc223d91 | Stavros Sachtouris | :param content_type: (str)
|
1098 | bc223d91 | Stavros Sachtouris |
|
1099 | bc223d91 | Stavros Sachtouris | :param delimiter: (str)
|
1100 | bc223d91 | Stavros Sachtouris | """
|
1101 | 277ca4ed | Dionysis Zindros | self._assert_account()
|
1102 | 7d420701 | Stavros Sachtouris | self.container = dst_container
|
1103 | 7d420701 | Stavros Sachtouris | src_path = path4url(src_container, src_object) |
1104 | 24ff0a35 | Stavros Sachtouris | r = self.object_put(
|
1105 | 33487500 | Stavros Sachtouris | dst_object or src_object,
|
1106 | 3dabe5d2 | Stavros Sachtouris | success=201,
|
1107 | 3dabe5d2 | Stavros Sachtouris | copy_from=src_path, |
1108 | 3dabe5d2 | Stavros Sachtouris | content_length=0,
|
1109 | 3dabe5d2 | Stavros Sachtouris | source_version=source_version, |
1110 | 3a066af4 | Stavros Sachtouris | source_account=source_account, |
1111 | 3dabe5d2 | Stavros Sachtouris | public=public, |
1112 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
1113 | 7d420701 | Stavros Sachtouris | delimiter=delimiter) |
1114 | 6ce9fc72 | Stavros Sachtouris | r.release() |
1115 | a5e0629d | Stavros Sachtouris | |
1116 | 24ff0a35 | Stavros Sachtouris | def move_object( |
1117 | 24ff0a35 | Stavros Sachtouris | self, src_container, src_object, dst_container,
|
1118 | 24ff0a35 | Stavros Sachtouris | dst_object=False,
|
1119 | 4f266635 | Stavros Sachtouris | source_account=None,
|
1120 | 24ff0a35 | Stavros Sachtouris | source_version=None,
|
1121 | 24ff0a35 | Stavros Sachtouris | public=False,
|
1122 | 24ff0a35 | Stavros Sachtouris | content_type=None,
|
1123 | 24ff0a35 | Stavros Sachtouris | delimiter=None):
|
1124 | bc223d91 | Stavros Sachtouris | """
|
1125 | bc223d91 | Stavros Sachtouris | :param src_container: (str) source container
|
1126 | bc223d91 | Stavros Sachtouris |
|
1127 | bc223d91 | Stavros Sachtouris | :param src_object: (str) source object path
|
1128 | bc223d91 | Stavros Sachtouris |
|
1129 | bc223d91 | Stavros Sachtouris | :param dst_container: (str) destination container
|
1130 | bc223d91 | Stavros Sachtouris |
|
1131 | bc223d91 | Stavros Sachtouris | :param dst_object: (str) destination object path
|
1132 | bc223d91 | Stavros Sachtouris |
|
1133 | 4f266635 | Stavros Sachtouris | :param source_account: (str) account to move from
|
1134 | 4f266635 | Stavros Sachtouris |
|
1135 | bc223d91 | Stavros Sachtouris | :param source_version: (str) source object version
|
1136 | bc223d91 | Stavros Sachtouris |
|
1137 | bc223d91 | Stavros Sachtouris | :param public: (bool)
|
1138 | bc223d91 | Stavros Sachtouris |
|
1139 | bc223d91 | Stavros Sachtouris | :param content_type: (str)
|
1140 | bc223d91 | Stavros Sachtouris |
|
1141 | bc223d91 | Stavros Sachtouris | :param delimiter: (str)
|
1142 | bc223d91 | Stavros Sachtouris | """
|
1143 | 277ca4ed | Dionysis Zindros | self._assert_account()
|
1144 | a5e0629d | Stavros Sachtouris | self.container = dst_container
|
1145 | a5e0629d | Stavros Sachtouris | dst_object = dst_object or src_object
|
1146 | a5e0629d | Stavros Sachtouris | src_path = path4url(src_container, src_object) |
1147 | 24ff0a35 | Stavros Sachtouris | r = self.object_put(
|
1148 | 24ff0a35 | Stavros Sachtouris | dst_object, |
1149 | 3dabe5d2 | Stavros Sachtouris | success=201,
|
1150 | 3dabe5d2 | Stavros Sachtouris | move_from=src_path, |
1151 | 3dabe5d2 | Stavros Sachtouris | content_length=0,
|
1152 | 4f266635 | Stavros Sachtouris | source_account=source_account, |
1153 | 3dabe5d2 | Stavros Sachtouris | source_version=source_version, |
1154 | 3dabe5d2 | Stavros Sachtouris | public=public, |
1155 | 3dabe5d2 | Stavros Sachtouris | content_type=content_type, |
1156 | a23f6ffe | Stavros Sachtouris | delimiter=delimiter) |
1157 | 6ce9fc72 | Stavros Sachtouris | r.release() |
1158 | a23f6ffe | Stavros Sachtouris | |
1159 | a23f6ffe | Stavros Sachtouris | def get_sharing_accounts(self, limit=None, marker=None, *args, **kwargs): |
1160 | bc223d91 | Stavros Sachtouris | """Get accounts that share with self.account
|
1161 | bc223d91 | Stavros Sachtouris |
|
1162 | bc223d91 | Stavros Sachtouris | :param limit: (str)
|
1163 | bc223d91 | Stavros Sachtouris |
|
1164 | bc223d91 | Stavros Sachtouris | :param marker: (str)
|
1165 | bc223d91 | Stavros Sachtouris |
|
1166 | bc223d91 | Stavros Sachtouris | :returns: (dict)
|
1167 | bc223d91 | Stavros Sachtouris | """
|
1168 | 277ca4ed | Dionysis Zindros | self._assert_account()
|
1169 | a23f6ffe | Stavros Sachtouris | |
1170 | 3dabe5d2 | Stavros Sachtouris | self.set_param('format', 'json') |
1171 | 3dabe5d2 | Stavros Sachtouris | self.set_param('limit', limit, iff=limit is not None) |
1172 | 3dabe5d2 | Stavros Sachtouris | self.set_param('marker', marker, iff=marker is not None) |
1173 | a23f6ffe | Stavros Sachtouris | |
1174 | a23f6ffe | Stavros Sachtouris | path = ''
|
1175 | a23f6ffe | Stavros Sachtouris | success = kwargs.pop('success', (200, 204)) |
1176 | 3dabe5d2 | Stavros Sachtouris | r = self.get(path, *args, success=success, **kwargs)
|
1177 | 38dc5d2f | Stavros Sachtouris | return r.json
|
1178 | 38dc5d2f | Stavros Sachtouris | |
1179 | bc223d91 | Stavros Sachtouris | def get_object_versionlist(self, obj): |
1180 | bc223d91 | Stavros Sachtouris | """
|
1181 | bc223d91 | Stavros Sachtouris | :param obj: (str) remote object path
|
1182 | bc223d91 | Stavros Sachtouris |
|
1183 | bc223d91 | Stavros Sachtouris | :returns: (list)
|
1184 | bc223d91 | Stavros Sachtouris | """
|
1185 | 277ca4ed | Dionysis Zindros | self._assert_container()
|
1186 | bc223d91 | Stavros Sachtouris | r = self.object_get(obj, format='json', version='list') |
1187 | 38dc5d2f | Stavros Sachtouris | return r.json['versions'] |