1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
3 # Redistribution and use in source and binary forms, with or
4 # without modification, are permitted provided that the following
7 # 1. Redistributions of source code must retain the above
8 # copyright notice, this list of conditions and the following
11 # 2. Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following
13 # disclaimer in the documentation and/or other materials
14 # provided with the distribution.
16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 # POSSIBILITY OF SUCH DAMAGE.
29 # The views and conclusions contained in the software and
30 # documentation are those of the authors and should not be
31 # interpreted as representing official policies, either expressed
32 # or implied, of GRNET S.A.
39 from .storage import StorageClient, ClientError
40 from .utils import path4url, params4url, prefix_keys, filter_in, filter_out, list2str
43 def pithos_hash(block, blockhash):
44 h = hashlib.new(blockhash)
45 h.update(block.rstrip('\x00'))
48 class PithosClient(StorageClient):
49 """GRNet Pithos API client"""
51 def account_head(self, until = None,
52 if_modified_since=None, if_unmodified_since=None, *args, **kwargs):
53 """ Full Pithos+ HEAD at account level
54 --- request parameters ---
55 @param until (string): optional timestamp
56 --- --- optional request headers ---
57 @param if_modified_since (string): Retrieve if account has changed since provided timestamp
58 @param if_unmodified_since (string): Retrieve if account has not changed since provided timestamp
61 path = path4url(self.account)
63 path += '' if until is None else params4url({'until':until})
64 self.set_header('If-Modified-Since', if_modified_since)
65 self.set_header('If-Unmodified-Since', if_unmodified_since)
67 success = kwargs.pop('success', 204)
68 return self.head(path, *args, success=success, **kwargs)
70 def account_get(self, limit=None, marker=None, format='json', show_only_shared=False, until=None,
71 if_modified_since=None, if_unmodified_since=None, *args, **kwargs):
72 """ Full Pithos+ GET at account level
73 --- request parameters ---
74 @param limit (integer): The amount of results requested (server will use default value if None)
75 @param marker (string): Return containers with name lexicographically after marker
76 @param format (string): reply format can be json or xml (default: json)
77 @param shared (bool): If true, only shared containers will be included in results
78 @param until (string): optional timestamp
79 --- --- optional request headers ---
80 @param if_modified_since (string): Retrieve if account has changed since provided timestamp
81 @param if_unmodified_since (string): Retrieve if account has not changed since provided timestamp
85 param_dict = {} if format is None else dict(format=format)
87 param_dict['limit'] = limit
88 if marker is not None:
89 param_dict['marker'] = marker
91 param_dict['shared'] = None
93 param_dict['until'] = until
95 path = path4url(self.account)+params4url(param_dict)
96 self.set_header('If-Modified-Since', if_modified_since)
97 self.set_header('If-Unmodified-Since', if_unmodified_since)
99 success = kwargs.pop('success', (200, 204))
100 return self.get(path, *args, success = success, **kwargs)
102 def account_post(self, update=True,
103 groups={}, metadata={}, quota=None, versioning=None, *args, **kwargs):
104 """ Full Pithos+ POST at account level
105 --- request parameters ---
106 @param update (bool): if True, Do not replace metadata/groups
107 --- request headers ---
108 @groups (dict): Optional user defined groups in the form
109 { 'group1':['user1', 'user2', ...],
110 'group2':['userA', 'userB', ...], ...
112 @metadata (dict): Optional user defined metadata in the form
114 'name2': 'value2', ...
116 @param quota(integer): If supported, sets the Account quota
117 @param versioning(string): If supported, sets the Account versioning
118 to 'auto' or some other supported versioning string
120 self.assert_account()
121 path = path4url(self.account) + params4url({'update':None}) if update else ''
122 for group, usernames in groups.items():
125 for user in usernames:
126 userstr = userstr + dlm + user
128 self.set_header('X-Account-Group-'+group, userstr)
129 for metaname, metaval in metadata.items():
130 self.set_header('X-Account-Meta-'+metaname, metaval)
131 self.set_header('X-Account-Policy-Quota', quota)
132 self.set_header('X-Account-Policy-Versioning', versioning)
134 success = kwargs.pop('success', 202)
135 return self.post(path, *args, success=success, **kwargs)
137 def container_head(self, until=None,
138 if_modified_since=None, if_unmodified_since=None, *args, **kwargs):
139 """ Full Pithos+ HEAD at container level
140 --- request params ---
141 @param until (string): optional timestamp
142 --- optional request headers ---
143 @param if_modified_since (string): Retrieve if account has changed since provided timestamp
144 @param if_unmodified_since (string): Retrieve if account has not changed since provided timestamp
146 self.assert_container()
147 path = path4url(self.account, self.container)
148 path += '' if until is None else params4url(dict(until=until))
149 self.set_header('If-Modified-Since', if_modified_since)
150 self.set_header('If-Unmodified-Since', if_unmodified_since)
151 success = kwargs.pop('success', 204)
152 return self.head(path, *args, success=success, **kwargs)
154 def container_get(self, limit = None, marker = None, prefix=None, delimiter=None, path = None, format='json', meta=[], show_only_shared=False, until=None,
155 if_modified_since=None, if_unmodified_since=None, *args, **kwargs):
156 """ Full Pithos+ GET at container level
157 --- request parameters ---
158 @param limit (integer): The amount of results requested (server qill use default value if None)
159 @param marker (string): Return containers with name lexicographically after marker
160 @param prefix (string): Return objects starting with prefix
161 @param delimiter (string): Return objects up to the delimiter
162 @param path (string): assume prefix = path and delimiter = / (overwrites prefix
164 @param format (string): reply format can be json or xml (default: json)
165 @param meta (list): Return objects that satisfy the key queries in the specified
166 comma separated list (use <key>, !<key> for existence queries, <key><op><value>
167 for value queries, where <op> can be one of =, !=, <=, >=, <, >)
168 @param shared (bool): If true, only shared containers will be included in results
169 @param until (string): optional timestamp
170 --- --- optional request headers ---
171 @param if_modified_since (string): Retrieve if account has changed since provided timestamp
172 @param if_unmodified_since (string): Retrieve if account has not changed since provided timestamp
174 self.assert_container()
176 param_dict = {} if format is None else dict(format=format)
177 if limit is not None:
178 param_dict['limit'] = limit
179 if marker is not None:
180 param_dict['marker'] = marker
182 param_dict['path'] = path
184 if prefix is not None:
185 param_dict['prefix'] = prefix
186 if delimiter is not None:
187 param_dict['delimiter'] = delimiter
189 param_dict['shared'] = None
190 if meta is not None and len(meta) > 0:
191 param_dict['meta'] = list2str(meta)
192 if until is not None:
193 param_dict['until'] = until
194 path = path4url(self.account, self.container)+params4url(param_dict)
195 self.set_header('If-Modified-Since', if_modified_since)
196 self.set_header('If-Unmodified-Since', if_unmodified_since)
197 success = kwargs.pop('success', 200)
198 return self.get(path, *args, success=success, **kwargs)
200 def container_put(self, quota=None, versioning=None, metadata={}, *args, **kwargs):
201 """ Full Pithos+ PUT at container level
202 --- request headers ---
203 @param quota (integer): Size limit in KB
204 @param versioning (string): 'auto' or other string supported by server
205 @metadata (dict): Optional user defined metadata in the form
207 'name2': 'value2', ...
210 self.assert_container()
211 path = path4url(self.account, self.container)
212 for metaname, metaval in metadata.items():
213 self.set_header('X-Container-Meta-'+metaname, metaval)
214 self.set_header('X-Container-Policy-Quota', quota)
215 self.set_header('X-Container-Policy-Versioning', versioning)
216 success = kwargs.pop('success',(201, 202))
217 return self.put(path, *args, success=success, **kwargs)
219 def container_post(self, update=True, format='json',
220 quota=None, versioning=None, metadata={}, content_type=None, content_length=None, transfer_encoding=None,
222 """ Full Pithos+ POST at container level
223 --- request params ---
224 @param update (bool): if True, Do not replace metadata/groups
225 @param format(string): json (default) or xml
226 --- request headers ---
227 @param quota (integer): Size limit in KB
228 @param versioning (string): 'auto' or other string supported by server
229 @metadata (dict): Optional user defined metadata in the form
231 'name2': 'value2', ...
233 @param content_type (string): set a custom content type
234 @param content_length (string): set a custrom content length
235 @param transfer_encoding (string): set a custrom transfer encoding
237 self.assert_container()
238 param_dict = {} if format is None else dict(format=format)
240 param_dict['update'] = None
241 path = path4url(self.account, self.container)+params4url(param_dict)
243 for metaname, metaval in metadata.items():
244 self.set_header('X-Container-Meta-'+metaname, metaval)
245 self.set_header('X-Container-Policy-Quota', quota)
246 self.set_header('X-Container-Policy-Versioning', versioning)
247 self.set_header('Content-Type', content_type)
248 self.set_header('Content-Length', content_length)
249 self.set_header('Transfer-Encoding', transfer_encoding)
250 success = kwargs.pop('success', 202)
251 return self.post(path, *args, success=success, **kwargs)
253 def container_delete(self, until=None, *args, **kwargs):
254 """ Full Pithos+ DELETE at container level
255 --- request parameters ---
256 @param until (timestamp string): if defined, container is purged up to that time
258 self.assert_container()
259 path=path4url(self.account, self.container)
260 path += '' if until is None else params4url(dict(until=until))
261 success = kwargs.pop('success', 204)
262 return self.delete(path, success=success)
264 def object_head(self, object, version=None,
265 if_etag_match=None, if_etag_not_match = None, if_modified_since = None, if_unmodified_since = None, *args, **kwargs):
266 """ Full Pithos+ HEAD at object level
267 --- request parameters ---
268 @param version (string): optional version identified
269 --- request headers ---
270 @param if_etag_match (string): if provided, return only results
271 with etag matching with this
272 @param if_etag_not_match (string): if provided, return only results
273 with etag not matching with this
274 @param if_modified_since (string): Retrieve if account has changed since provided timestamp
275 @param if_unmodified_since (string): Retrieve if account has not changed since provided timestamp
277 self.assert_container()
278 path=path4url(self.account, self.container, object)
279 path += '' if version is None else params4url(dict(version=version))
280 self.set_header('If-Match', if_etag_match)
281 self.set_header('If-None-Match', if_etag_not_match)
282 self.set_header('If-Modified-Since', if_modified_since)
283 self.set_header('If-Unmodified-Since', if_unmodified_since)
284 success = kwargs.pop('success', 200)
285 return self.head(path, *args, success=success, **kwargs)
287 def object_get(self, object, format='json', hashmap=False, version=None,
288 data_range=None, if_range=False, if_etag_match=None, if_etag_not_match = None, if_modified_since = None, if_unmodified_since = None, *args, **kwargs):
289 """ Full Pithos+ GET at object level
290 --- request parameters ---
291 @param format (string): json (default) or xml
292 @param hashmap (bool): Optional request for hashmap
293 @param version (string): optional version identified
294 --- request headers ---
295 @param data_range (string): Optional range of data to retrieve
296 @param if_range (bool):
297 @param if_etag_match (string): if provided, return only results
298 with etag matching with this
299 @param if_etag_not_match (string): if provided, return only results
300 with etag not matching with this
301 @param if_modified_since (string): Retrieve if account has changed since provided timestamp
302 @param if_unmodified_since (string): Retrieve if account has not changed since provided timestamp
304 self.assert_container()
305 param_dict = {} if format is None else dict(format=format)
307 param_dict['hashmap']=None
308 if version is not None:
309 param_dict['version']=version
310 path=path4url(self.account, self.container, object)+params4url(param_dict)
311 self.set_header('Range', data_range)
312 self.set_header('If-Range', '', if_range is True and data_range is not None)
313 self.set_header('If-Match', if_etag_match, )
314 self.set_header('If-None-Match', if_etag_not_match)
315 self.set_header('If-Modified-Since', if_modified_since)
316 self.set_header('If-Unmodified-Since', if_unmodified_since)
317 success = kwargs.pop('success', 200)
318 return self.get(path, *args, success=success, **kwargs)
320 def object_put(self, object, format='json', hashmap=False,
321 if_etag_match=None, if_etag_not_match = None, etag=None, content_length = None, content_type=None, transfer_encoding=None,
322 copy_from=None, move_from=None, source_account=None, source_version=None, content_encoding = None, content_disposition=None,
323 manifest = None, permitions = {}, public = None, metadata={}, *args, **kwargs):
324 """ Full Pithos+ PUT at object level
325 --- request parameters ---
326 @param format (string): json (default) or xml
327 @param hashmap (bool): Optional hashmap provided instead of data
328 --- request headers ---
329 @param if_etag_match (string): if provided, return only results
330 with etag matching with this
331 @param if_etag_not_match (string): if provided, return only results
332 with etag not matching with this
333 @param etag (string): The MD5 hash of the object (optional to check written data)
334 @param content_length (integer): The size of the data written
335 @param content_type (string): The MIME content type of the object
336 @param transfer_encoding (string): Set to chunked to specify incremental uploading (if used, Content-Length is ignored)
337 @param copy_from (string): The source path in the form /<container>/<object>
338 @param move_from (string): The source path in the form /<container>/<object>
339 @param source_account (string): The source account to copy/move from
340 @param source_version (string): The source version to copy from
341 @param conent_encoding (string): The encoding of the object
342 @param content_disposition (string): The presentation style of the object
343 @param manifest (string): Object parts prefix in /<container>/<object> form
344 @param permitions (dict): Object permissions in the form (all fields are optional)
345 {'read':[user1, group1, user2, ...], 'write':['user3, group2, group3, ...]}
346 @param public (bool): If true, Object is publicly accessible, if false, not
347 @param metadata (dict): Optional user defined metadata in the form
348 {'meta-key-1':'meta-value-1', 'meta-key-2':'meta-value-2', ...}
350 self.assert_container()
351 param_dict = {} if format is None else dict(format=format)
353 param_dict['hashmap'] = None
354 path=path4url(self.account, self.container, object)+params4url(param_dict)
355 self.set_header('If-Match', if_etag_match)
356 self.set_header('If-None-Match', if_etag_not_match)
357 self.set_header('ETag', etag)
358 self.set_header('Content-Length', content_length)
359 self.set_header('Content-Type', content_type)
360 self.set_header('Transfer-Encoding', transfer_encoding)
361 self.set_header('X-Copy-From', copy_from)
362 self.set_header('X-Move-From', move_from)
363 self.set_header('X-Source-Account', source_account)
364 self.set_header('X-Source-Version', source_version)
365 self.set_header('Content-Encoding', content_encoding)
366 self.set_header('Content-Disposition', content_disposition)
367 self.set_header('X-Object-Manifest', manifest)
369 for permition_type, permition_list in permitions.items():
371 perms = '' #Remove permitions
372 if len(permition_list) == 0:
374 perms += ';'+permition_type if len(perms) > 0 else permition_type
375 perms += '='+list2str(permition_list, seperator=',')
376 self.set_header('X-Object-Sharing', perms)
377 self.set_header('X-Object-Public', public)
378 for key, val in metadata.items():
379 self.set_header('X-Object-Meta-'+key, val)
381 success = kwargs.pop('success', 201)
382 return self.put(path, *args, success=success, **kwargs)
384 def object_copy(self, object, destination, format='json', ignore_content_type=False,
385 if_etag_match=None, if_etag_not_match=None, destination_account=None,
386 content_type=None, content_encoding=None, content_disposition=None, source_version=None,
387 manifest=None, permitions={}, public=False, metadata={}, *args, **kwargs):
388 """ Full Pithos+ COPY at object level
389 --- request parameters ---
390 @param format (string): json (default) or xml
391 @param ignore_content_type (bool): Ignore the supplied Content-Type
392 --- request headers ---
393 @param if_etag_match (string): if provided, copy only results
394 with etag matching with this
395 @param if_etag_not_match (string): if provided, copy only results
396 with etag not matching with this
397 @param destination (string): The destination path in the form /<container>/<object>
398 @param destination_account (string): The destination account to copy to
399 @param content_type (string): The MIME content type of the object
400 @param content_encoding (string): The encoding of the object
401 @param content_disposition (string): The presentation style of the object
402 @param source_version (string): The source version to copy from
403 @param manifest (string): Object parts prefix in /<container>/<object> form
404 @param permitions (dict): Object permissions in the form (all fields are optional)
405 {'read':[user1, group1, user2, ...], 'write':['user3, group2, group3, ...]}
406 permitions override source permitions, removing any old permitions
407 @param public (bool): If true, Object is publicly accessible, if else, not
408 @param metadata (dict): Optional user defined metadata in the form
409 {'meta-key-1':'meta-value-1', 'meta-key-2':'meta-value-2', ...}
410 Metadata are appended to the source metadata. In case of same keys, they
411 replace the old metadata
413 self.assert_container()
414 param_dict = {} if format is None else dict(format=format)
415 if ignore_content_type:
416 param_dict['ignore_content_type'] = None
417 path = path4url(self.account, self.container, object)+params4url(param_dict)
418 self.set_header('If-Match', if_etag_match)
419 self.set_header('If-None-Match', if_etag_not_match)
420 self.set_header('Destination', destination)
421 self.set_header('Destination-Account', destination_account)
422 self.set_header('Content-Type', content_type)
423 self.set_header('Content-Encoding', content_encoding)
424 self.set_header('Content-Disposition', content_disposition)
425 self.set_header('X-Source-Version', source_version)
426 self.set_header('X-Object-Manifest', manifest)
428 for permition_type, permition_list in permitions.items():
430 perms = '' #Remove permitions
431 if len(permition_list) == 0:
433 perms += ';'+permition_type if len(perms) > 0 else permition_type
434 perms += '='+list2str(permition_list, seperator=',')
435 self.set_header('X-Object-Sharing', perms)
436 self.set_header('X-Object-Public', public)
437 for key, val in metadata.items():
438 self.set_header('X-Object-Meta-'+key, val)
439 success = kwargs.pop('success', 201)
440 return self.copy(path, *args, success=success, **kwargs)
442 def object_move(self, object, format='json', ignore_content_type=False,
443 if_etag_match=None, if_etag_not_match=None, destination=None, destination_account=None,
444 content_type=None, content_encoding=None, content_disposition=None, manifest=None,
445 permitions={}, public=False, metadata={}, *args, **kwargs):
446 """ Full Pithos+ COPY at object level
447 --- request parameters ---
448 @param format (string): json (default) or xml
449 @param ignore_content_type (bool): Ignore the supplied Content-Type
450 --- request headers ---
451 @param if_etag_match (string): if provided, return only results
452 with etag matching with this
453 @param if_etag_not_match (string): if provided, return only results
454 with etag not matching with this
455 @param destination (string): The destination path in the form /<container>/<object>
456 @param destination_account (string): The destination account to copy to
457 @param content_type (string): The MIME content type of the object
458 @param content_encoding (string): The encoding of the object
459 @param content_disposition (string): The presentation style of the object
460 @param source_version (string): The source version to copy from
461 @param manifest (string): Object parts prefix in /<container>/<object> form
462 @param permitions (dict): Object permissions in the form (all fields are optional)
463 {'read':[user1, group1, user2, ...], 'write':['user3, group2, group3, ...]}
464 @param public (bool): If true, Object is publicly accessible, if false, not
465 @param metadata (dict): Optional user defined metadata in the form
466 {'meta-key-1':'meta-value-1', 'meta-key-2':'meta-value-2', ...}
468 self.assert_container()
469 param_dict = {} if format is None else dict(format=format)
470 if ignore_content_type:
471 param_dict['ignore_content_type']=None
472 path = path4url(self.account, self.container, object)+params4url(param_dict)
473 self.set_header('If-Match', if_etag_match)
474 self.set_header('If-None-Match', if_etag_not_match)
475 self.set_header('Destination', destination)
476 self.set_header('Destination-Account', destination_account)
477 self.set_header('Content-Type', content_type)
478 self.set_header('Content-Encoding', content_encoding)
479 self.set_header('Content-Disposition', content_disposition)
480 self.set_header('X-Object-Manifest', manifest)
482 for permition_type, permition_list in permitions.items():
484 perms = '' #Remove permitions
485 if len(permition_list) == 0:
487 perms += ';'+permition_type if len(perms) > 0 else permition_type
488 perms += '='+list2str(permition_list, seperator=',')
489 self.set_header('X-Object-Sharing', perms)
490 self.set_header('X-Object-Public', public)
491 for key, val in metadata.items():
492 self.set_header('X-Object-Meta-'+key, val)
493 success = kwargs.pop('success', 201)
494 return self.move(path, *args, success=success, **kwargs)
496 def object_post(self, object, format='json', update=True,
497 if_etag_match=None, if_etag_not_match=None, content_length=None, content_type=None,
498 content_range=None, transfer_encoding=None, content_encoding=None, content_disposition=None,
499 source_object=None, source_account=None, source_version=None, object_bytes=None,
500 manifest=None, permitions={}, public=False, metadata={}, *args, **kwargs):
501 """ Full Pithos+ POST at object level
502 --- request parameters ---
503 @param format (string): json (default) or xml
504 @param update (bool): Do not replace metadata
505 --- request headers ---
506 @param if_etag_match (string): if provided, return only results
507 with etag matching with this
508 @param if_etag_not_match (string): if provided, return only results
509 with etag not matching with this
510 @param content_length (string): The size of the data written
511 @param content_type (string): The MIME content type of the object
512 @param content_range (string): The range of data supplied
513 @param transfer_encoding (string): Set to chunked to specify incremental uploading
514 (if used, Content-Length is ignored)
515 @param content_encoding (string): The encoding of the object
516 @param content_disposition (string): The presentation style of the object
517 @param source_object (string): Update with data from the object at path /<container>/<object>
518 @param source_account (string): The source account to update from
519 @param source_version (string): The source version to copy from
520 @param object_bytes (integer): The updated objects final size
521 @param manifest (string): Object parts prefix in /<container>/<object> form
522 @param permitions (dict): Object permissions in the form (all fields are optional)
523 {'read':[user1, group1, user2, ...], 'write':['user3, group2, group3, ...]}
524 @param public (bool): If true, Object is publicly accessible, if false, not
525 @param metadata (dict): Optional user defined metadata in the form
526 {'meta-key-1':'meta-value-1', 'meta-key-2':'meta-value-2', ...}
528 self.assert_container()
529 param_dict = {} if format is None else dict(format=format)
531 param_dict['update'] = None
532 path = path4url(self.account, self.container, object)+params4url(param_dict)
533 self.set_header('If-Match', if_etag_match)
534 self.set_header('If-None-Match', if_etag_not_match)
535 self.set_header('Content-Length', content_length, iff=transfer_encoding is None)
536 self.set_header('Content-Type', content_type)
537 self.set_header('Content-Range', content_range)
538 self.set_header('Transfer-Encoding', transfer_encoding)
539 self.set_header('Content-Encoding', content_encoding)
540 self.set_header('Content-Disposition', content_disposition)
541 self.set_header('X-Source-Object', source_object)
542 self.set_header('X-Source-Account', source_account)
543 self.set_header('X-Source-Version', source_version)
544 self.set_header('X-Object-Bytes', object_bytes)
545 self.set_header('X-Object-Manifest', manifest)
547 for permition_type, permition_list in permitions.items():
549 perms = '' #Remove permitions
550 if len(permition_list) == 0:
552 perms += ';'+permition_type if len(perms) > 0 else permition_type
553 perms += '='+list2str(permition_list, seperator=',')
554 self.set_header('X-Object-Sharing', perms)
555 self.set_header('X-Object-Public', public)
556 for key, val in metadata.items():
557 self.set_header('X-Object-Meta-'+key, val)
558 success=kwargs.pop('success', (202, 204))
559 return self.post(path, *args, success=success, **kwargs)
561 def object_delete(self, object, until=None, *args, **kwargs):
562 """ Full Pithos+ DELETE at object level
563 --- request parameters ---
564 @param until (string): Optional timestamp
566 self.assert_container()
567 path = path4url(self.account, self.container, object)
568 path += '' if until is None else params4url(dict(until=until))
569 success = kwargs.pop('success', 204)
570 return self.delete(path, *args, success=success, **kwargs)
572 def purge_container(self):
573 self.container_delete(until=unicode(time()))
575 def put_block(self, data, hash):
576 r = self.container_post(update=True, content_type='application/octet-stream',
577 content_length=len(data), data=data, format='json')
579 assert r.json[0] == hash, 'Local hash does not match server'
581 def create_object(self, object, f, size=None, hash_cb=None,
583 """Create an object by uploading only the missing blocks
584 hash_cb is a generator function taking the total number of blocks to
585 be hashed as an argument. Its next() will be called every time a block
587 upload_cb is a generator function with the same properties that is
588 called every time a block is uploaded.
590 self.assert_container()
592 meta = self.get_container_info(self.container)
593 blocksize = int(meta['x-container-block-size'])
594 blockhash = meta['x-container-block-hash']
596 size = size if size is not None else os.fstat(f.fileno()).st_size
597 nblocks = 1 + (size - 1) // blocksize
604 hash_gen = hash_cb(nblocks)
607 for i in range(nblocks):
608 block = f.read(min(blocksize, size - offset))
610 hash = pithos_hash(block, blockhash)
612 map[hash] = (offset, bytes)
617 assert offset == size
619 hashmap = dict(bytes=size, hashes=hashes)
620 content_type = 'application/octet-stream'
621 r = self.object_put(object, format='json', hashmap=True,
622 content_type=content_type, json=hashmap, success=(201, 409))
624 if r.status_code == 201:
630 upload_gen = upload_cb(len(missing))
634 offset, bytes = map[hash]
637 self.put_block(data, hash)
641 r = self.object_put(object, format='json', hashmap=True,
642 content_type=content_type, json=hashmap, success=201)
644 def set_account_group(self, group, usernames):
645 self.account_post(update=True, groups = {group:usernames})
647 def del_account_group(self, group):
648 return self.account_post(update=True, groups={group:[]})
650 def get_account_info(self):
651 r = self.account_head()
652 from datetime import datetime
653 r = self.account_head(if_modified_since=datetime.now())
654 if r.status_code == 401:
655 raise ClientError("No authorization")
658 def get_account_quota(self):
659 return filter_in(self.get_account_info(), 'X-Account-Policy-Quota', exactMatch = True)
661 def get_account_versioning(self):
662 return filter_in(self.get_account_info(), 'X-Account-Policy-Versioning', exactMatch = True)
664 def get_account_meta(self):
665 return filter_in(self.get_account_info(), 'X-Account-Meta-')
667 def get_account_group(self):
668 return filter_in(self.get_account_info(), 'X-Account-Group-')
670 def set_account_meta(self, metapairs):
671 assert(type(metapairs) is dict)
672 self.account_post(update=True, metadata=metapairs)
674 def del_account_meta(self, metakey):
675 self.account_post(update=True, metadata={metakey:''})
677 def set_account_quota(self, quota):
678 self.account_post(update=True, quota=quota)
680 def set_account_versioning(self, versioning):
681 self.account_post(update=True, versioning = versioning)
683 def list_containers(self):
684 r = self.account_get()
687 def get_container_versioning(self, container):
688 return filter_in(self.get_container_info(container), 'X-Container-Policy-Versioning')
690 def get_container_quota(self, container):
691 return filter_in(self.get_container_info(container), 'X-Container-Policy-Quota')
693 def get_container_meta(self, container):
694 return filter_in(self.get_container_info(container), 'X-Container-Meta-')
696 def get_container_object_meta(self, container):
697 return filter_in(self.get_container_info(container), 'X-Container-Object-Meta')
699 def set_container_meta(self, metapairs):
700 assert(type(metapairs) is dict)
701 self.container_post(update=True, metadata=metapairs)
703 def del_container_meta(self, metakey):
704 self.container_post(update=True, metadata={metakey:''})
706 def set_container_quota(self, quota):
707 self.container_post(update=True, quota=quota)
709 def set_container_versioning(self, versioning):
710 self.container_post(update=True, versioning=versioning)
712 def set_object_meta(self, object, metapairs):
713 assert(type(metapairs) is dict)
714 self.object_post(object, update=True, metadata=metapairs)
716 def del_object_meta(self, metakey, object):
717 self.object_post(object, update=True, metadata={metakey:''})
719 def publish_object(self, object):
720 self.object_post(object, update=True, public=True)
722 def unpublish_object(self, object):
723 self.object_post(object, update=True, public=False)
725 def get_object_sharing(self, object):
726 return filter_in(self.get_object_info(object), 'X-Object-Sharing', exactMatch = True)
728 def set_object_sharing(self, object, read_permition = False, write_permition = False):
729 """Give read/write permisions to an object.
730 @param object is the object to change sharing permitions onto
731 @param read_permition is a list of users and user groups that get read permition for this object
732 False means all previous read permitions will be removed
733 @param write_perimition is a list of users and user groups to get write permition for this object
734 False means all previous read permitions will be removed
737 perms['read'] = read_permition if isinstance(read_permition, list) else ''
738 perms['write'] = write_permition if isinstance(write_permition, list) else ''
739 self.object_post(object, update=True, permitions=perms)
741 def del_object_sharing(self, object):
742 self.set_object_sharing(object)
744 def append_object(self, object, source_file, upload_cb = None):
745 """@param upload_db is a generator for showing progress of upload
746 to caller application, e.g. a progress bar. Its next is called
747 whenever a block is uploaded
749 self.assert_container()
750 meta = self.get_container_info(self.container)
751 blocksize = int(meta['x-container-block-size'])
752 filesize = os.fstat(source_file.fileno()).st_size
753 nblocks = 1 + (filesize - 1)//blocksize
755 if upload_cb is not None:
756 upload_gen = upload_cb(nblocks)
757 for i in range(nblocks):
758 block = source_file.read(min(blocksize, filesize - offset))
760 self.object_post(object, update=True,
761 content_range='bytes */*', content_type='application/octet-stream',
762 content_length=len(block), data=block)
763 if upload_cb is not None:
766 def truncate_object(self, object, upto_bytes):
767 self.object_post(object, update=True, content_range='bytes 0-%s/*'%upto_bytes,
768 content_type='application/octet-stream', object_bytes=upto_bytes,
769 source_object=path4url(self.container, object))
771 def overwrite_object(self, object, start, end, source_file, upload_cb=None):
772 """Overwrite a part of an object with given source file
773 @start the part of the remote object to start overwriting from, in bytes
774 @end the part of the remote object to stop overwriting to, in bytes
776 self.assert_container()
777 meta = self.get_container_info(self.container)
778 blocksize = int(meta['x-container-block-size'])
779 filesize = os.fstat(source_file.fileno()).st_size
780 datasize = int(end) - int(start) + 1
781 nblocks = 1 + (datasize - 1)//blocksize
783 if upload_cb is not None:
784 upload_gen = upload_cb(nblocks)
785 for i in range(nblocks):
786 block = source_file.read(min(blocksize, filesize - offset, datasize - offset))
788 self.object_post(object, update=True, content_type='application/octet-stream',
789 content_length=len(block), content_range='bytes %s-%s/*'%(start,end), data=block)
790 if upload_cb is not None: