46 |
46 |
from StringIO import StringIO
|
47 |
47 |
|
48 |
48 |
|
49 |
|
def pithos_hash(block, blockhash):
|
|
49 |
def _pithos_hash(block, blockhash):
|
50 |
50 |
h = newhashlib(blockhash)
|
51 |
51 |
h.update(block.rstrip('\x00'))
|
52 |
52 |
return h.hexdigest()
|
... | ... | |
74 |
74 |
super(PithosClient, self).__init__(base_url, token, account, container)
|
75 |
75 |
|
76 |
76 |
def purge_container(self):
|
|
77 |
"""Delete an empty container and destroy associated blocks
|
|
78 |
"""
|
77 |
79 |
r = self.container_delete(until=unicode(time()))
|
78 |
80 |
r.release()
|
79 |
81 |
|
... | ... | |
86 |
88 |
content_type=None,
|
87 |
89 |
sharing=None,
|
88 |
90 |
public=None):
|
|
91 |
"""
|
|
92 |
:param obj: (str) remote object path
|
|
93 |
|
|
94 |
:param f: open file descriptor
|
|
95 |
|
|
96 |
:param withHashFile: (bool)
|
|
97 |
|
|
98 |
:param size: (int) size of data to upload
|
|
99 |
|
|
100 |
:param etag: (str)
|
|
101 |
|
|
102 |
:param content_encoding: (str)
|
|
103 |
|
|
104 |
:param content_disposition: (str)
|
|
105 |
|
|
106 |
:param content_type: (str)
|
|
107 |
|
|
108 |
:param sharing: {'read':[user and/or grp names],
|
|
109 |
'write':[usr and/or grp names]}
|
|
110 |
|
|
111 |
:param public: (bool)
|
|
112 |
"""
|
89 |
113 |
self.assert_container()
|
90 |
114 |
|
91 |
115 |
if withHashFile:
|
... | ... | |
112 |
136 |
success=201)
|
113 |
137 |
r.release()
|
114 |
138 |
|
115 |
|
# upload_* auxiliary methods
|
116 |
|
def put_block_async(self, data, hash):
|
117 |
|
event = SilentEvent(method=self.put_block, data=data, hash=hash)
|
118 |
|
event.start()
|
119 |
|
return event
|
120 |
|
|
121 |
|
def put_block(self, data, hash):
|
122 |
|
r = self.container_post(update=True,
|
123 |
|
content_type='application/octet-stream',
|
124 |
|
content_length=len(data),
|
125 |
|
data=data,
|
126 |
|
format='json')
|
127 |
|
assert r.json[0] == hash, 'Local hash does not match server'
|
128 |
|
|
129 |
139 |
def create_object_by_manifestation(self, obj,
|
130 |
140 |
etag=None,
|
131 |
141 |
content_encoding=None,
|
... | ... | |
133 |
143 |
content_type=None,
|
134 |
144 |
sharing=None,
|
135 |
145 |
public=None):
|
|
146 |
"""
|
|
147 |
:param obj: (str) remote object path
|
|
148 |
|
|
149 |
:param etag: (str)
|
|
150 |
|
|
151 |
:param content_encoding: (str)
|
|
152 |
|
|
153 |
:param content_disposition: (str)
|
|
154 |
|
|
155 |
:param content_type: (str)
|
|
156 |
|
|
157 |
:param sharing: {'read':[user and/or grp names],
|
|
158 |
'write':[usr and/or grp names]}
|
|
159 |
|
|
160 |
:param public: (bool)
|
|
161 |
"""
|
136 |
162 |
self.assert_container()
|
137 |
163 |
r = self.object_put(obj,
|
138 |
164 |
content_length=0,
|
... | ... | |
145 |
171 |
manifest='%s/%s' % (self.container, obj))
|
146 |
172 |
r.release()
|
147 |
173 |
|
|
174 |
# upload_* auxiliary methods
|
|
175 |
def _put_block_async(self, data, hash):
|
|
176 |
event = SilentEvent(method=self._put_block, data=data, hash=hash)
|
|
177 |
event.start()
|
|
178 |
return event
|
|
179 |
|
|
180 |
def _put_block(self, data, hash):
|
|
181 |
r = self.container_post(update=True,
|
|
182 |
content_type='application/octet-stream',
|
|
183 |
content_length=len(data),
|
|
184 |
data=data,
|
|
185 |
format='json')
|
|
186 |
assert r.json[0] == hash, 'Local hash does not match server'
|
|
187 |
|
148 |
188 |
def _get_file_block_info(self, fileobj, size=None):
|
149 |
189 |
meta = self.get_container_info()
|
150 |
190 |
blocksize = int(meta['x-container-block-size'])
|
... | ... | |
197 |
237 |
for i in range(nblocks):
|
198 |
238 |
block = fileobj.read(min(blocksize, size - offset))
|
199 |
239 |
bytes = len(block)
|
200 |
|
hash = pithos_hash(block, blockhash)
|
|
240 |
hash = _pithos_hash(block, blockhash)
|
201 |
241 |
hashes.append(hash)
|
202 |
242 |
hmap[hash] = (offset, bytes)
|
203 |
243 |
offset += bytes
|
... | ... | |
219 |
259 |
offset, bytes = hmap[hash]
|
220 |
260 |
fileobj.seek(offset)
|
221 |
261 |
data = fileobj.read(bytes)
|
222 |
|
r = self.put_block_async(data, hash)
|
|
262 |
r = self._put_block_async(data, hash)
|
223 |
263 |
flying.append(r)
|
224 |
264 |
unfinished = []
|
225 |
265 |
for i, thread in enumerate(flying):
|
... | ... | |
260 |
300 |
content_type=None,
|
261 |
301 |
sharing=None,
|
262 |
302 |
public=None):
|
|
303 |
"""Upload an object using multiple connections (threads)
|
|
304 |
|
|
305 |
:param obj: (str) remote object path
|
|
306 |
|
|
307 |
:param f: open file descriptor (rb)
|
|
308 |
|
|
309 |
:param hash_cb: optional progress.bar object for calculating hashes
|
|
310 |
|
|
311 |
:param upload_cb: optional progress.bar object for uploading
|
|
312 |
|
|
313 |
:param etag: (str)
|
|
314 |
|
|
315 |
:param content_encoding: (str)
|
|
316 |
|
|
317 |
:param content_disposition: (str)
|
|
318 |
|
|
319 |
:param content_type: (str)
|
|
320 |
|
|
321 |
:param sharing: {'read':[user and/or grp names],
|
|
322 |
'write':[usr and/or grp names]}
|
|
323 |
|
|
324 |
:param public: (bool)
|
|
325 |
"""
|
263 |
326 |
self.assert_container()
|
264 |
327 |
|
265 |
328 |
#init
|
... | ... | |
308 |
371 |
def _get_remote_blocks_info(self, obj, **restargs):
|
309 |
372 |
#retrieve object hashmap
|
310 |
373 |
myrange = restargs.pop('data_range', None)
|
311 |
|
hashmap = self.get_object_hashmap(obj, **restargs)
|
|
374 |
hashmap = self.get_object_hashmapp(obj, **restargs)
|
312 |
375 |
restargs['data_range'] = myrange
|
313 |
376 |
blocksize = int(hashmap['block_size'])
|
314 |
377 |
blockhash = hashmap['block_hash']
|
... | ... | |
431 |
494 |
dst,
|
432 |
495 |
download_cb=None,
|
433 |
496 |
version=None,
|
434 |
|
overide=False,
|
435 |
497 |
resume=False,
|
436 |
498 |
range=None,
|
437 |
499 |
if_match=None,
|
438 |
500 |
if_none_match=None,
|
439 |
501 |
if_modified_since=None,
|
440 |
502 |
if_unmodified_since=None):
|
|
503 |
"""Download an object using multiple connections (threads) and
|
|
504 |
writing to random parts of the file
|
|
505 |
|
|
506 |
:param obj: (str) remote object path
|
|
507 |
|
|
508 |
:param dst: open file descriptor (wb+)
|
|
509 |
|
|
510 |
:param download_cb: optional progress.bar object for downloading
|
|
511 |
|
|
512 |
:param version: (str) file version
|
|
513 |
|
|
514 |
:param resume: (bool) if set, preserve already downloaded file parts
|
|
515 |
|
|
516 |
:param range: (str) from-to where from and to are integers denoting
|
|
517 |
file positions in bytes
|
|
518 |
|
|
519 |
:param if_match: (str)
|
|
520 |
|
|
521 |
:param if_none_match: (str)
|
|
522 |
|
|
523 |
:param if_modified_since: (str) formated date
|
|
524 |
|
|
525 |
:param if_unmodified_since: (str) formated date
|
|
526 |
"""
|
441 |
527 |
|
442 |
528 |
restargs = dict(version=version,
|
443 |
529 |
data_range=None if range is None else 'bytes=%s' % range,
|
... | ... | |
495 |
581 |
except:
|
496 |
582 |
break
|
497 |
583 |
|
498 |
|
# Untested - except is download_object is tested first
|
499 |
|
def get_object_hashmap(self, obj,
|
|
584 |
def get_object_hashmapp(self, obj,
|
500 |
585 |
version=None,
|
501 |
586 |
if_match=None,
|
502 |
587 |
if_none_match=None,
|
503 |
588 |
if_modified_since=None,
|
504 |
589 |
if_unmodified_since=None,
|
505 |
590 |
data_range=None):
|
|
591 |
"""
|
|
592 |
:param obj: (str) remote object path
|
|
593 |
|
|
594 |
:param if_match: (str)
|
|
595 |
|
|
596 |
:param if_none_match: (str)
|
|
597 |
|
|
598 |
:param if_modified_since: (str) formated date
|
|
599 |
|
|
600 |
:param if_unmodified_since: (str) formated date
|
|
601 |
|
|
602 |
:param data_range: (str) from-to where from and to are integers
|
|
603 |
denoting file positions in bytes
|
|
604 |
|
|
605 |
:returns: (list)
|
|
606 |
"""
|
506 |
607 |
try:
|
507 |
608 |
r = self.object_get(obj,
|
508 |
609 |
hashmap=True,
|
... | ... | |
519 |
620 |
return r.json
|
520 |
621 |
|
521 |
622 |
def set_account_group(self, group, usernames):
|
|
623 |
"""
|
|
624 |
:param group: (str)
|
|
625 |
|
|
626 |
:param usernames: (list)
|
|
627 |
"""
|
522 |
628 |
r = self.account_post(update=True, groups={group: usernames})
|
523 |
629 |
r.release()
|
524 |
630 |
|
525 |
631 |
def del_account_group(self, group):
|
|
632 |
"""
|
|
633 |
:param group: (str)
|
|
634 |
"""
|
526 |
635 |
r = self.account_post(update=True, groups={group: []})
|
527 |
636 |
r.release()
|
528 |
637 |
|
529 |
638 |
def get_account_info(self, until=None):
|
|
639 |
"""
|
|
640 |
:param until: (str) formated date
|
|
641 |
|
|
642 |
:returns: (dict)
|
|
643 |
"""
|
530 |
644 |
r = self.account_head(until=until)
|
531 |
645 |
if r.status_code == 401:
|
532 |
646 |
raise ClientError("No authorization")
|
533 |
647 |
return r.headers
|
534 |
648 |
|
535 |
649 |
def get_account_quota(self):
|
|
650 |
"""
|
|
651 |
:returns: (dict)
|
|
652 |
"""
|
536 |
653 |
return filter_in(self.get_account_info(),
|
537 |
654 |
'X-Account-Policy-Quota',
|
538 |
655 |
exactMatch=True)
|
539 |
656 |
|
540 |
657 |
def get_account_versioning(self):
|
|
658 |
"""
|
|
659 |
:returns: (dict)
|
|
660 |
"""
|
541 |
661 |
return filter_in(self.get_account_info(),
|
542 |
662 |
'X-Account-Policy-Versioning',
|
543 |
663 |
exactMatch=True)
|
544 |
664 |
|
545 |
665 |
def get_account_meta(self, until=None):
|
|
666 |
"""
|
|
667 |
:meta until: (str) formated date
|
|
668 |
|
|
669 |
:returns: (dict)
|
|
670 |
"""
|
546 |
671 |
return filter_in(self.get_account_info(until=until), 'X-Account-Meta-')
|
547 |
672 |
|
548 |
673 |
def get_account_group(self):
|
|
674 |
"""
|
|
675 |
:returns: (dict)
|
|
676 |
"""
|
549 |
677 |
return filter_in(self.get_account_info(), 'X-Account-Group-')
|
550 |
678 |
|
551 |
679 |
def set_account_meta(self, metapairs):
|
|
680 |
"""
|
|
681 |
:param metapairs: (dict) {key1:val1, key2:val2, ...}
|
|
682 |
"""
|
552 |
683 |
assert(type(metapairs) is dict)
|
553 |
684 |
r = self.account_post(update=True, metadata=metapairs)
|
554 |
685 |
r.release()
|
555 |
686 |
|
556 |
687 |
def del_account_meta(self, metakey):
|
|
688 |
"""
|
|
689 |
:param metakey: (str) metadatum key
|
|
690 |
"""
|
557 |
691 |
r = self.account_post(update=True, metadata={metakey: ''})
|
558 |
692 |
r.release()
|
559 |
693 |
|
560 |
694 |
def set_account_quota(self, quota):
|
|
695 |
"""
|
|
696 |
:param quota: (int)
|
|
697 |
"""
|
561 |
698 |
r = self.account_post(update=True, quota=quota)
|
562 |
699 |
r.release()
|
563 |
700 |
|
564 |
701 |
def set_account_versioning(self, versioning):
|
|
702 |
"""
|
|
703 |
"param versioning: (str)
|
|
704 |
"""
|
565 |
705 |
r = self.account_post(update=True, versioning=versioning)
|
566 |
706 |
r.release()
|
567 |
707 |
|
568 |
708 |
def list_containers(self):
|
|
709 |
"""
|
|
710 |
:returns: (dict)
|
|
711 |
"""
|
569 |
712 |
r = self.account_get()
|
570 |
713 |
return r.json
|
571 |
714 |
|
572 |
715 |
def del_container(self, until=None, delimiter=None):
|
|
716 |
"""
|
|
717 |
:param until: (str) formated date
|
|
718 |
|
|
719 |
:param delimiter: (str)
|
|
720 |
|
|
721 |
:raises ClientError: 404 Container does not exist
|
|
722 |
|
|
723 |
:raises ClientError: 409 Container is not empty
|
|
724 |
"""
|
573 |
725 |
self.assert_container()
|
574 |
726 |
r = self.container_delete(until=until,
|
575 |
727 |
delimiter=delimiter,
|
... | ... | |
583 |
735 |
r.status_code)
|
584 |
736 |
|
585 |
737 |
def get_container_versioning(self, container):
|
|
738 |
"""
|
|
739 |
:param container: (str)
|
|
740 |
|
|
741 |
:returns: (dict)
|
|
742 |
"""
|
586 |
743 |
self.container = container
|
587 |
744 |
return filter_in(self.get_container_info(),
|
588 |
745 |
'X-Container-Policy-Versioning')
|
589 |
746 |
|
590 |
747 |
def get_container_quota(self, container):
|
|
748 |
"""
|
|
749 |
:param container: (str)
|
|
750 |
|
|
751 |
:returns: (dict)
|
|
752 |
"""
|
591 |
753 |
self.container = container
|
592 |
754 |
return filter_in(self.get_container_info(), 'X-Container-Policy-Quota')
|
593 |
755 |
|
594 |
756 |
def get_container_info(self, until=None):
|
|
757 |
"""
|
|
758 |
:param until: (str) formated date
|
|
759 |
|
|
760 |
:returns: (dict)
|
|
761 |
"""
|
595 |
762 |
r = self.container_head(until=until)
|
596 |
763 |
return r.headers
|
597 |
764 |
|
598 |
765 |
def get_container_meta(self, until=None):
|
|
766 |
"""
|
|
767 |
:param until: (str) formated date
|
|
768 |
|
|
769 |
:returns: (dict)
|
|
770 |
"""
|
599 |
771 |
return filter_in(self.get_container_info(until=until),
|
600 |
772 |
'X-Container-Meta')
|
601 |
773 |
|
602 |
774 |
def get_container_object_meta(self, until=None):
|
|
775 |
"""
|
|
776 |
:param until: (str) formated date
|
|
777 |
|
|
778 |
:returns: (dict)
|
|
779 |
"""
|
603 |
780 |
return filter_in(self.get_container_info(until=until),
|
604 |
781 |
'X-Container-Object-Meta')
|
605 |
782 |
|
606 |
783 |
def set_container_meta(self, metapairs):
|
|
784 |
"""
|
|
785 |
:param metapairs: (dict) {key1:val1, key2:val2, ...}
|
|
786 |
"""
|
607 |
787 |
assert(type(metapairs) is dict)
|
608 |
788 |
r = self.container_post(update=True, metadata=metapairs)
|
609 |
789 |
r.release()
|
610 |
790 |
|
611 |
791 |
def del_container_meta(self, metakey):
|
|
792 |
"""
|
|
793 |
:param metakey: (str) metadatum key
|
|
794 |
"""
|
612 |
795 |
r = self.container_post(update=True, metadata={metakey: ''})
|
613 |
796 |
r.release()
|
614 |
797 |
|
615 |
798 |
def set_container_quota(self, quota):
|
|
799 |
"""
|
|
800 |
:param quota: (int)
|
|
801 |
"""
|
616 |
802 |
r = self.container_post(update=True, quota=quota)
|
617 |
803 |
r.release()
|
618 |
804 |
|
619 |
805 |
def set_container_versioning(self, versioning):
|
|
806 |
"""
|
|
807 |
:param versioning: (str)
|
|
808 |
"""
|
620 |
809 |
r = self.container_post(update=True, versioning=versioning)
|
621 |
810 |
r.release()
|
622 |
811 |
|
623 |
812 |
def del_object(self, obj, until=None, delimiter=None):
|
|
813 |
"""
|
|
814 |
:param obj: (str) remote object path
|
|
815 |
|
|
816 |
:param until: (str) formated date
|
|
817 |
|
|
818 |
:param delimiter: (str)
|
|
819 |
"""
|
624 |
820 |
self.assert_container()
|
625 |
821 |
r = self.object_delete(obj, until=until, delimiter=delimiter)
|
626 |
822 |
r.release()
|
627 |
823 |
|
628 |
|
def set_object_meta(self, object, metapairs):
|
|
824 |
def set_object_meta(self, obj, metapairs):
|
|
825 |
"""
|
|
826 |
:param obj: (str) remote object path
|
|
827 |
|
|
828 |
:param metapairs: (dict) {key1:val1, key2:val2, ...}
|
|
829 |
"""
|
629 |
830 |
assert(type(metapairs) is dict)
|
630 |
|
r = self.object_post(object, update=True, metadata=metapairs)
|
|
831 |
r = self.object_post(obj, update=True, metadata=metapairs)
|
631 |
832 |
r.release()
|
632 |
833 |
|
633 |
|
def del_object_meta(self, metakey, object):
|
|
834 |
def del_object_meta(self, metakey, obj):
|
|
835 |
"""
|
|
836 |
:param metakey: (str) metadatum key
|
|
837 |
|
|
838 |
:param obj: (str) remote object path
|
|
839 |
"""
|
634 |
840 |
r = self.object_post(object, update=True, metadata={metakey: ''})
|
635 |
841 |
r.release()
|
636 |
842 |
|