Revision ca092af4
| b/kamaki/cli/argument.py | ||
|---|---|---|
| 44 | 44 |
try: |
| 45 | 45 |
from progress.bar import ShadyBar as KamakiProgressBar |
| 46 | 46 |
except ImportError: |
| 47 |
try: |
|
| 48 |
from progress.bar import Bar as KamakiProgressBar |
|
| 49 |
except ImportError: |
|
| 50 |
pass |
|
| 47 | 51 |
# progress not installed - pls, pip install progress |
| 48 | 52 |
pass |
| 49 | 53 |
|
| b/kamaki/cli/commands/errors.py | ||
|---|---|---|
| 37 | 37 |
from kamaki.clients import ClientError |
| 38 | 38 |
from kamaki.cli.errors import CLIError, raiseCLIError, CLISyntaxError |
| 39 | 39 |
from kamaki.cli import _debug, kloger |
| 40 |
from kamaki.cli.utils import format_size |
|
| 40 | 41 |
|
| 41 | 42 |
sendlog = logging.getLogger('clients.send')
|
| 42 | 43 |
datasendlog = logging.getLogger('data.send')
|
| ... | ... | |
| 398 | 399 |
' 1. Set store.container variable (permanent)', |
| 399 | 400 |
' /config set store.container <container>', |
| 400 | 401 |
' 2. --container=<container> (temporary, overrides 1)', |
| 401 |
' 3. Use the container:path format (temporary, overrides all)'] |
|
| 402 |
' 3. Use the container:path format (temporary, overrides all)', |
|
| 403 |
'For a list of containers: /store list'] |
|
| 402 | 404 |
|
| 403 | 405 |
@classmethod |
| 404 | 406 |
def connection(this, foo): |
| ... | ... | |
| 429 | 431 |
return foo(self, *args, **kwargs) |
| 430 | 432 |
except ClientError as ce: |
| 431 | 433 |
if ce.status == 404 and 'container' in ('%s' % ce).lower():
|
| 432 |
cont = '%s or %s' if dst_cont else self.container |
|
| 434 |
cont = '%s or %s' % (self.container, dst_cont)\ |
|
| 435 |
if dst_cont else self.container |
|
| 433 | 436 |
raiseCLIError(ce, |
| 434 |
'No container %s in account %s' % (
|
|
| 437 |
'Is container %s in account %s ?' % (
|
|
| 435 | 438 |
cont, |
| 436 | 439 |
self.account), |
| 437 | 440 |
details=this.container_howto) |
| ... | ... | |
| 452 | 455 |
details=this.container_howto) |
| 453 | 456 |
raise |
| 454 | 457 |
return _raise |
| 458 |
|
|
| 459 |
@classmethod |
|
| 460 |
def object_size(this, foo): |
|
| 461 |
def _raise(self, *args, **kwargs): |
|
| 462 |
size = kwargs.get('size', None)
|
|
| 463 |
start = kwargs.get('start', 0)
|
|
| 464 |
end = kwargs.get('end', 0)
|
|
| 465 |
if size: |
|
| 466 |
try: |
|
| 467 |
size = int(size) |
|
| 468 |
except ValueError as ve: |
|
| 469 |
raiseCLIError(ve, |
|
| 470 |
'Invalid file size %s ' % size, |
|
| 471 |
details=['size must be a positive integer'], |
|
| 472 |
importance=1) |
|
| 473 |
else: |
|
| 474 |
try: |
|
| 475 |
start = int(start) |
|
| 476 |
except ValueError as e: |
|
| 477 |
raiseCLIError(e, |
|
| 478 |
'Invalid start value %s in range' % start, |
|
| 479 |
details=['size must be a positive integer'], |
|
| 480 |
importance=1) |
|
| 481 |
try: |
|
| 482 |
end = int(end) |
|
| 483 |
except ValueError as e: |
|
| 484 |
raiseCLIError(e, |
|
| 485 |
'Invalid end value %s in range' % end, |
|
| 486 |
details=['size must be a positive integer'], |
|
| 487 |
importance=1) |
|
| 488 |
if start > end: |
|
| 489 |
raiseCLIError( |
|
| 490 |
'Invalid range %s-%s' % (start, end), |
|
| 491 |
details=['size must be a positive integer'], |
|
| 492 |
importance=1) |
|
| 493 |
size = end - start |
|
| 494 |
try: |
|
| 495 |
return foo(self, *args, **kwargs) |
|
| 496 |
except ClientError as ce: |
|
| 497 |
err_msg = ('%s' % ce).lower()
|
|
| 498 |
if size and (ce.status == 416 or |
|
| 499 |
(ce.status == 400 and\ |
|
| 500 |
'object length is smaller than range length' in err_msg)): |
|
| 501 |
raiseCLIError(ce, |
|
| 502 |
'Remote object %s:%s <= %s %s' % ( |
|
| 503 |
self.container, |
|
| 504 |
self.path, |
|
| 505 |
format_size(size), |
|
| 506 |
('(%sB)' % size) if size >= 1024 else ''))
|
|
| 507 |
raise |
|
| 508 |
return _raise |
|
| b/kamaki/cli/commands/pithos_cli.py | ||
|---|---|---|
| 696 | 696 |
default=False) |
| 697 | 697 |
) |
| 698 | 698 |
|
| 699 |
def main(self, local_path, container___path): |
|
| 700 |
super(self.__class__, self).main( |
|
| 701 |
container___path, |
|
| 702 |
path_is_optional=False) |
|
| 703 |
progress_bar = self.arguments['progress_bar'] |
|
| 704 |
try: |
|
| 705 |
upload_cb = progress_bar.get_generator('Appending blocks')
|
|
| 706 |
except Exception: |
|
| 707 |
upload_cb = None |
|
| 699 |
@errors.generic.all |
|
| 700 |
@errors.pithos.connection |
|
| 701 |
@errors.pithos.container |
|
| 702 |
@errors.pithos.object_path |
|
| 703 |
def _run(self, local_path): |
|
| 704 |
(progress_bar, upload_cb) = self._safe_progress_bar('Appending')
|
|
| 708 | 705 |
try: |
| 709 | 706 |
f = open(local_path, 'rb') |
| 710 | 707 |
self.client.append_object(self.path, f, upload_cb) |
| 711 |
except ClientError as err: |
|
| 712 |
progress_bar.finish() |
|
| 713 |
if err.status == 404: |
|
| 714 |
if 'container' in ('%s' % err).lower():
|
|
| 715 |
raiseCLIError( |
|
| 716 |
err, |
|
| 717 |
'No container %s in account %s'\ |
|
| 718 |
% (self.container, self.account), |
|
| 719 |
details=errors.pithos.container_howto) |
|
| 720 |
elif 'object' in ('%s' % err).lower():
|
|
| 721 |
raiseCLIError( |
|
| 722 |
err, |
|
| 723 |
'No object %s in container %s'\ |
|
| 724 |
% (self.path, self.container), |
|
| 725 |
details=errors.pithos.container_howto) |
|
| 726 |
raise_connection_errors(err) |
|
| 727 |
raiseCLIError(err) |
|
| 728 |
except Exception as e: |
|
| 729 |
progress_bar.finish() |
|
| 730 |
raiseCLIError(e) |
|
| 708 |
except Exception: |
|
| 709 |
self._safe_progress_bar_finish(progress_bar) |
|
| 710 |
raise |
|
| 731 | 711 |
finally: |
| 732 |
progress_bar.finish() |
|
| 712 |
self._safe_progress_bar_finish(progress_bar) |
|
| 713 |
|
|
| 714 |
def main(self, local_path, container___path): |
|
| 715 |
super(self.__class__, self)._run( |
|
| 716 |
container___path, |
|
| 717 |
path_is_optional=False) |
|
| 718 |
self._run(local_path) |
|
| 733 | 719 |
|
| 734 | 720 |
|
| 735 | 721 |
@command(pithos_cmds) |
| 736 | 722 |
class store_truncate(_store_container_command): |
| 737 | 723 |
"""Truncate remote file up to a size (default is 0)""" |
| 738 | 724 |
|
| 725 |
@errors.generic.all |
|
| 726 |
@errors.pithos.connection |
|
| 727 |
@errors.pithos.container |
|
| 728 |
@errors.pithos.object_path |
|
| 729 |
@errors.pithos.object_size |
|
| 730 |
def _run(self, size=0): |
|
| 731 |
self.client.truncate_object(self.path, size) |
|
| 732 |
|
|
| 739 | 733 |
def main(self, container___path, size=0): |
| 740 |
super(self.__class__, self).main(container___path) |
|
| 741 |
try: |
|
| 742 |
self.client.truncate_object(self.path, size) |
|
| 743 |
except ClientError as err: |
|
| 744 |
if err.status == 404: |
|
| 745 |
if 'container' in ('%s' % err).lower():
|
|
| 746 |
raiseCLIError( |
|
| 747 |
err, |
|
| 748 |
'No container %s in account %s'\ |
|
| 749 |
% (self.container, self.account), |
|
| 750 |
details=errors.pithos.container_howto) |
|
| 751 |
elif 'object' in ('%s' % err).lower():
|
|
| 752 |
raiseCLIError( |
|
| 753 |
err, |
|
| 754 |
'No object %s in container %s'\ |
|
| 755 |
% (self.path, self.container), |
|
| 756 |
details=errors.pithos.container_howto) |
|
| 757 |
if err.status == 400 and\ |
|
| 758 |
'object length is smaller than range length'\ |
|
| 759 |
in ('%s' % err).lower():
|
|
| 760 |
raiseCLIError(err, 'Object %s:%s <= %sb' % ( |
|
| 761 |
self.container, |
|
| 762 |
self.path, |
|
| 763 |
size)) |
|
| 764 |
raise_connection_errors(err) |
|
| 765 |
raiseCLIError(err) |
|
| 766 |
except Exception as e: |
|
| 767 |
raiseCLIError(e) |
|
| 734 |
super(self.__class__, self)._run(container___path) |
|
| 735 |
self._run(size=size) |
|
| 768 | 736 |
|
| 769 | 737 |
|
| 770 | 738 |
@command(pithos_cmds) |
| ... | ... | |
| 784 | 752 |
default=False) |
| 785 | 753 |
) |
| 786 | 754 |
|
| 787 |
def main(self, local_path, container____path__, start, end): |
|
| 788 |
(start, end) = check_range(start, end) |
|
| 789 |
super(self.__class__, self).main(container____path__) |
|
| 790 |
try: |
|
| 791 |
f = open(local_path, 'rb') |
|
| 792 |
f.seek(0, 2) |
|
| 793 |
f_size = f.tell() |
|
| 794 |
f.seek(start, 0) |
|
| 795 |
except Exception as e: |
|
| 796 |
raiseCLIError(e) |
|
| 797 |
progress_bar = self.arguments['progress_bar'] |
|
| 798 |
try: |
|
| 799 |
upload_cb = progress_bar.get_generator( |
|
| 800 |
'Overwriting %s blocks' % end - start) |
|
| 801 |
except Exception: |
|
| 802 |
upload_cb = None |
|
| 755 |
def _open_file(self, local_path, start): |
|
| 756 |
f = open(path.abspath(local_path), 'rb') |
|
| 757 |
f.seek(0, 2) |
|
| 758 |
f_size = f.tell() |
|
| 759 |
f.seek(start, 0) |
|
| 760 |
return (f, f_size) |
|
| 761 |
|
|
| 762 |
@errors.generic.all |
|
| 763 |
@errors.pithos.connection |
|
| 764 |
@errors.pithos.container |
|
| 765 |
@errors.pithos.object_path |
|
| 766 |
@errors.pithos.object_size |
|
| 767 |
def _run(self, local_path, start, end): |
|
| 768 |
(start, end) = (int(start), int(end)) |
|
| 769 |
(f, f_size) = self._open_file(local_path, start) |
|
| 770 |
(progress_bar, upload_cb) = self._safe_progress_bar( |
|
| 771 |
'Overwrite %s bytes' % (end - start)) |
|
| 803 | 772 |
try: |
| 804 |
self.path = self.path if self.path else path.basename(local_path) |
|
| 805 | 773 |
self.client.overwrite_object( |
| 806 | 774 |
obj=self.path, |
| 807 | 775 |
start=start, |
| 808 | 776 |
end=end, |
| 809 | 777 |
source_file=f, |
| 810 | 778 |
upload_cb=upload_cb) |
| 811 |
except ClientError as err: |
|
| 812 |
progress_bar.finish() |
|
| 813 |
if (err.status == 400 and\ |
|
| 814 |
'content length does not match range' in ('%s' % err).lower()) or\
|
|
| 815 |
err.status == 416: |
|
| 816 |
raiseCLIError(err, details=[ |
|
| 817 |
'Content size: %s' % f_size, |
|
| 818 |
'Range: %s-%s (size: %s)' % (start, end, end - start)]) |
|
| 819 |
elif err.status == 404: |
|
| 820 |
if 'container' in ('%s' % err).lower():
|
|
| 821 |
raiseCLIError( |
|
| 822 |
err, |
|
| 823 |
'No container %s in account %s'\ |
|
| 824 |
% (self.container, self.account), |
|
| 825 |
details=errors.pithos.container_howto) |
|
| 826 |
elif 'object' in ('%s' % err).lower():
|
|
| 827 |
raiseCLIError( |
|
| 828 |
err, |
|
| 829 |
'No object %s in container %s'\ |
|
| 830 |
% (self.path, self.container), |
|
| 831 |
details=errors.pithos.container_howto) |
|
| 832 |
raise_connection_errors(err) |
|
| 833 |
raiseCLIError(err) |
|
| 834 |
except Exception as e: |
|
| 835 |
progress_bar.finish() |
|
| 836 |
raiseCLIError(e) |
|
| 779 |
except Exception: |
|
| 780 |
self._safe_progress_bar_finish(progress_bar) |
|
| 781 |
raise |
|
| 837 | 782 |
finally: |
| 838 |
progress_bar.finish() |
|
| 783 |
self._safe_progress_bar_finish(progress_bar) |
|
| 784 |
|
|
| 785 |
def main(self, local_path, container____path__, start, end): |
|
| 786 |
super(self.__class__, self)._run(container____path__) |
|
| 787 |
self.path = self.path if self.path else path.basename(local_path) |
|
| 788 |
self._run(local_path=local_path, start=start, end=end) |
|
| 839 | 789 |
|
| 840 | 790 |
|
| 841 | 791 |
@command(pithos_cmds) |
| b/kamaki/clients/pithos.py | ||
|---|---|---|
| 908 | 908 |
|
| 909 | 909 |
:returns: (dict) |
| 910 | 910 |
""" |
| 911 |
r = self.object_head(obj, version=version) |
|
| 912 |
return r.headers |
|
| 911 |
try: |
|
| 912 |
r = self.object_head(obj, version=version) |
|
| 913 |
return r.headers |
|
| 914 |
except ClientError as ce: |
|
| 915 |
if ce.status == 404: |
|
| 916 |
raise ClientError('Object not found', status=404)
|
|
| 917 |
raise |
|
| 913 | 918 |
|
| 914 | 919 |
def get_object_meta(self, obj, version=None): |
| 915 | 920 |
""" |
| ... | ... | |
| 985 | 990 |
filesize = fstat(source_file.fileno()).st_size |
| 986 | 991 |
nblocks = 1 + (filesize - 1) // blocksize |
| 987 | 992 |
offset = 0 |
| 988 |
if upload_cb is not None:
|
|
| 993 |
if upload_cb: |
|
| 989 | 994 |
upload_gen = upload_cb(nblocks) |
| 995 |
upload_gen.next() |
|
| 990 | 996 |
for i in range(nblocks): |
| 991 | 997 |
block = source_file.read(min(blocksize, filesize - offset)) |
| 992 | 998 |
offset += len(block) |
| ... | ... | |
| 998 | 1004 |
data=block) |
| 999 | 1005 |
r.release() |
| 1000 | 1006 |
|
| 1001 |
if upload_cb is not None:
|
|
| 1007 |
if upload_cb: |
|
| 1002 | 1008 |
upload_gen.next() |
| 1003 | 1009 |
|
| 1004 | 1010 |
def truncate_object(self, obj, upto_bytes): |
| ... | ... | |
| 1034 | 1040 |
:param upload_db: progress.bar for uploading |
| 1035 | 1041 |
""" |
| 1036 | 1042 |
|
| 1043 |
r = self.get_object_info(obj) |
|
| 1044 |
rf_size = int(r['content-length']) |
|
| 1045 |
if rf_size < int(start): |
|
| 1046 |
raise ClientError( |
|
| 1047 |
'Range start exceeds file size', |
|
| 1048 |
status=416) |
|
| 1049 |
elif rf_size < int(end): |
|
| 1050 |
raise ClientError( |
|
| 1051 |
'Range end exceeds file size', |
|
| 1052 |
status=416) |
|
| 1037 | 1053 |
self._assert_container() |
| 1038 | 1054 |
meta = self.get_container_info() |
| 1039 | 1055 |
blocksize = int(meta['x-container-block-size']) |
| ... | ... | |
| 1041 | 1057 |
datasize = int(end) - int(start) + 1 |
| 1042 | 1058 |
nblocks = 1 + (datasize - 1) // blocksize |
| 1043 | 1059 |
offset = 0 |
| 1044 |
if upload_cb is not None:
|
|
| 1060 |
if upload_cb: |
|
| 1045 | 1061 |
upload_gen = upload_cb(nblocks) |
| 1062 |
upload_gen.next() |
|
| 1046 | 1063 |
for i in range(nblocks): |
| 1047 | 1064 |
block = source_file.read(min(blocksize, |
| 1048 | 1065 |
filesize - offset, |
| 1049 | 1066 |
datasize - offset)) |
| 1050 |
offset += len(block) |
|
| 1051 | 1067 |
r = self.object_post(obj, |
| 1052 | 1068 |
update=True, |
| 1053 | 1069 |
content_type='application/octet-stream', |
| 1054 | 1070 |
content_length=len(block), |
| 1055 |
content_range='bytes %s-%s/*' % (start, end), |
|
| 1071 |
content_range='bytes %s-%s/*' % ( |
|
| 1072 |
start + offset, |
|
| 1073 |
start + offset + len(block) - 1), |
|
| 1056 | 1074 |
data=block) |
| 1075 |
offset += len(block) |
|
| 1057 | 1076 |
r.release() |
| 1058 | 1077 |
|
| 1059 |
if upload_cb is not None:
|
|
| 1078 |
if upload_cb: |
|
| 1060 | 1079 |
upload_gen.next() |
| 1061 | 1080 |
|
| 1062 | 1081 |
def copy_object(self, src_container, src_object, dst_container, |
| b/setup.py | ||
|---|---|---|
| 34 | 34 |
# or implied, of GRNET S.A. |
| 35 | 35 |
|
| 36 | 36 |
from setuptools import setup |
| 37 |
#from sys import version_info
|
|
| 37 |
from sys import version_info |
|
| 38 | 38 |
import collections |
| 39 | 39 |
|
| 40 | 40 |
import kamaki |
| ... | ... | |
| 42 | 42 |
|
| 43 | 43 |
optional = ['ansicolors', |
| 44 | 44 |
'progress>=1.0.2'] |
| 45 |
requires = ['objpool', |
|
| 46 |
'argparse'] |
|
| 45 |
requires = ['objpool'] |
|
| 46 |
|
|
| 47 |
if version_info[:1] == (2, 6): |
|
| 48 |
requires.append('argparse')
|
|
| 47 | 49 |
|
| 48 | 50 |
if not hasattr(collections, "OrderedDict"): # Python 2.6 |
| 49 | 51 |
requires.append("ordereddict")
|
Also available in: Unified diff