From ca092af4f6349a4b5a3ca637b7931d082ac59f79 Mon Sep 17 00:00:00 2001 From: Stavros Sachtouris Date: Fri, 25 Jan 2013 17:58:50 +0200 Subject: [PATCH] Adjust up to store-overwrite --- kamaki/cli/argument.py | 4 + kamaki/cli/commands/errors.py | 60 ++++++++++++++- kamaki/cli/commands/pithos_cli.py | 154 +++++++++++++------------------------ kamaki/clients/pithos.py | 35 +++++++-- setup.py | 8 +- 5 files changed, 145 insertions(+), 116 deletions(-) diff --git a/kamaki/cli/argument.py b/kamaki/cli/argument.py index 0a72ecc..40e8b56 100644 --- a/kamaki/cli/argument.py +++ b/kamaki/cli/argument.py @@ -44,6 +44,10 @@ from argparse import RawDescriptionHelpFormatter try: from progress.bar import ShadyBar as KamakiProgressBar except ImportError: + try: + from progress.bar import Bar as KamakiProgressBar + except ImportError: + pass # progress not installed - pls, pip install progress pass diff --git a/kamaki/cli/commands/errors.py b/kamaki/cli/commands/errors.py index 0cf9c8b..75f6e09 100644 --- a/kamaki/cli/commands/errors.py +++ b/kamaki/cli/commands/errors.py @@ -37,6 +37,7 @@ import logging from kamaki.clients import ClientError from kamaki.cli.errors import CLIError, raiseCLIError, CLISyntaxError from kamaki.cli import _debug, kloger +from kamaki.cli.utils import format_size sendlog = logging.getLogger('clients.send') datasendlog = logging.getLogger('data.send') @@ -398,7 +399,8 @@ class pithos(object): ' 1. Set store.container variable (permanent)', ' /config set store.container ', ' 2. --container= (temporary, overrides 1)', - ' 3. Use the container:path format (temporary, overrides all)'] + ' 3. Use the container:path format (temporary, overrides all)', + 'For a list of containers: /store list'] @classmethod def connection(this, foo): @@ -429,9 +431,10 @@ class pithos(object): return foo(self, *args, **kwargs) except ClientError as ce: if ce.status == 404 and 'container' in ('%s' % ce).lower(): - cont = '%s or %s' if dst_cont else self.container + cont = '%s or %s' % (self.container, dst_cont)\ + if dst_cont else self.container raiseCLIError(ce, - 'No container %s in account %s' % ( + 'Is container %s in account %s ?' % ( cont, self.account), details=this.container_howto) @@ -452,3 +455,54 @@ class pithos(object): details=this.container_howto) raise return _raise + + @classmethod + def object_size(this, foo): + def _raise(self, *args, **kwargs): + size = kwargs.get('size', None) + start = kwargs.get('start', 0) + end = kwargs.get('end', 0) + if size: + try: + size = int(size) + except ValueError as ve: + raiseCLIError(ve, + 'Invalid file size %s ' % size, + details=['size must be a positive integer'], + importance=1) + else: + try: + start = int(start) + except ValueError as e: + raiseCLIError(e, + 'Invalid start value %s in range' % start, + details=['size must be a positive integer'], + importance=1) + try: + end = int(end) + except ValueError as e: + raiseCLIError(e, + 'Invalid end value %s in range' % end, + details=['size must be a positive integer'], + importance=1) + if start > end: + raiseCLIError( + 'Invalid range %s-%s' % (start, end), + details=['size must be a positive integer'], + importance=1) + size = end - start + try: + return foo(self, *args, **kwargs) + except ClientError as ce: + err_msg = ('%s' % ce).lower() + if size and (ce.status == 416 or + (ce.status == 400 and\ + 'object length is smaller than range length' in err_msg)): + raiseCLIError(ce, + 'Remote object %s:%s <= %s %s' % ( + self.container, + self.path, + format_size(size), + ('(%sB)' % size) if size >= 1024 else '')) + raise + return _raise diff --git a/kamaki/cli/commands/pithos_cli.py b/kamaki/cli/commands/pithos_cli.py index cd806c6..f898ece 100644 --- a/kamaki/cli/commands/pithos_cli.py +++ b/kamaki/cli/commands/pithos_cli.py @@ -696,75 +696,43 @@ class store_append(_store_container_command): default=False) ) - def main(self, local_path, container___path): - super(self.__class__, self).main( - container___path, - path_is_optional=False) - progress_bar = self.arguments['progress_bar'] - try: - upload_cb = progress_bar.get_generator('Appending blocks') - except Exception: - upload_cb = None + @errors.generic.all + @errors.pithos.connection + @errors.pithos.container + @errors.pithos.object_path + def _run(self, local_path): + (progress_bar, upload_cb) = self._safe_progress_bar('Appending') try: f = open(local_path, 'rb') self.client.append_object(self.path, f, upload_cb) - except ClientError as err: - progress_bar.finish() - if err.status == 404: - if 'container' in ('%s' % err).lower(): - raiseCLIError( - err, - 'No container %s in account %s'\ - % (self.container, self.account), - details=errors.pithos.container_howto) - elif 'object' in ('%s' % err).lower(): - raiseCLIError( - err, - 'No object %s in container %s'\ - % (self.path, self.container), - details=errors.pithos.container_howto) - raise_connection_errors(err) - raiseCLIError(err) - except Exception as e: - progress_bar.finish() - raiseCLIError(e) + except Exception: + self._safe_progress_bar_finish(progress_bar) + raise finally: - progress_bar.finish() + self._safe_progress_bar_finish(progress_bar) + + def main(self, local_path, container___path): + super(self.__class__, self)._run( + container___path, + path_is_optional=False) + self._run(local_path) @command(pithos_cmds) class store_truncate(_store_container_command): """Truncate remote file up to a size (default is 0)""" + @errors.generic.all + @errors.pithos.connection + @errors.pithos.container + @errors.pithos.object_path + @errors.pithos.object_size + def _run(self, size=0): + self.client.truncate_object(self.path, size) + def main(self, container___path, size=0): - super(self.__class__, self).main(container___path) - try: - self.client.truncate_object(self.path, size) - except ClientError as err: - if err.status == 404: - if 'container' in ('%s' % err).lower(): - raiseCLIError( - err, - 'No container %s in account %s'\ - % (self.container, self.account), - details=errors.pithos.container_howto) - elif 'object' in ('%s' % err).lower(): - raiseCLIError( - err, - 'No object %s in container %s'\ - % (self.path, self.container), - details=errors.pithos.container_howto) - if err.status == 400 and\ - 'object length is smaller than range length'\ - in ('%s' % err).lower(): - raiseCLIError(err, 'Object %s:%s <= %sb' % ( - self.container, - self.path, - size)) - raise_connection_errors(err) - raiseCLIError(err) - except Exception as e: - raiseCLIError(e) + super(self.__class__, self)._run(container___path) + self._run(size=size) @command(pithos_cmds) @@ -784,58 +752,40 @@ class store_overwrite(_store_container_command): default=False) ) - def main(self, local_path, container____path__, start, end): - (start, end) = check_range(start, end) - super(self.__class__, self).main(container____path__) - try: - f = open(local_path, 'rb') - f.seek(0, 2) - f_size = f.tell() - f.seek(start, 0) - except Exception as e: - raiseCLIError(e) - progress_bar = self.arguments['progress_bar'] - try: - upload_cb = progress_bar.get_generator( - 'Overwriting %s blocks' % end - start) - except Exception: - upload_cb = None + def _open_file(self, local_path, start): + f = open(path.abspath(local_path), 'rb') + f.seek(0, 2) + f_size = f.tell() + f.seek(start, 0) + return (f, f_size) + + @errors.generic.all + @errors.pithos.connection + @errors.pithos.container + @errors.pithos.object_path + @errors.pithos.object_size + def _run(self, local_path, start, end): + (start, end) = (int(start), int(end)) + (f, f_size) = self._open_file(local_path, start) + (progress_bar, upload_cb) = self._safe_progress_bar( + 'Overwrite %s bytes' % (end - start)) try: - self.path = self.path if self.path else path.basename(local_path) self.client.overwrite_object( obj=self.path, start=start, end=end, source_file=f, upload_cb=upload_cb) - except ClientError as err: - progress_bar.finish() - if (err.status == 400 and\ - 'content length does not match range' in ('%s' % err).lower()) or\ - err.status == 416: - raiseCLIError(err, details=[ - 'Content size: %s' % f_size, - 'Range: %s-%s (size: %s)' % (start, end, end - start)]) - elif err.status == 404: - if 'container' in ('%s' % err).lower(): - raiseCLIError( - err, - 'No container %s in account %s'\ - % (self.container, self.account), - details=errors.pithos.container_howto) - elif 'object' in ('%s' % err).lower(): - raiseCLIError( - err, - 'No object %s in container %s'\ - % (self.path, self.container), - details=errors.pithos.container_howto) - raise_connection_errors(err) - raiseCLIError(err) - except Exception as e: - progress_bar.finish() - raiseCLIError(e) + except Exception: + self._safe_progress_bar_finish(progress_bar) + raise finally: - progress_bar.finish() + self._safe_progress_bar_finish(progress_bar) + + def main(self, local_path, container____path__, start, end): + super(self.__class__, self)._run(container____path__) + self.path = self.path if self.path else path.basename(local_path) + self._run(local_path=local_path, start=start, end=end) @command(pithos_cmds) diff --git a/kamaki/clients/pithos.py b/kamaki/clients/pithos.py index 2541aec..1afebe3 100644 --- a/kamaki/clients/pithos.py +++ b/kamaki/clients/pithos.py @@ -908,8 +908,13 @@ class PithosClient(PithosRestAPI): :returns: (dict) """ - r = self.object_head(obj, version=version) - return r.headers + try: + r = self.object_head(obj, version=version) + return r.headers + except ClientError as ce: + if ce.status == 404: + raise ClientError('Object not found', status=404) + raise def get_object_meta(self, obj, version=None): """ @@ -985,8 +990,9 @@ class PithosClient(PithosRestAPI): filesize = fstat(source_file.fileno()).st_size nblocks = 1 + (filesize - 1) // blocksize offset = 0 - if upload_cb is not None: + if upload_cb: upload_gen = upload_cb(nblocks) + upload_gen.next() for i in range(nblocks): block = source_file.read(min(blocksize, filesize - offset)) offset += len(block) @@ -998,7 +1004,7 @@ class PithosClient(PithosRestAPI): data=block) r.release() - if upload_cb is not None: + if upload_cb: upload_gen.next() def truncate_object(self, obj, upto_bytes): @@ -1034,6 +1040,16 @@ class PithosClient(PithosRestAPI): :param upload_db: progress.bar for uploading """ + r = self.get_object_info(obj) + rf_size = int(r['content-length']) + if rf_size < int(start): + raise ClientError( + 'Range start exceeds file size', + status=416) + elif rf_size < int(end): + raise ClientError( + 'Range end exceeds file size', + status=416) self._assert_container() meta = self.get_container_info() blocksize = int(meta['x-container-block-size']) @@ -1041,22 +1057,25 @@ class PithosClient(PithosRestAPI): datasize = int(end) - int(start) + 1 nblocks = 1 + (datasize - 1) // blocksize offset = 0 - if upload_cb is not None: + if upload_cb: upload_gen = upload_cb(nblocks) + upload_gen.next() for i in range(nblocks): block = source_file.read(min(blocksize, filesize - offset, datasize - offset)) - offset += len(block) r = self.object_post(obj, update=True, content_type='application/octet-stream', content_length=len(block), - content_range='bytes %s-%s/*' % (start, end), + content_range='bytes %s-%s/*' % ( + start + offset, + start + offset + len(block) - 1), data=block) + offset += len(block) r.release() - if upload_cb is not None: + if upload_cb: upload_gen.next() def copy_object(self, src_container, src_object, dst_container, diff --git a/setup.py b/setup.py index fc20d0c..e3ce1f0 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ # or implied, of GRNET S.A. from setuptools import setup -#from sys import version_info +from sys import version_info import collections import kamaki @@ -42,8 +42,10 @@ import kamaki optional = ['ansicolors', 'progress>=1.0.2'] -requires = ['objpool', - 'argparse'] +requires = ['objpool'] + +if version_info[:1] == (2, 6): + requires.append('argparse') if not hasattr(collections, "OrderedDict"): # Python 2.6 requires.append("ordereddict") -- 1.7.10.4