From 56f0908a5e4389c9817bdaba4656edac5c8452eb Mon Sep 17 00:00:00 2001 From: Stavros Sachtouris Date: Thu, 23 Aug 2012 19:15:44 +0300 Subject: [PATCH] Improved incremental download, hashmap, versions --- kamaki/clients/pithos.py | 62 +++++++++++++++++++++++++++++++++- kamaki/clients/pithos_cli.py | 70 +++++++++++++++++++++++++++------------ kamaki/clients/pithos_sh_cli.py | 70 ++++++++++++++++++++++----------------- 3 files changed, 149 insertions(+), 53 deletions(-) diff --git a/kamaki/clients/pithos.py b/kamaki/clients/pithos.py index 6d556ca..78ad56d 100644 --- a/kamaki/clients/pithos.py +++ b/kamaki/clients/pithos.py @@ -299,7 +299,8 @@ class PithosClient(StorageClient): return self.head(path, *args, success=success, **kwargs) def object_get(self, object, format='json', hashmap=False, version=None, - data_range=None, if_range=False, if_etag_match=None, if_etag_not_match = None, if_modified_since = None, if_unmodified_since = None, *args, **kwargs): + data_range=None, if_range=False, if_etag_match=None, if_etag_not_match = None, + if_modified_since = None, if_unmodified_since = None, *args, **kwargs): """ Full Pithos+ GET at object level --- request parameters --- @param format (string): json (default) or xml @@ -714,6 +715,65 @@ class PithosClient(StorageClient): self.object_put(object, format='json', hashmap=True, content_type=obj_content_type, json=hashmap, success=201) + def download_object(self, obj, f, download_cb=None): + + self.assert_container() + + #retrieve object hashmap + hashmap = self.get_object_hashmap(obj) + blocksize = int(hashmap['block_size']) + blockhash = hashmap['block_hash'] + total_size = hashmap['bytes'] + map = hashmap['hashes'] + map_dict = {} + for h in map: + map_dict[h] = True + download_bars = len(map) + + #load progress bar + if download_cb is not None: + download_gen = download_cb(total_size/blocksize + 1) + download_gen.next() + + #load local file existing hashmap + if not f.isatty(): + hash_dict = {} + index = 0 + from os import path + if path.exists(f.name): + from binascii import hexlify + from .pithos_sh_lib.hashmap import HashMap + h = HashMap(blocksize, blockhash) + h.load(f) + for x in h: + existing_hash = hexlify(x) + if existing_hash not in map_dict: + raise ClientError(message='Local file is substantialy different', + status=600) + hash_dict[existing_hash] = index + index += 1 + if download_cb: + download_gen.next() + + #download and print + for i, h in enumerate(map): + if not f.isatty() and h in hash_dict: + continue + if download_cb is not None: + download_gen.next() + start = i*blocksize + end = start + blocksize -1 if start+blocksize < total_size else total_size -1 + data_range = 'bytes=%s-%s'%(start, end) + data = self.object_get(obj, data_range=data_range, success=(200, 206)) + if not f.isatty(): + f.seek(start) + f.write(data.text) + + def get_object_hashmap(self, obj, version=None): + r = self.object_get(obj, hashmap=True, version=version) + from json import loads + return loads(r.text) + def set_account_group(self, group, usernames): self.account_post(update=True, groups = {group:usernames}) diff --git a/kamaki/clients/pithos_cli.py b/kamaki/clients/pithos_cli.py index fb0a838..b8915ce 100644 --- a/kamaki/clients/pithos_cli.py +++ b/kamaki/clients/pithos_cli.py @@ -39,13 +39,15 @@ from .pithos import PithosClient, ClientError from .cli_utils import raiseCLIError from kamaki.utils import print_dict, pretty_keys, print_list from colors import bold -from sys import stdout +from sys import stdout, exit +import signal from progress.bar import IncrementalBar class ProgressBar(IncrementalBar): - suffix = '%(percent)d%% - %(eta)ds' + #suffix = '%(percent)d%% - %(eta)ds' + suffix = '%(percent)d%%' class _pithos_init(object): def main(self): @@ -377,7 +379,6 @@ class store_move(_store_container_command): class store_append(_store_container_command): """Append local file to (existing) remote object""" - def main(self, local_path, container___path): super(self.__class__, self).main(container___path, path_is_optional=False) try: @@ -527,31 +528,58 @@ class store_upload(_store_container_command): class store_download(_store_container_command): """Download a file""" - def main(self, container___path, local_path='-'): + def main(self, container___path, local_path=None): super(self.__class__, self).main(container___path, path_is_optional=False) + + #setup output stream + if local_path is None: + out = stdout + else: + try: + out = open(local_path, 'a+') + except IOError as err: + raise CLIError(message='Cannot write to file %s - %s'%(local_path,unicode(err)), + importance=1) + download_cb = self.progress('Downloading') + try: - f, size = self.client.get_object(self.path) + self.client.download_object(self.path, out, download_cb) except ClientError as err: raiseCLIError(err) - try: - out = open(local_path, 'w') if local_path != '-' else stdout - except IOError: - raise CLIError(message='Cannot write to file %s'%local_path, importance=1) + except KeyboardInterrupt: + print('\ndownload canceled by user') + if local_path is not None: + print('re-run command to resume') - blocksize = 4 * 1024 ** 2 - nblocks = 1 + (size - 1) // blocksize +@command() +class store_hashmap(_store_container_command): + """Get the hashmap of an object""" - cb = self.progress('Downloading blocks') if local_path != '-' else None - if cb: - gen = cb(nblocks) - gen.next() + def update_parser(self, parser): + super(self.__class__, self).update_parser(parser) + parser.add_argument('--range', action='store', dest='range', default=None, + help='show range of data') + parser.add_argument('--if-range', action='store', dest='if_range', default=None, + help='show range of data') + parser.add_argument('--if-match', action='store', dest='if_match', default=None, + help='show output if ETags match') + parser.add_argument('--if-none-match', action='store', dest='if_none_match', default=None, + help='show output if ETags don\'t match') + parser.add_argument('--if-modified-since', action='store', dest='if_modified_since', + default=None, help='show output if modified since then') + parser.add_argument('--if-unmodified-since', action='store', dest='if_unmodified_since', + default=None, help='show output if not modified since then') + parser.add_argument('--object-version', action='store', dest='object_version', default=None, + help='get the specific version') - data = f.read(blocksize) - while data: - out.write(data) - data = f.read(blocksize) - if cb: - gen.next() + def main(self, container___path): + super(self.__class__, self).main(container___path, path_is_optional=False) + try: + data = self.client.get_object_hashmap(self.path, + version=getattr(self.args, 'object_version')) + except ClientError as err: + raiseCLIError(err) + print_dict(data) @command() class store_delete(_store_container_command): diff --git a/kamaki/clients/pithos_sh_cli.py b/kamaki/clients/pithos_sh_cli.py index ebb85ad..c3c1584 100644 --- a/kamaki/clients/pithos_sh_cli.py +++ b/kamaki/clients/pithos_sh_cli.py @@ -37,6 +37,7 @@ from kamaki.cli import command, CLIError from kamaki.utils import print_list from .utils import dict2file, list2file from .pithos_cli import _store_container_command, _store_account_command +from colors import bold #from getpass import getuser #from optparse import OptionParser @@ -80,11 +81,28 @@ def _build_args(arglist, attrs): return args @command() -class store_download(_pithos_sh_container_command): +class store_versions(_pithos_sh_container_command): + """Get the version list of an object""" + + def main(self, container___path): + super(store_versions, self).main(container___path) + try: + data = self.client.retrieve_object_versionlist(self.container, self.path) + except Fault as err: + raise CLIError(message=unicode(err), status=err.status) + from time import localtime, strftime + print('%s:%s version ids:'%(self.container,self.path)) + for vitem in data['versions']: + t = localtime(float(vitem[1])) + vid = bold(unicode(vitem[0])) + print('\t%s \t(%s)'%(vid, strftime('%d-%m-%Y %H:%M:%S', t))) + +@command() +class store_downloadz(_pithos_sh_container_command): """Download an object""" def update_parser(self, parser): - super(store_download, self).update_parser(parser) + super(store_downloadz, self).update_parser(parser) parser.add_argument('-l', action='store_true', dest='detail', default=False, help='show detailed output') parser.add_argument('--range', action='store', dest='range', default=None, @@ -101,17 +119,13 @@ class store_download(_pithos_sh_container_command): default=None, help='show output if not modified since then') parser.add_argument('--object-version', action='store', dest='object-version', default=None, help='get the specific version') - parser.add_argument('--versionlist', action='store_true', dest='versionlist', default=False, - help='get the full object version list') - parser.add_argument('--hashmap', action='store_true', dest='hashmap', default=False, - help='get the object hashmap instead') def main(self, container___path, outputFile=None): - super(store_download, self).main(container___path) + super(store_downloadz, self).main(container___path) #prepare attributes and headers attrs = ['if_match', 'if_none_match', 'if_modified_since', - 'if_unmodified_since', 'hashmap'] + 'if_unmodified_since'] args = _build_args(self.args, attrs) args['format'] = 'json' if hasattr(self.args,'detail') else 'text' if getattr(self.args, 'range') is not None: @@ -120,27 +134,18 @@ class store_download(_pithos_sh_container_command): args['if-range'] = 'If-Range:%s' % getattr(self.args, 'if_range') #branch through options - if getattr(self.args,'versionlist'): - try: - args.pop('detail') - except KeyError: - pass - args.pop('format') - data=self.client.retrieve_object_versionlist(self.container, self.path, **args) - elif getattr(self.args,'object-version'): - data = self.client.retrieve_object_version(self.container, self.path, - version=getattr(self.args, 'object-version'), **args) - elif getattr(self.args, 'hashmap'): - try: - args.pop('detail') - except KeyError: - pass - data=self.client.retrieve_object_hashmap(self.container, self.path, **args) - elif outputFile is None: - cat(self.client, self.container, self.path) - else: - download(self.client, self.container, self.path, outputFile) - return + try: + if getattr(self.args,'object-version'): + data = self.client.retrieve_object_version(self.container, self.path, + version=getattr(self.args, 'object-version'), **args) + elif outputFile is None: + cat(self.client, self.container, self.path) + return + else: + download(self.client, self.container, self.path, outputFile) + return + except Fault as err: + raise CLIError(message=unicode(err), status=err.status, importance=err.status/100) f = stdout if outputFile is None else open(outputFile, 'w') if type(data) is dict: @@ -169,5 +174,8 @@ class store_sharers(_pithos_sh_account_command): attrs = ['limit', 'marker'] args = _build_args(self.args, attrs) args['format'] = 'json' if getattr(self.args, 'detail') else 'text' - - print_list(self.client.list_shared_by_others(**args)) + + try: + print_list(self.client.list_shared_by_others(**args)) + except Fault as err: + raise CLIError(message=unicode(err), status=err.status, importance=err.status/100) -- 1.7.10.4