From: Stavros Sachtouris Date: Thu, 16 May 2013 13:31:40 +0000 (+0300) Subject: Implement download_to_string in pithos client X-Git-Tag: 0.9rc1~13^2~4 X-Git-Url: https://code.grnet.gr/git/kamaki/commitdiff_plain/49cc29b2cb540690b19ec2c7ff901ed7871b8db1 Implement download_to_string in pithos client The download_to_string method downloads a remote object from pithos into a string, which is then returned. Also, implemented unit and functional tests. Changelog is updated Refs: #3608 --- diff --git a/Changelog b/Changelog index bedf1d3..3e9ae5c 100644 --- a/Changelog +++ b/Changelog @@ -32,4 +32,5 @@ Features: - Add a -l option to upload, for listing uploaded objects details [#3730] - Add option to cache container info for upload_object [#3707] - Add enumeration to all listing commands, make it optional [#3739] +- Add a download_to_string method in pithos client [#3608] diff --git a/kamaki/clients/livetest/pithos.py b/kamaki/clients/livetest/pithos.py index b2f23ed..d6acc29 100644 --- a/kamaki/clients/livetest/pithos.py +++ b/kamaki/clients/livetest/pithos.py @@ -677,10 +677,18 @@ class Pithos(livetest.Generic): self.client.download_object(trg_fname, dnl_f) print('\tCheck if files match...') - for pos in (0, f_size / 2, f_size - 20): + for pos in (0, f_size / 2, f_size - 128): src_f.seek(pos) dnl_f.seek(pos) - self.assertEqual(src_f.read(10), dnl_f.read(10)) + self.assertEqual(src_f.read(64), dnl_f.read(64)) + + print('\tDownload KiBs to string and check again...') + for pos in (0, f_size / 2, f_size - 256): + src_f.seek(pos) + tmp_s = self.client.download_to_string( + trg_fname, + range_str='%s-%s' % (pos, (pos + 128))) + self.assertEqual(tmp_s, src_f.read(len(tmp_s))) """Upload a boring file""" trg_fname = 'boringfile_%s' % self.now diff --git a/kamaki/clients/pithos/__init__.py b/kamaki/clients/pithos/__init__.py index bf2c5a5..87c7103 100644 --- a/kamaki/clients/pithos/__init__.py +++ b/kamaki/clients/pithos/__init__.py @@ -36,6 +36,7 @@ from threading import enumerate as activethreads from os import fstat from hashlib import new as newhashlib from time import time +from StringIO import StringIO from binascii import hexlify @@ -43,7 +44,6 @@ from kamaki.clients import SilentEvent, sendlog from kamaki.clients.pithos.rest_api import PithosRestClient from kamaki.clients.storage import ClientError from kamaki.clients.utils import path4url, filter_in -from StringIO import StringIO def _pithos_hash(block, blockhash): @@ -623,6 +623,71 @@ class PithosClient(PithosRestClient): self._complete_cb() + def download_to_string( + self, obj, + download_cb=None, + version=None, + range_str=None, + if_match=None, + if_none_match=None, + if_modified_since=None, + if_unmodified_since=None): + """Download an object to a string (multiple connections) + + :param obj: (str) remote object path + + :param download_cb: optional progress.bar object for downloading + + :param version: (str) file version + + :param range_str: (str) from, to are file positions (int) in bytes + + :param if_match: (str) + + :param if_none_match: (str) + + :param if_modified_since: (str) formated date + + :param if_unmodified_since: (str) formated date + + :returns: (str) the whole object contents + """ + restargs = dict( + version=version, + data_range=None if range_str is None else 'bytes=%s' % range_str, + if_match=if_match, + if_none_match=if_none_match, + if_modified_since=if_modified_since, + if_unmodified_since=if_unmodified_since) + + ( + blocksize, + blockhash, + total_size, + hash_list, + remote_hashes) = self._get_remote_blocks_info(obj, **restargs) + assert total_size >= 0 + + if download_cb: + self.progress_bar_gen = download_cb(len(hash_list)) + self._cb_next() + + ret = '' + for blockid, blockhash in enumerate(remote_hashes): + start = blocksize * blockid + is_last = start + blocksize > total_size + end = (total_size - 1) if is_last else (start + blocksize - 1) + (start, end) = _range_up(start, end, range_str) + if start == end: + continue + restargs['data_range'] = 'bytes=%s-%s' % (start, end) + r = self.object_get(obj, success=(200, 206), **restargs) + ret += r.content + self._cb_next() + + self._complete_cb() + return ret + #Command Progress Bar method def _cb_next(self, step=1): if hasattr(self, 'progress_bar_gen'): diff --git a/kamaki/clients/pithos/test.py b/kamaki/clients/pithos/test.py index 83028f2..e6f9da5 100644 --- a/kamaki/clients/pithos/test.py +++ b/kamaki/clients/pithos/test.py @@ -1103,6 +1103,37 @@ class PithosClient(TestCase): @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap) @patch('%s.object_get' % pithos_pkg, return_value=FR()) + def test_download_to_string(self, GET, GOH): + FR.content = 'some sample content' + num_of_blocks = len(object_hashmap['hashes']) + r = self.client.download_to_string(obj) + expected_content = FR.content * num_of_blocks + self.assertEqual(expected_content, r) + self.assertEqual(len(GET.mock_calls), num_of_blocks) + self.assertEqual(GET.mock_calls[-1][1], (obj,)) + + kwargs = dict( + version='version', + range_str='10-20', + if_match='if and only if', + if_none_match='if and only not', + if_modified_since='what if not?', + if_unmodified_since='this happens if not!') + expargs = dict(kwargs) + expargs.pop('range_str') + for k in expargs: + expargs[k] = None + GOH.assert_called_once_with(obj, **expargs) + + r = self.client.download_to_string(obj, **kwargs) + expargs['data_range'] = 'bytes=%s' % kwargs['range_str'] + for k, v in expargs.items(): + self.assertEqual( + GET.mock_calls[-1][2][k], + v or kwargs.get(k)) + + @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap) + @patch('%s.object_get' % pithos_pkg, return_value=FR()) def test_download_object(self, GET, GOH): num_of_blocks = 8 tmpFile = self._create_temp_file(num_of_blocks)