Implement download_to_string in pithos client

The download_to_string method downloads a remote object from pithos into a
string, which is then returned.

Also, implemented unit and functional tests. Changelog is updated

Refs: #3608

diff --git a/Changelog b/Changelog
index bedf1d3..3e9ae5c 100644 (file)
--- a/Changelog
+++ b/Changelog
@@ -32,4 +32,5 @@ Features:
 - Add a -l option to upload, for listing uploaded objects details [#3730]
 - Add option to cache container info for upload_object [#3707]
 - Add enumeration to all listing commands, make it optional [#3739]
+- Add a download_to_string method in pithos client [#3608]
 

diff --git a/kamaki/clients/livetest/pithos.py b/kamaki/clients/livetest/pithos.py
index b2f23ed..d6acc29 100644 (file)
--- a/kamaki/clients/livetest/pithos.py
+++ b/kamaki/clients/livetest/pithos.py
@@ -677,10 +677,18 @@ class Pithos(livetest.Generic):
         self.client.download_object(trg_fname, dnl_f)
 
         print('\tCheck if files match...')
-        for pos in (0, f_size / 2, f_size - 20):
+        for pos in (0, f_size / 2, f_size - 128):
             src_f.seek(pos)
             dnl_f.seek(pos)
-            self.assertEqual(src_f.read(10), dnl_f.read(10))
+            self.assertEqual(src_f.read(64), dnl_f.read(64))
+
+        print('\tDownload KiBs to string and check again...')
+        for pos in (0, f_size / 2, f_size - 256):
+            src_f.seek(pos)
+            tmp_s = self.client.download_to_string(
+                trg_fname,
+                range_str='%s-%s' % (pos, (pos + 128)))
+            self.assertEqual(tmp_s, src_f.read(len(tmp_s)))
 
         """Upload a boring file"""
         trg_fname = 'boringfile_%s' % self.now

diff --git a/kamaki/clients/pithos/__init__.py b/kamaki/clients/pithos/__init__.py
index bf2c5a5..87c7103 100644 (file)
--- a/kamaki/clients/pithos/__init__.py
+++ b/kamaki/clients/pithos/__init__.py
@@ -36,6 +36,7 @@ from threading import enumerate as activethreads
 from os import fstat
 from hashlib import new as newhashlib
 from time import time
+from StringIO import StringIO
 
 from binascii import hexlify
 
@@ -43,7 +44,6 @@ from kamaki.clients import SilentEvent, sendlog
 from kamaki.clients.pithos.rest_api import PithosRestClient
 from kamaki.clients.storage import ClientError
 from kamaki.clients.utils import path4url, filter_in
-from StringIO import StringIO
 
 
 def _pithos_hash(block, blockhash):
@@ -623,6 +623,71 @@ class PithosClient(PithosRestClient):
 
         self._complete_cb()
 
+    def download_to_string(
+            self, obj,
+            download_cb=None,
+            version=None,
+            range_str=None,
+            if_match=None,
+            if_none_match=None,
+            if_modified_since=None,
+            if_unmodified_since=None):
+        """Download an object to a string (multiple connections)
+
+        :param obj: (str) remote object path
+
+        :param download_cb: optional progress.bar object for downloading
+
+        :param version: (str) file version
+
+        :param range_str: (str) from, to are file positions (int) in bytes
+
+        :param if_match: (str)
+
+        :param if_none_match: (str)
+
+        :param if_modified_since: (str) formated date
+
+        :param if_unmodified_since: (str) formated date
+
+        :returns: (str) the whole object contents
+        """
+        restargs = dict(
+            version=version,
+            data_range=None if range_str is None else 'bytes=%s' % range_str,
+            if_match=if_match,
+            if_none_match=if_none_match,
+            if_modified_since=if_modified_since,
+            if_unmodified_since=if_unmodified_since)
+
+        (
+            blocksize,
+            blockhash,
+            total_size,
+            hash_list,
+            remote_hashes) = self._get_remote_blocks_info(obj, **restargs)
+        assert total_size >= 0
+
+        if download_cb:
+            self.progress_bar_gen = download_cb(len(hash_list))
+            self._cb_next()
+
+        ret = ''
+        for blockid, blockhash in enumerate(remote_hashes):
+            start = blocksize * blockid
+            is_last = start + blocksize > total_size
+            end = (total_size - 1) if is_last else (start + blocksize - 1)
+            (start, end) = _range_up(start, end, range_str)
+            if start == end:
+                continue
+            restargs['data_range'] = 'bytes=%s-%s' % (start, end)
+            r = self.object_get(obj, success=(200, 206), **restargs)
+            ret += r.content
+            self._cb_next()
+
+        self._complete_cb()
+        return ret
+
     #Command Progress Bar method
     def _cb_next(self, step=1):
         if hasattr(self, 'progress_bar_gen'):

diff --git a/kamaki/clients/pithos/test.py b/kamaki/clients/pithos/test.py
index 83028f2..e6f9da5 100644 (file)
--- a/kamaki/clients/pithos/test.py
+++ b/kamaki/clients/pithos/test.py
@@ -1103,6 +1103,37 @@ class PithosClient(TestCase):
 
     @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
     @patch('%s.object_get' % pithos_pkg, return_value=FR())
+    def test_download_to_string(self, GET, GOH):
+        FR.content = 'some sample content'
+        num_of_blocks = len(object_hashmap['hashes'])
+        r = self.client.download_to_string(obj)
+        expected_content = FR.content * num_of_blocks
+        self.assertEqual(expected_content, r)
+        self.assertEqual(len(GET.mock_calls), num_of_blocks)
+        self.assertEqual(GET.mock_calls[-1][1], (obj,))
+
+        kwargs = dict(
+            version='version',
+            range_str='10-20',
+            if_match='if and only if',
+            if_none_match='if and only not',
+            if_modified_since='what if not?',
+            if_unmodified_since='this happens if not!')
+        expargs = dict(kwargs)
+        expargs.pop('range_str')
+        for k in expargs:
+            expargs[k] = None
+        GOH.assert_called_once_with(obj, **expargs)
+
+        r = self.client.download_to_string(obj, **kwargs)
+        expargs['data_range'] = 'bytes=%s' % kwargs['range_str']
+        for k, v in expargs.items():
+            self.assertEqual(
+                GET.mock_calls[-1][2][k],
+                v or kwargs.get(k))
+
+    @patch('%s.get_object_hashmap' % pithos_pkg, return_value=object_hashmap)
+    @patch('%s.object_get' % pithos_pkg, return_value=FR())
     def test_download_object(self, GET, GOH):
         num_of_blocks = 8
         tmpFile = self._create_temp_file(num_of_blocks)