Revision 776b275c

b/kamaki/cli/commands/pithos.py
131 131
        return getattr(self, '_value', self.default)
132 132

  
133 133
    @value.setter
134
    def value(self, newvalue):
135
        if newvalue is None:
134
    def value(self, newvalues):
135
        if not newvalues:
136 136
            self._value = self.default
137 137
            return
138
        start, sep, end = newvalue.partition('-')
139
        if sep:
140
            if start:
141
                self._value = '%s-%s' % (int(start), int(end))
138
        self._value = ''
139
        for newvalue in newvalues.split(','):
140
            self._value = ('%s,' % self._value) if self._value else ''
141
            start, sep, end = newvalue.partition('-')
142
            if sep:
143
                if start:
144
                    start, end = (int(start), int(end))
145
                    assert start <= end, 'Invalid range value %s' % newvalue
146
                    self._value += '%s-%s' % (int(start), int(end))
147
                else:
148
                    self._value += '-%s' % int(end)
142 149
            else:
143
                self._value = '-%s' % int(end)
144
        else:
145
            self._value = '%s' % int(start)
150
                self._value += '%s' % int(start)
146 151

  
147 152

  
148 153
# Command specs
b/kamaki/clients/__init__.py
322 322
    MAX_THREADS = 7
323 323
    DATE_FORMATS = ['%a %b %d %H:%M:%S %Y', ]
324 324
    LOG_TOKEN = False
325
    LOG_DATA = True
325
    LOG_DATA = False
326 326
    CONNECTION_RETRY_LIMIT = 0
327 327

  
328 328
    def __init__(self, base_url, token):
b/kamaki/clients/livetest/pithos.py
556 556
        r = self.client.object_head(obj)
557 557
        self.assertEqual(r.status_code, 200)
558 558
        etag = r.headers['etag']
559
        real_version = r.headers['x-object-version']
559 560

  
560
        r = self.client.object_head(obj, version=40)
561
        self.assertEqual(r.headers['x-object-version'], '40')
561
        self.assertRaises(
562
            ClientError, self.client.object_head, obj, version=-10)
563
        r = self.client.object_head(obj, version=real_version)
564
        self.assertEqual(r.headers['x-object-version'], real_version)
562 565

  
563 566
        r = self.client.object_head(obj, if_etag_match=etag)
564 567
        self.assertEqual(r.status_code, 200)
......
568 571
        self.assertNotEqual(r.status_code, 200)
569 572

  
570 573
        r = self.client.object_head(
571
            obj, version=40, if_etag_match=etag, success=412)
572
        self.assertEqual(r.status_code, 412)
574
            obj, version=real_version, if_etag_match=etag, success=200)
575
        self.assertEqual(r.status_code, 200)
573 576

  
574 577
        """Check and if(un)modified_since"""
575 578
        for format in self.client.DATE_FORMATS:
......
1254 1257

  
1255 1258
    def create_large_file(self, size):
1256 1259
        """Create a large file at fs"""
1260
        print
1257 1261
        self.files.append(NamedTemporaryFile())
1258 1262
        f = self.files[-1]
1259 1263
        Ki = size / 8
b/kamaki/clients/pithos/__init__.py
52 52
    return h.hexdigest()
53 53

  
54 54

  
55
def _range_up(start, end, a_range):
56
    if a_range:
57
        rstart, sep, rend = a_range.partition('-')
58
        if rstart:
55
def _range_up(start, end, max_value, a_range):
56
    """
57
    :param start: (int) the window bottom
58

  
59
    :param end: (int) the window top
60

  
61
    :param max_value: (int) maximum accepted value
62

  
63
    :param a_range: (str) a range string in the form X[,X'[,X''[...]]]
64
        where X: x|x-y|-x where x < y and x, y natural numbers
65

  
66
    :returns: (str) a range string cut-off for the start-end range
67
        an empty response means this window is out of range
68
    """
69
    assert start >= 0, '_range_up was called with start < 0'
70
    assert end >= start, '_range_up was called with end < start'
71
    assert end <= max_value, '_range_up was called with max_value < end'
72
    if not a_range:
73
        return '%s-%s' % (start, end)
74
    selected = []
75
    for some_range in a_range.split(','):
76
        v0, sep, v1 = some_range.partition('-')
77
        if v0:
78
            v0 = int(v0)
59 79
            if sep:
60
                rstart, rend = int(rstart), int(rend)
80
                v1 = int(v1)
81
                if v1 < start or v0 > end or v1 < v0:
82
                    continue
83
                v0 = v0 if v0 > start else start
84
                v1 = v1 if v1 < end else end
85
                selected.append('%s-%s' % (v0, v1))
86
            elif v0 < start:
87
                continue
61 88
            else:
62
                rstart, rend = 0, int(rstart)
63
        elif sep:
64
            return (0, - int(rend))
89
                v1 = v0 if v0 <= end else end
90
                selected.append('%s-%s' % (start, v1))
65 91
        else:
66
            return (0, 0)
67
        if rstart > end or rend < start:
68
            return (0, 0)
69
        if rstart > start:
70
            start = rstart
71
        if rend < end:
72
            end = rend
73
    return (start, end)
92
            v1 = int(v1)
93
            if max_value - v1 > end:
94
                continue
95
            v0 = (max_value - v1) if max_value - v1 > start else start
96
            selected.append('%s-%s' % (v0, end))
97
    return ','.join(selected)
74 98

  
75 99

  
76 100
class PithosClient(PithosRestClient):
......
641 665
                start = blocksize * blockid
642 666
                is_last = start + blocksize > total_size
643 667
                end = (total_size - 1) if is_last else (start + blocksize - 1)
644
                (start, end) = _range_up(start, end, crange)
645
                args['data_range'] = 'bytes=%s-%s' % (
646
                    (start, end) if end >= 0 else ('', - end))
668
                data_range = _range_up(start, end, total_size, crange)
669
                if not data_range:
670
                    self._cb_next()
671
                    continue
672
                args['data_range'] = 'bytes=%s' % data_range
647 673
                r = self.object_get(obj, success=(200, 206), **args)
648 674
                self._cb_next()
649 675
                dst.write(r.content)
......
689 715
        flying = dict()
690 716
        blockid_dict = dict()
691 717
        offset = 0
692
        if filerange is not None:
693
            rstart = int(filerange.split('-')[0])
694
            offset = rstart if blocksize > rstart else rstart % blocksize
695 718

  
696 719
        self._init_thread_limit()
697 720
        for block_hash, blockids in remote_hashes.items():
......
708 731
                    **restargs)
709 732
                end = total_size - 1 if (
710 733
                    key + blocksize > total_size) else key + blocksize - 1
711
                start, end = _range_up(key, end, filerange)
712
                if start == end:
734
                data_range = _range_up(key, end, total_size, filerange)
735
                if not data_range:
713 736
                    self._cb_next()
714 737
                    continue
715
                restargs['async_headers'] = {
716
                    'Range': 'bytes=%s-%s' % (
717
                        (start, end) if end >= 0 else ('', - end))}
738
                restargs['async_headers'] = {'Range': 'bytes=%s' % data_range}
718 739
                flying[key] = self._get_block_async(obj, **restargs)
719 740
                blockid_dict[key] = unsaved
720 741

  
......
857 878
                start = blocksize * blockid
858 879
                is_last = start + blocksize > total_size
859 880
                end = (total_size - 1) if is_last else (start + blocksize - 1)
860
                (start, end) = _range_up(start, end, range_str)
861
                if start < end or end < 0:
881
                data_range_str = _range_up(start, end, end, range_str)
882
                if data_range_str:
862 883
                    self._watch_thread_limit(flying.values())
884
                    restargs['data_range'] = 'bytes=%s' % data_range_str
863 885
                    flying[blockid] = self._get_block_async(obj, **restargs)
864 886
                for runid, thread in flying.items():
865 887
                    if (blockid + 1) == num_of_blocks:
......
899 921
            if_match=None,
900 922
            if_none_match=None,
901 923
            if_modified_since=None,
902
            if_unmodified_since=None,
903
            data_range=None):
924
            if_unmodified_since=None):
904 925
        """
905 926
        :param obj: (str) remote object path
906 927

  
......
912 933

  
913 934
        :param if_unmodified_since: (str) formated date
914 935

  
915
        :param data_range: (str) from-to where from and to are integers
916
            denoting file positions in bytes
917

  
918 936
        :returns: (list)
919 937
        """
920 938
        try:
......
925 943
                if_etag_match=if_match,
926 944
                if_etag_not_match=if_none_match,
927 945
                if_modified_since=if_modified_since,
928
                if_unmodified_since=if_unmodified_since,
929
                data_range=data_range)
946
                if_unmodified_since=if_unmodified_since)
930 947
        except ClientError as err:
931 948
            if err.status == 304 or err.status == 412:
932 949
                return {}
b/kamaki/clients/pithos/test.py
242 242
            pm = pm[:-2]
243 243
            self.client.account_post(*(pm + args), **kwargs)
244 244
            upd = pm[0]
245
            self.assertEqual(SP.mock_calls[-1], call('update', iff=upd))
245
            self.assertEqual(SP.mock_calls[-1], call('update', '', iff=upd))
246 246
            expected = []
247 247
            if pm[1]:
248 248
                expected += [
......
369 369
            self.client.container_post(*(pm + args), **kwargs)
370 370
            upd, frmt = pm[:2]
371 371
            self.assertEqual(SP.mock_calls[-2:], [
372
                call('update', iff=upd),
372
                call('update', '', iff=upd),
373 373
                call('format', frmt, iff=frmt)])
374 374
            qta, vrs, metas, ctype, clen, trenc = pm[2:]
375 375
            prfx = 'X-Container-Meta-'
......
637 637
                    if pval:
638 638
                        perm_str += ';' if perm_str else ''
639 639
                        perm_str += '%s=%s' % (ptype, ','.join(pval))
640
                exp += [call('X-Object-Sharing', perm_str)]
640
                exp += [call('X-Object-Sharing', perm_str, iff=perms)]
641
            else:
642
                exp += [call('X-Object-Sharing', '', iff={})]
641 643
            exp += [call('X-Object-Public', public, public is not None)]
642 644
            for k, v in metas.items():
643 645
                exp += [call('X-Object-Meta-%s' % k, v)]
......
674 676
            format, update = pm[:2]
675 677
            self.assertEqual(SP.mock_calls[-2:], [
676 678
                call('format', format, iff=format),
677
                call('update', iff=update)])
679
                call('update', '', iff=update)])
678 680
            (
679 681
                im, inm, clen, ctype, crng, trenc, cenc,
680 682
                condis, srcobj, srcacc, srcvrs, obytes, mnfs) = terms
......
699 701
                    if pval:
700 702
                        perm_str += ';' if perm_str else ''
701 703
                        perm_str += '%s=%s' % (ptype, ','.join(pval))
702
                exp += [call('X-Object-Sharing', perm_str)]
704
                exp += [call('X-Object-Sharing', perm_str, iff=perms)]
705
            else:
706
                exp += [call('X-Object-Sharing', '', iff={})]
703 707
            exp += [call('X-Object-Public', public, public is not None)]
704 708
            for k, v in metas.items():
705 709
                exp += [call('X-Object-Meta-%s' % k, v)]
......
737 741
                **kwargs))
738 742

  
739 743

  
744
class PithosMethods(TestCase):
745

  
746
    def test__range_up(self):
747
        from kamaki.clients.pithos import _range_up
748
        for args, expected in (
749
                ((0, 100, 1000, '10'), '0-10'),
750
                ((0, 100, 1000, '-10'), ''),
751
                ((900, 1000, 1000, '-10'), '990-1000'),
752
                ((150, 250, 1000, '10'), ''),
753
                ((10, 200, 1000, '130-170'), '130-170'),
754
                ((150, 200, 1000, '130-170'), '150-170'),
755
                ((100, 150, 1000, '130-170'), '130-150'),
756
                ((200, 250, 1000, '130-170'), ''),
757
                ((100, 250, 1000, '30-170,200-270'), '100-170,200-250'),
758
                ((40, 950, 1000, '-170,200-270,50',), '830-950,200-270,40-50'),
759
                ((740, 900, 1000, '-170,200-270,50',), '830-900'),
760
                ((42, 333, 800, '100,50-200,-600',), '42-100,50-200,200-333')):
761
            self.assertEqual(_range_up(*args), expected)
762

  
763

  
740 764
class PithosClient(TestCase):
741 765

  
742 766
    files = []
......
1379 1403
                self.assertEqual(r, {})
1380 1404
        exp_args = dict(
1381 1405
            hashmap=True,
1382
            data_range=None,
1383 1406
            version=None,
1384 1407
            if_etag_match=None,
1385 1408
            if_etag_not_match=None,
......
1390 1413
            if_match='if match',
1391 1414
            if_none_match='if non match',
1392 1415
            if_modified_since='some date here',
1393
            if_unmodified_since='some date here',
1394
            data_range='10-20')
1416
            if_unmodified_since='some date here')
1395 1417
        with patch.object(
1396
                pithos.PithosClient, 'object_get',
1397
                return_value=FR()) as get:
1418
                pithos.PithosClient, 'object_get', return_value=FR()) as get:
1398 1419
            r = self.client.get_object_hashmap(obj)
1399 1420
            self.assertEqual(r, object_hashmap)
1400 1421
            self.assertEqual(get.mock_calls[-1], call(obj, **exp_args))
......
1811 1832
    if not argv[1:] or argv[1] == 'PithosRestClient':
1812 1833
        not_found = False
1813 1834
        runTestCase(PithosRestClient, 'PithosRest Client', argv[2:])
1835
    if not argv[1:] or argv[1] == 'PithosMethods':
1836
        not_found = False
1837
        runTestCase(PithosRestClient, 'Pithos Methods', argv[2:])
1814 1838
    if not_found:
1815 1839
        print('TestCase %s not found' % argv[1])
b/kamaki/clients/test.py
45 45
from kamaki.clients.cyclades.test import CycladesRestClient
46 46
from kamaki.clients.image.test import ImageClient
47 47
from kamaki.clients.storage.test import StorageClient
48
from kamaki.clients.pithos.test import PithosClient, PithosRestClient
48
from kamaki.clients.pithos.test import (
49
    PithosClient, PithosRestClient, PithosMethods)
49 50

  
50 51

  
51 52
class ClientError(TestCase):
......
300 301
        self.assertEqual(self.client.base_url, self.base_url)
301 302
        self.assertEqual(self.client.token, self.token)
302 303
        self.assert_dicts_are_equal(self.client.headers, {})
303
        DATE_FORMATS = [
304
            '%a %b %d %H:%M:%S %Y',
305
            '%A, %d-%b-%y %H:%M:%S GMT',
306
            '%a, %d %b %Y %H:%M:%S GMT']
304
        DATE_FORMATS = ['%a %b %d %H:%M:%S %Y']
307 305
        self.assertEqual(self.client.DATE_FORMATS, DATE_FORMATS)
308 306

  
309 307
    def test__init_thread_limit(self):

Also available in: Unified diff