Revision 371d907a

b/pithos/api/functions.py
326 326
    try:
327 327
        meta = request.backend.get_container_meta(request.user_uniq, v_account,
328 328
                                                    v_container, 'pithos', until)
329
        meta['object_meta'] = request.backend.list_object_meta(request.user_uniq,
329
        meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
330 330
                                                v_account, v_container, 'pithos', until)
331 331
        policy = request.backend.get_container_policy(request.user_uniq, v_account,
332 332
                                                        v_container)
......
463 463
    try:
464 464
        meta = request.backend.get_container_meta(request.user_uniq, v_account,
465 465
                                                    v_container, 'pithos', until)
466
        meta['object_meta'] = request.backend.list_object_meta(request.user_uniq,
466
        meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
467 467
                                                v_account, v_container, 'pithos', until)
468 468
        policy = request.backend.get_container_policy(request.user_uniq, v_account,
469 469
                                                        v_container)
......
515 515
    if 'shared' in request.GET:
516 516
        shared = True
517 517
    
518
    try:
519
        objects = request.backend.list_objects(request.user_uniq, v_account,
520
                                    v_container, prefix, delimiter, marker,
521
                                    limit, virtual, 'pithos', keys, shared, until)
522
    except NotAllowedError:
523
        raise Forbidden('Not allowed')
524
    except NameError:
525
        raise ItemNotFound('Container does not exist')
526
    
527 518
    if request.serialization == 'text':
519
        try:
520
            objects = request.backend.list_objects(request.user_uniq, v_account,
521
                                        v_container, prefix, delimiter, marker,
522
                                        limit, virtual, 'pithos', keys, shared, until)
523
        except NotAllowedError:
524
            raise Forbidden('Not allowed')
525
        except NameError:
526
            raise ItemNotFound('Container does not exist')
527
        
528 528
        if len(objects) == 0:
529 529
            # The cloudfiles python bindings expect 200 if json/xml.
530 530
            response.status_code = 204
......
533 533
        response.content = '\n'.join([x[0] for x in objects]) + '\n'
534 534
        return response
535 535
    
536
    try:
537
        objects = request.backend.list_object_meta(request.user_uniq, v_account,
538
                                    v_container, prefix, delimiter, marker,
539
                                    limit, virtual, 'pithos', keys, shared, until)
540
    except NotAllowedError:
541
        raise Forbidden('Not allowed')
542
    except NameError:
543
        raise ItemNotFound('Container does not exist')
544
    
545
#     object_meta = []
546
#     for x in objects:
547
#         if x[1] is None:
548
#             # Virtual objects/directories.
549
#             object_meta.append({'subdir': x[0]})
550
#         else:
551
#             try:
552
#                 meta = request.backend.get_object_meta(request.user_uniq, v_account,
553
#                                                         v_container, x[0], 'pithos', x[1])
554
#                 if until is None:
555
#                     permissions = request.backend.get_object_permissions(
556
#                                     request.user_uniq, v_account, v_container, x[0])
557
#                     public = request.backend.get_object_public(request.user_uniq,
558
#                                                 v_account, v_container, x[0])
559
#                 else:
560
#                     permissions = None
561
#                     public = None
562
#             except NotAllowedError:
563
#                 raise Forbidden('Not allowed')
564
#             except NameError:
565
#                 pass
566
#             else:
567
#                 rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
568
#                 rename_meta_key(meta, 'checksum', 'hash')
569
#                 rename_meta_key(meta, 'type', 'content_type')
570
#                 rename_meta_key(meta, 'uuid', 'x_object_uuid')
571
#                 rename_meta_key(meta, 'modified', 'last_modified')
572
#                 rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
573
#                 rename_meta_key(meta, 'version', 'x_object_version')
574
#                 rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
575
#                 m = dict([(k[14:], v) for k, v in meta.iteritems() if k.startswith('X-Object-Meta-')])
576
#                 for k in m:
577
#                     del(meta['X-Object-Meta-' + k])
578
#                 if m:
579
#                     meta['X-Object-Meta'] = printable_header_dict(m)
580
#                 update_sharing_meta(request, permissions, v_account, v_container, x[0], meta)
581
#                 update_public_meta(public, meta)
582
#                 object_meta.append(printable_header_dict(meta))
583
#     if request.serialization == 'xml':
584
#         data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
585
#     elif request.serialization  == 'json':
586
#         data = json.dumps(object_meta, default=json_encode_decimal)
587
#     response.status_code = 200
588
#     response.content = data
589
#     return response
590
    
536 591
    object_meta = []
537
    for x in objects:
538
        if x[1] is None:
592
    for meta in objects:
593
        if len(meta) == 1:
539 594
            # Virtual objects/directories.
540
            object_meta.append({'subdir': x[0]})
595
            object_meta.append(meta)
541 596
        else:
542
            try:
543
                meta = request.backend.get_object_meta(request.user_uniq, v_account,
544
                                                        v_container, x[0], 'pithos', x[1])
545
                if until is None:
546
                    permissions = request.backend.get_object_permissions(
547
                                    request.user_uniq, v_account, v_container, x[0])
548
                    public = request.backend.get_object_public(request.user_uniq,
549
                                                v_account, v_container, x[0])
550
                else:
551
                    permissions = None
552
                    public = None
553
            except NotAllowedError:
554
                raise Forbidden('Not allowed')
555
            except NameError:
556
                pass
597
            rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
598
            rename_meta_key(meta, 'checksum', 'hash')
599
            rename_meta_key(meta, 'type', 'content_type')
600
            rename_meta_key(meta, 'uuid', 'x_object_uuid')
601
            if until is not None and 'modified' in meta:
602
                del(meta['modified'])
557 603
            else:
558
                rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
559
                rename_meta_key(meta, 'checksum', 'hash')
560
                rename_meta_key(meta, 'type', 'content_type')
561
                rename_meta_key(meta, 'uuid', 'x_object_uuid')
562 604
                rename_meta_key(meta, 'modified', 'last_modified')
563
                rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
564
                rename_meta_key(meta, 'version', 'x_object_version')
565
                rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
566
                m = dict([(k[14:], v) for k, v in meta.iteritems() if k.startswith('X-Object-Meta-')])
567
                for k in m:
568
                    del(meta['X-Object-Meta-' + k])
569
                if m:
570
                    meta['X-Object-Meta'] = printable_header_dict(m)
571
                update_sharing_meta(request, permissions, v_account, v_container, x[0], meta)
572
                update_public_meta(public, meta)
573
                object_meta.append(printable_header_dict(meta))
605
            rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
606
            rename_meta_key(meta, 'version', 'x_object_version')
607
            rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
608
            object_meta.append(printable_header_dict(meta))
574 609
    if request.serialization == 'xml':
575 610
        data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
576 611
    elif request.serialization  == 'json':
b/pithos/api/util.py
104 104
    Format 'last_modified' timestamp.
105 105
    """
106 106
    
107
    if 'last_modified' in d:
107
    if 'last_modified' in d and d['last_modified']:
108 108
        d['last_modified'] = isoformat(datetime.fromtimestamp(d['last_modified']))
109 109
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
110 110

  
b/pithos/backends/base.py
185 185
        """
186 186
        return []
187 187
    
188
    def list_container_meta(self, user, account, container, domain, until=None):
189
        """Return a list with all the container's object meta keys for the domain.
190
        
191
        Raises:
192
            NotAllowedError: Operation not permitted
193
            
194
            NameError: Container does not exist
195
        """
196
        return []
197
    
188 198
    def get_container_meta(self, user, account, container, domain, until=None):
189 199
        """Return a dictionary with the container metadata for the domain.
190 200
        
......
310 320
        """
311 321
        return []
312 322
    
313
    def list_object_meta(self, user, account, container, domain, until=None):
314
        """Return a list with all the container's object meta keys for the domain.
323
    def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None):
324
        """Return a list of object metadata dicts existing under a container.
315 325
        
316
        Raises:
317
            NotAllowedError: Operation not permitted
318
            
319
            NameError: Container does not exist
326
        Same parameters with list_objects. Returned dicts have no user-defined
327
        metadata and, if until is not None, a None 'modified' timestamp.
320 328
        """
321 329
        return []
322 330
    
b/pithos/backends/lib/sqlalchemy/node.py
784 784
    
785 785
    def latest_version_list(self, parent, prefix='', delimiter=None,
786 786
                            start='', limit=10000, before=inf,
787
                            except_cluster=0, pathq=[], domain=None, filterq=[], sizeq=None):
787
                            except_cluster=0, pathq=[], domain=None,
788
                            filterq=[], sizeq=None, all_props=False):
788 789
        """Return a (list of (path, serial) tuples, list of common prefixes)
789 790
           for the current versions of the paths with the given parent,
790 791
           matching the following criteria.
......
831 832
           will always match.
832 833
           
833 834
           Limit applies to the first list of tuples returned.
835
           
836
           If all_props is True, return all properties after path, not just serial.
834 837
        """
835 838
        
836 839
        if not start or start < prefix:
......
839 842
        
840 843
        v = self.versions.alias('v')
841 844
        n = self.nodes.alias('n')
842
        s = select([n.c.path, v.c.serial]).distinct()
845
        if not all_props:
846
            s = select([n.c.path, v.c.serial]).distinct()
847
        else:
848
            s = select([n.c.path,
849
                        v.c.serial, v.c.node, v.c.hash,
850
                        v.c.size, v.c.type, v.c.source,
851
                        v.c.mtime, v.c.muser, v.c.uuid,
852
                        v.c.checksum, v.c.cluster]).distinct()
843 853
        filtered = select([func.max(self.versions.c.serial)])
844 854
        if before != inf:
845 855
            filtered = filtered.where(self.versions.c.mtime < before)
......
910 920
            props = rp.fetchone()
911 921
            if props is None:
912 922
                break
913
            path, serial = props
923
            path = props[0]
924
            serial = props[1]
914 925
            idx = path.find(delimiter, pfz)
915 926
            
916 927
            if idx < 0:
b/pithos/backends/lib/sqlite/node.py
714 714
    
715 715
    def latest_version_list(self, parent, prefix='', delimiter=None,
716 716
                            start='', limit=10000, before=inf,
717
                            except_cluster=0, pathq=[], domain=None, filterq=[], sizeq=None):
717
                            except_cluster=0, pathq=[], domain=None,
718
                            filterq=[], sizeq=None, all_props=False):
718 719
        """Return a (list of (path, serial) tuples, list of common prefixes)
719 720
           for the current versions of the paths with the given parent,
720 721
           matching the following criteria.
......
761 762
           will always match.
762 763
           
763 764
           Limit applies to the first list of tuples returned.
765
           
766
           If all_props is True, return all properties after path, not just serial.
764 767
        """
765 768
        
766 769
        execute = self.execute
......
769 772
            start = strprevling(prefix)
770 773
        nextling = strnextling(prefix)
771 774
        
772
        q = ("select distinct n.path, v.serial "
775
        q = ("select distinct n.path, %s "
773 776
             "from versions v, nodes n "
774 777
             "where v.serial = (select max(serial) "
775 778
                               "from versions "
......
780 783
                            "where parent = ?) "
781 784
             "and n.node = v.node "
782 785
             "and n.path > ? and n.path < ?")
786
        if not all_props:
787
            q = q % "v.serial"
788
        else:
789
            q = q % "v.serial, v.node, v.hash, v.size, v.type, v.source, v.mtime, v.muser, v.uuid, v.checksum, v.cluster"
783 790
        args = [before, except_cluster, parent, start, nextling]
784 791
        
785 792
        subq, subargs = self._construct_paths(pathq)
......
819 826
            props = fetchone()
820 827
            if props is None:
821 828
                break
822
            path, serial = props
829
            path = props[0]
830
            serial = props[1]
823 831
            idx = path.find(delimiter, pfz)
824 832
            
825 833
            if idx < 0:
b/pithos/backends/modular.py
296 296
            start, limit = self._list_limits(allowed, marker, limit)
297 297
            return allowed[start:start + limit]
298 298
        node = self.node.node_lookup(account)
299
        return [x[0] for x in self._list_objects(node, account, '', '/', marker, limit, False, None, [], until)]
299
        return [x[0] for x in self._list_object_properties(node, account, '', '/', marker, limit, False, None, [], until)]
300
    
301
    @backend_method
302
    def list_container_meta(self, user, account, container, domain, until=None):
303
        """Return a list with all the container's object meta keys for the domain."""
304
        
305
        logger.debug("list_container_meta: %s %s %s %s", account, container, domain, until)
306
        allowed = []
307
        if user != account:
308
            if until:
309
                raise NotAllowedError
310
            allowed = self.permissions.access_list_paths(user, '/'.join((account, container)))
311
            if not allowed:
312
                raise NotAllowedError
313
        path, node = self._lookup_container(account, container)
314
        before = until if until is not None else inf
315
        allowed = self._get_formatted_paths(allowed)
316
        return self.node.latest_attribute_keys(node, domain, before, CLUSTER_DELETED, allowed)
300 317
    
301 318
    @backend_method
302 319
    def get_container_meta(self, user, account, container, domain, until=None):
......
405 422
        self.node.node_remove(node)
406 423
        self._report_size_change(user, account, -size, {'action': 'container delete'})
407 424
    
408
    @backend_method
409
    def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None):
410
        """Return a list of objects existing under a container."""
411
        
412
        logger.debug("list_objects: %s %s %s %s %s %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until)
425
    def _list_objects(self, user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, all_props):
413 426
        allowed = []
414 427
        if user != account:
415 428
            if until:
......
424 437
                    return []
425 438
        path, node = self._lookup_container(account, container)
426 439
        allowed = self._get_formatted_paths(allowed)
427
        return self._list_objects(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, allowed)
440
        return self._list_object_properties(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, size_range, allowed, all_props)
428 441
    
429 442
    @backend_method
430
    def list_object_meta(self, user, account, container, domain, until=None):
431
        """Return a list with all the container's object meta keys for the domain."""
443
    def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None):
444
        """Return a list of object (name, version_id) tuples existing under a container."""
432 445
        
433
        logger.debug("list_object_meta: %s %s %s %s", account, container, domain, until)
434
        allowed = []
435
        if user != account:
436
            if until:
437
                raise NotAllowedError
438
            allowed = self.permissions.access_list_paths(user, '/'.join((account, container)))
439
            if not allowed:
440
                raise NotAllowedError
441
        path, node = self._lookup_container(account, container)
442
        before = until if until is not None else inf
443
        allowed = self._get_formatted_paths(allowed)
444
        return self.node.latest_attribute_keys(node, domain, before, CLUSTER_DELETED, allowed)
446
        logger.debug("list_objects: %s %s %s %s %s %s %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range)
447
        return self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, False)
448
    
449
    @backend_method
450
    def list_object_meta(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], shared=False, until=None, size_range=None):
451
        """Return a list of object metadata dicts existing under a container."""
452
        
453
        logger.debug("list_object_meta: %s %s %s %s %s %s %s %s %s %s %s %s", account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range)
454
        props = self._list_objects(user, account, container, prefix, delimiter, marker, limit, virtual, domain, keys, shared, until, size_range, True)
455
        objects = []
456
        for p in props:
457
            if len(p) == 2:
458
                objects.append({'subdir': p[0]})
459
            else:
460
                objects.append({'name': p[0],
461
                                'bytes': p[self.SIZE + 1],
462
                                'type': p[self.TYPE + 1],
463
                                'hash': p[self.HASH + 1],
464
                                'version': p[self.SERIAL + 1],
465
                                'version_timestamp': p[self.MTIME + 1],
466
                                'modified': p[self.MTIME + 1] if until is None else None,
467
                                'modified_by': p[self.MUSER + 1],
468
                                'uuid': p[self.UUID + 1],
469
                                'checksum': p[self.CHECKSUM + 1]})
470
        return objects
445 471
    
446 472
    @backend_method
447 473
    def get_object_meta(self, user, account, container, name, domain, version=None):
......
466 492
        meta.update({'name': name,
467 493
                     'bytes': props[self.SIZE],
468 494
                     'type': props[self.TYPE],
469
                     'hash':props[self.HASH],
495
                     'hash': props[self.HASH],
470 496
                     'version': props[self.SERIAL],
471 497
                     'version_timestamp': props[self.MTIME],
472 498
                     'modified': modified,
......
894 920
            limit = 10000
895 921
        return start, limit
896 922
    
897
    def _list_objects(self, parent, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], until=None, size_range=None, allowed=[]):
923
    def _list_object_properties(self, parent, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], until=None, size_range=None, allowed=[], all_props=False):
898 924
        cont_prefix = path + '/'
899 925
        prefix = cont_prefix + prefix
900 926
        start = cont_prefix + marker if marker else None
......
902 928
        filterq = keys if domain else []
903 929
        sizeq = size_range
904 930
        
905
        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, domain, filterq, sizeq)
931
        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, domain, filterq, sizeq, all_props)
906 932
        objects.extend([(p, None) for p in prefixes] if virtual else [])
907 933
        objects.sort(key=lambda x: x[0])
908
        objects = [(x[0][len(cont_prefix):], x[1]) for x in objects]
934
        objects = [(x[0][len(cont_prefix):],) + x[1:] for x in objects]
909 935
        
910 936
        start, limit = self._list_limits([x[0] for x in objects], marker, limit)
911 937
        return objects[start:start + limit]

Also available in: Unified diff