Revision 4819d34f

b/pithos/backends/lib/sqlite/node.py
157 157
        
158 158
        execute(""" create table if not exists attributes
159 159
                          ( serial integer,
160
                            domain text,
160 161
                            key    text,
161 162
                            value  text,
162
                            primary key (serial, key)
163
                            primary key (serial, domain, key)
163 164
                            foreign key (serial)
164 165
                            references versions(serial)
165 166
                            on update cascade
......
546 547
        self.execute(q, (serial,))
547 548
        return hash
548 549
    
549
    def attribute_get(self, serial, keys=()):
550
    def attribute_get(self, serial, domain, keys=()):
550 551
        """Return a list of (key, value) pairs of the version specified by serial.
551 552
           If keys is empty, return all attributes.
552 553
           Othwerise, return only those specified.
......
556 557
        if keys:
557 558
            marks = ','.join('?' for k in keys)
558 559
            q = ("select key, value from attributes "
559
                 "where key in (%s) and serial = ?" % (marks,))
560
            execute(q, keys + (serial,))
560
                 "where key in (%s) and serial = ? and domain = ?" % (marks,))
561
            execute(q, keys + (serial, domain))
561 562
        else:
562 563
            q = "select key, value from attributes where serial = ?"
563 564
            execute(q, (serial,))
564 565
        return self.fetchall()
565 566
    
566
    def attribute_set(self, serial, items):
567
    def attribute_set(self, serial, domain, items):
567 568
        """Set the attributes of the version specified by serial.
568 569
           Receive attributes as an iterable of (key, value) pairs.
569 570
        """
570 571
        
571
        q = ("insert or replace into attributes (serial, key, value) "
572
             "values (?, ?, ?)")
573
        self.executemany(q, ((serial, k, v) for k, v in items))
572
        q = ("insert or replace into attributes (serial, domain, key, value) "
573
             "values (?, ?, ?, ?)")
574
        self.executemany(q, ((serial, domain, k, v) for k, v in items))
574 575
    
575
    def attribute_del(self, serial, keys=()):
576
    def attribute_del(self, serial, domain, keys=()):
576 577
        """Delete attributes of the version specified by serial.
577 578
           If keys is empty, delete all attributes.
578 579
           Otherwise delete those specified.
579 580
        """
580 581
        
581 582
        if keys:
582
            q = "delete from attributes where serial = ? and key = ?"
583
            self.executemany(q, ((serial, key) for key in keys))
583
            q = "delete from attributes where serial = ? and domain = ? and key = ?"
584
            self.executemany(q, ((serial, domain, key) for key in keys))
584 585
        else:
585
            q = "delete from attributes where serial = ?"
586
            self.execute(q, (serial,))
586
            q = "delete from attributes where serial = ? and domain = ?"
587
            self.execute(q, (serial, domain))
587 588
    
588 589
    def attribute_copy(self, source, dest):
589 590
        q = ("insert or replace into attributes "
590
             "select ?, key, value from attributes "
591
             "select ?, domain, key, value from attributes "
591 592
             "where serial = ?")
592 593
        self.execute(q, (dest, source))
593 594
    
594
    def _construct_filters(self, filterq):
595
        if not filterq:
595
    def _construct_filters(self, domain, filterq):
596
        print '***', domain, filterq
597
        if not domain or not filterq:
596 598
            return None, None
597 599
        
598 600
        subqlist = []
599 601
        append = subqlist.append
600
        included, excluded, opers = parse_filters(filterq.split(','))
602
        included, excluded, opers = parse_filters(filterq)
601 603
        args = []
602 604
        
603 605
        if included:
......
616 618
            t = (("(a.key = ? and a.value %s ?)" % (o,)) for k, o, v in opers)
617 619
            subq = "(" + ' or '.join(t) + ")"
618 620
            for k, o, v in opers:
619
                args += (k, v)
621
                args += [k, v]
620 622
            append(subq)
621 623
        
622 624
        if not subqlist:
623 625
            return None, None
624 626
        
625
        subq = ' ' + ' and '.join(subqlist)
627
        subq = ' and a.domain = ? and ' + ' and '.join(subqlist)
628
        args = [domain] + args
626 629
        
627 630
        return subq, args
628 631
    
......
637 640
        
638 641
        return subq, args
639 642
    
640
    def latest_attribute_keys(self, parent, before=inf, except_cluster=0, pathq=[]):
643
    def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):
641 644
        """Return a list with all keys pairs defined
642 645
           for all latest versions under parent that
643 646
           do not belong to the cluster.
......
654 657
                           "from nodes "
655 658
                           "where parent = ?) "
656 659
             "and a.serial = v.serial "
660
             "and a.domain = ? "
657 661
             "and n.node = v.node")
658
        args = (before, except_cluster, parent)
662
        args = (before, except_cluster, parent, domain)
659 663
        subq, subargs = self._construct_paths(pathq)
660 664
        if subq is not None:
661 665
            q += subq
......
665 669
    
666 670
    def latest_version_list(self, parent, prefix='', delimiter=None,
667 671
                            start='', limit=10000, before=inf,
668
                            except_cluster=0, pathq=[], filterq=None):
672
                            except_cluster=0, pathq=[], domain=None, filterq=[]):
669 673
        """Return a (list of (path, serial) tuples, list of common prefixes)
670 674
           for the current versions of the paths with the given parent,
671 675
           matching the following criteria.
......
736 740
        if subq is not None:
737 741
            q += subq
738 742
            args += subargs
739
        subq, subargs = self._construct_filters(filterq)
743
        subq, subargs = self._construct_filters(domain, filterq)
740 744
        if subq is not None:
741 745
            q += subq
742 746
            args += subargs
b/pithos/backends/modular.py
146 146
        else:
147 147
            meta = {}
148 148
            if props is not None:
149
                meta.update(dict(self.node.attribute_get(props[self.SERIAL])))
149
                meta.update(dict(self.node.attribute_get(props[self.SERIAL], domain)))
150 150
            if until is not None:
151 151
                meta.update({'until_timestamp': tstamp})
152 152
            meta.update({'name': account, 'count': count, 'bytes': bytes})
......
161 161
        if user != account:
162 162
            raise NotAllowedError
163 163
        path, node = self._lookup_account(account, True)
164
        self._put_metadata(user, node, meta, replace)
164
        self._put_metadata(user, node, domain, meta, replace)
165 165
    
166 166
    @backend_method
167 167
    def get_account_groups(self, user, account):
......
261 261
            start, limit = self._list_limits(allowed, marker, limit)
262 262
            return allowed[start:start + limit]
263 263
        node = self.node.node_lookup(account)
264
        return [x[0] for x in self._list_objects(node, account, '', '/', marker, limit, False, [], until)]
264
        return [x[0] for x in self._list_objects(node, account, '', '/', marker, limit, False, None, [], until)]
265 265
    
266 266
    @backend_method
267 267
    def get_container_meta(self, user, account, container, domain, until=None):
......
285 285
        if user != account:
286 286
            meta = {'name': container}
287 287
        else:
288
            meta = dict(self.node.attribute_get(props[self.SERIAL]))
288
            meta = dict(self.node.attribute_get(props[self.SERIAL], domain))
289 289
            if until is not None:
290 290
                meta.update({'until_timestamp': tstamp})
291 291
            meta.update({'name': container, 'count': count, 'bytes': bytes})
......
300 300
        if user != account:
301 301
            raise NotAllowedError
302 302
        path, node = self._lookup_container(account, container)
303
        self._put_metadata(user, node, meta, replace)
303
        self._put_metadata(user, node, domain, meta, replace)
304 304
    
305 305
    @backend_method
306 306
    def get_container_policy(self, user, account, container):
......
386 386
                if not allowed:
387 387
                    return []
388 388
        path, node = self._lookup_container(account, container)
389
        return self._list_objects(node, path, prefix, delimiter, marker, limit, virtual, keys, until, allowed)
389
        return self._list_objects(node, path, prefix, delimiter, marker, limit, virtual, domain, keys, until, allowed)
390 390
    
391 391
    @backend_method
392 392
    def list_object_meta(self, user, account, container, domain, until=None):
......
402 402
                raise NotAllowedError
403 403
        path, node = self._lookup_container(account, container)
404 404
        before = until if until is not None else inf
405
        return self.node.latest_attribute_keys(node, before, CLUSTER_DELETED, allowed)
405
        return self.node.latest_attribute_keys(node, domain, before, CLUSTER_DELETED, allowed)
406 406
    
407 407
    @backend_method
408 408
    def get_object_meta(self, user, account, container, name, domain, version=None):
......
423 423
                    raise NameError('Object does not exist')
424 424
                modified = del_props[self.MTIME]
425 425
        
426
        meta = dict(self.node.attribute_get(props[self.SERIAL]))
426
        meta = dict(self.node.attribute_get(props[self.SERIAL], domain))
427 427
        meta.update({'name': name, 'bytes': props[self.SIZE], 'hash':props[self.HASH]})
428 428
        meta.update({'version': props[self.SERIAL], 'version_timestamp': props[self.MTIME]})
429 429
        meta.update({'modified': modified, 'modified_by': props[self.MUSER]})
......
436 436
        logger.debug("update_object_meta: %s %s %s %s %s %s", account, container, name, domain, meta, replace)
437 437
        self._can_write(user, account, container, name)
438 438
        path, node = self._lookup_object(account, container, name)
439
        src_version_id, dest_version_id = self._put_metadata(user, node, meta, replace)
439
        src_version_id, dest_version_id = self._put_metadata(user, node, domain, meta, replace)
440 440
        self._apply_versioning(account, container, src_version_id)
441 441
        return dest_version_id
442 442
    
......
505 505
        hashmap = self.store.map_get(binascii.unhexlify(props[self.HASH]))
506 506
        return props[self.SIZE], [binascii.hexlify(x) for x in hashmap]
507 507
    
508
    def _update_object_hash(self, user, account, container, name, size, hash, meta={}, replace_meta=False, permissions=None):
508
    def _update_object_hash(self, user, account, container, name, size, hash, permissions=None):
509 509
        if permissions is not None and user != account:
510 510
            raise NotAllowedError
511 511
        self._can_write(user, account, container, name)
......
528 528
                # This must be executed in a transaction, so the version is never created if it fails.
529 529
                raise QuotaError
530 530
        
531
        if not replace_meta and src_version_id is not None:
532
            self.node.attribute_copy(src_version_id, dest_version_id)
533
        self.node.attribute_set(dest_version_id, ((k, v) for k, v in meta.iteritems()))
534 531
        if permissions is not None:
535 532
            self.permissions.access_set(path, permissions)
536 533
        self._apply_versioning(account, container, src_version_id)
537
        return dest_version_id
534
        return src_version_id, dest_version_id
538 535
    
539 536
    @backend_method
540 537
    def update_object_hashmap(self, user, account, container, name, size, hashmap, domain, meta={}, replace_meta=False, permissions=None):
......
552 549
            raise ie
553 550
        
554 551
        hash = map.hash()
555
        dest_version_id = self._update_object_hash(user, account, container, name, size, binascii.hexlify(hash), meta, replace_meta, permissions)
552
        src_version_id, dest_version_id = self._update_object_hash(user, account, container, name, size, binascii.hexlify(hash), permissions)
553
        self._put_metadata_duplicate(src_version_id, dest_version_id, domain, meta, replace_meta)
556 554
        self.store.map_put(hash, map)
557 555
        return dest_version_id
558 556
    
559
    def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_meta={}, replace_meta=False, permissions=None, src_version=None):
557
    def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None):
560 558
        self._can_read(user, src_account, src_container, src_name)
561 559
        path, node = self._lookup_object(src_account, src_container, src_name)
562 560
        props = self._get_version(node, src_version)
......
564 562
        hash = props[self.HASH]
565 563
        size = props[self.SIZE]
566 564
        
567
        if (src_account, src_container, src_name) == (dest_account, dest_container, dest_name):
568
            dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, hash, dest_meta, replace_meta, permissions)
569
        else:
570
            if replace_meta:
571
                meta = dest_meta
572
            else:
573
                meta = {}
574
            dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, hash, meta, True, permissions)
575
            if not replace_meta:
576
                self.node.attribute_copy(src_version_id, dest_version_id)
577
                self.node.attribute_set(dest_version_id, ((k, v) for k, v in dest_meta.iteritems()))
565
        src_v_id, dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, hash, permissions)
566
        self._put_metadata_duplicate(src_version_id, dest_version_id, dest_domain, dest_meta, replace_meta)
578 567
        return dest_version_id
579 568
    
580 569
    @backend_method
......
582 571
        """Copy an object's data and metadata."""
583 572
        
584 573
        logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version)
585
        return self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_meta, replace_meta, permissions, src_version)
574
        return self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version)
586 575
    
587 576
    @backend_method
588 577
    def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta={}, replace_meta=False, permissions=None):
......
591 580
        logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions)
592 581
        if user != src_account:
593 582
            raise NotAllowedError
594
        dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, meta, replace_meta, permissions, None)
583
        dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, None)
595 584
        if (src_account, src_container, src_name) != (dest_account, dest_container, dest_name):
596 585
            self._delete_object(user, src_account, src_container, src_name)
597 586
        return dest_version_id
......
768 757
        dest_version_id, mtime = self.node.version_create(node, hash, size, src_version_id, user, cluster)
769 758
        return src_version_id, dest_version_id
770 759
    
771
    def _put_metadata(self, user, node, meta, replace=False):
760
    def _put_metadata_duplicate(self, src_version_id, dest_version_id, domain, meta, replace=False):
761
        if src_version_id is not None:
762
            self.node.attribute_copy(src_version_id, dest_version_id)
763
        if not replace:
764
            self.node.attribute_del(dest_version_id, domain, (k for k, v in meta.iteritems() if v == ''))
765
            self.node.attribute_set(dest_version_id, domain, ((k, v) for k, v in meta.iteritems() if v != ''))
766
        else:
767
            self.node.attribute_del(dest_version_id, domain)
768
            self.node.attribute_set(dest_version_id, domain, ((k, v) for k, v in meta.iteritems()))
769
    
770
    def _put_metadata(self, user, node, domain, meta, replace=False):
772 771
        """Create a new version and store metadata."""
773 772
        
774 773
        src_version_id, dest_version_id = self._put_version_duplicate(user, node)
775
        
776
        # TODO: Merge with other functions that update metadata...
777
        if not replace:
778
            if src_version_id is not None:
779
                self.node.attribute_copy(src_version_id, dest_version_id)
780
            self.node.attribute_del(dest_version_id, (k for k, v in meta.iteritems() if v == ''))
781
            self.node.attribute_set(dest_version_id, ((k, v) for k, v in meta.iteritems() if v != ''))
782
        else:
783
            self.node.attribute_set(dest_version_id, ((k, v) for k, v in meta.iteritems()))
774
        self._put_metadata_duplicate(src_version_id, dest_version_id, domain, meta, replace)
784 775
        return src_version_id, dest_version_id
785 776
    
786 777
    def _list_limits(self, listing, marker, limit):
......
794 785
            limit = 10000
795 786
        return start, limit
796 787
    
797
    def _list_objects(self, parent, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], until=None, allowed=[]):
788
    def _list_objects(self, parent, path, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, domain=None, keys=[], until=None, allowed=[]):
798 789
        cont_prefix = path + '/'
799 790
        prefix = cont_prefix + prefix
800 791
        start = cont_prefix + marker if marker else None
801 792
        before = until if until is not None else inf
802
        filterq = ','.join(keys) if keys else None
793
        filterq = keys if domain else []
803 794
        
804
        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, filterq)
795
        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, domain, filterq)
805 796
        objects.extend([(p, None) for p in prefixes] if virtual else [])
806 797
        objects.sort(key=lambda x: x[0])
807 798
        objects = [(x[0][len(cont_prefix):], x[1]) for x in objects]
b/pithos/lib/filter.py
34 34
import re
35 35

  
36 36

  
37
_regexfilter = re.compile('(!?)\s*([\w-]+)\s*(=|!=|<=|>=|<|>)?\s*(.*)$', re.UNICODE)
37
_regexfilter = re.compile('(!?)\s*(.+)\s*(=|!=|<=|>=|<|>)?\s*(.*)$', re.UNICODE)
38 38

  
39 39

  
40 40
def parse_filters(terms):

Also available in: Unified diff