Add size queries in backend object lists.

[pithos] / pithos / backends / lib / sqlalchemy / node.py

diff --git a/pithos/backends/lib/sqlalchemy/node.py b/pithos/backends/lib/sqlalchemy/node.py
index c8d9713..094e8d4 100644 (file)
--- a/pithos/backends/lib/sqlalchemy/node.py
+++ b/pithos/backends/lib/sqlalchemy/node.py
@@ -35,13 +35,14 @@ from time import time
 from sqlalchemy import Table, Integer, BigInteger, DECIMAL, Column, String, MetaData, ForeignKey
 from sqlalchemy.types import Text
 from sqlalchemy.schema import Index, Sequence
 from sqlalchemy import Table, Integer, BigInteger, DECIMAL, Column, String, MetaData, ForeignKey
 from sqlalchemy.types import Text
 from sqlalchemy.schema import Index, Sequence

-from sqlalchemy.sql import func, and_, or_, not_, null, select, bindparam, text
+from sqlalchemy.sql import func, and_, or_, not_, null, select, bindparam, text, exists

 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.engine.reflection import Inspector
 
 from dbworker import DBWorker
 
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.engine.reflection import Inspector
 
 from dbworker import DBWorker
 

-from pithos.lib.filter import parse_filters, OPERATORS
+from pithos.lib.filter import parse_filters
+

 
 ROOTNODE  = 0
 
 
 ROOTNODE  = 0
 

@@ -185,7 +186,7 @@ class Node(DBWorker):
         indexes = [elem['name'] for elem in insp.get_indexes('nodes')]
         if 'idx_nodes_path' not in indexes:
             explicit_length = '(%s)' %path_length if self.engine.name == 'mysql' else ''
         indexes = [elem['name'] for elem in insp.get_indexes('nodes')]
         if 'idx_nodes_path' not in indexes:
             explicit_length = '(%s)' %path_length if self.engine.name == 'mysql' else ''

-            s = text('CREATE INDEX idx_nodes_path ON nodes (path%s)' %explicit_length)
+            s = text('CREATE UNIQUE INDEX idx_nodes_path ON nodes (path%s)' %explicit_length)

             self.conn.execute(s).close()
         
         s = self.nodes.select().where(and_(self.nodes.c.node == ROOTNODE,
             self.conn.execute(s).close()
         
         s = self.nodes.select().where(and_(self.nodes.c.node == ROOTNODE,

@@ -767,7 +768,7 @@ class Node(DBWorker):
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,

-                            except_cluster=0, pathq=[], domain=None, filterq=[]):
+                            except_cluster=0, pathq=[], domain=None, filterq=[], sizeq=None):

         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.
         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.

@@ -802,6 +803,8 @@ class Node(DBWorker):
                    key ?op value
                        the attribute with this key satisfies the value
                        where ?op is one of ==, != <=, >=, <, >.
                    key ?op value
                        the attribute with this key satisfies the value
                        where ?op is one of ==, != <=, >=, <, >.

+                
+                h. the size is in the range set by sizeq

            
            The list of common prefixes includes the prefixes
            matching up to the first delimiter after prefix,
            
            The list of common prefixes includes the prefixes
            matching up to the first delimiter after prefix,

@@ -818,7 +821,6 @@ class Node(DBWorker):
             start = strprevling(prefix)
         nextling = strnextling(prefix)
         
             start = strprevling(prefix)
         nextling = strnextling(prefix)
         

-        a = self.attributes.alias('a')

         v = self.versions.alias('v')
         n = self.nodes.alias('n')
         s = select([n.c.path, v.c.serial]).distinct()
         v = self.versions.alias('v')
         n = self.nodes.alias('n')
         s = select([n.c.path, v.c.serial]).distinct()

@@ -835,43 +837,44 @@ class Node(DBWorker):
         conj = []
         for x in pathq:
             conj.append(n.c.path.like(self.escape_like(x) + '%', escape='\\'))
         conj = []
         for x in pathq:
             conj.append(n.c.path.like(self.escape_like(x) + '%', escape='\\'))

-        

         if conj:
             s = s.where(or_(*conj))
         
         if conj:
             s = s.where(or_(*conj))
         

-        s = s.order_by(n.c.path)
+        if sizeq and len(sizeq) == 2:
+            if sizeq[0]:
+                s = s.where(v.c.size >= sizeq[0])
+            if sizeq[1]:
+                s = s.where(v.c.size < sizeq[1])

         
         

-        def filterout(r):
-            if not filterq:
-                return False
-            path, serial = r
+        if domain and filterq:
+            a = self.attributes.alias('a')

             included, excluded, opers = parse_filters(filterq)
             included, excluded, opers = parse_filters(filterq)

-            
-            #retrieve metadata
-            meta = dict(self.attribute_get(serial, domain))
-            keyset= set([k.encode('utf8') for k in meta.keys()])
-            

             if included:
             if included:

-                if not set(included) & keyset:
-                    return True
+                subs = select([1])
+                subs = subs.where(a.c.serial == v.c.serial).correlate(v)
+                subs = subs.where(a.c.domain == domain)
+                subs = subs.where(or_(*[a.c.key.op('=')(x) for x in included]))
+                s = s.where(exists(subs))

             if excluded:
             if excluded:

-                if set(excluded) & keyset:
-                    return True
-            for k, op, v in opers:
-                k = k.decode('utf8')
-                v = v.decode('utf8')
-                if k not in meta:
-                    return True
-                operation = OPERATORS[op]
-                if not operation(meta[k], v):
-                    return True
-            return False
-    
+                subs = select([1])
+                subs = subs.where(a.c.serial == v.c.serial).correlate(v)
+                subs = subs.where(a.c.domain == domain)
+                subs = subs.where(or_(*[a.c.key.op('=')(x) for x in excluded]))
+                s = s.where(not_(exists(subs)))
+            if opers:
+                for k, o, val in opers:
+                    subs = select([1])
+                    subs = subs.where(a.c.serial == v.c.serial).correlate(v)
+                    subs = subs.where(a.c.domain == domain)
+                    subs = subs.where(and_(a.c.key.op('=')(k), a.c.value.op(o)(val)))
+                    s = s.where(exists(subs))
+        
+        s = s.order_by(n.c.path)
+        

         if not delimiter:
             s = s.limit(limit)
             rp = self.conn.execute(s, start=start)
         if not delimiter:
             s = s.limit(limit)
             rp = self.conn.execute(s, start=start)

-            filter_ = lambda r : [t for t in r if not filterout(t)]
-            r = filter_(rp.fetchall())
+            r = rp.fetchall()

             rp.close()
             return r, ()
         
             rp.close()
             return r, ()
         

@@ -888,8 +891,6 @@ class Node(DBWorker):
             props = rp.fetchone()
             if props is None:
                 break
             props = rp.fetchone()
             if props is None:
                 break

-            if filterout(props):
-                continue

             path, serial = props
             idx = path.find(delimiter, pfz)
             
             path, serial = props
             idx = path.find(delimiter, pfz)