Fix SQL for metadata queries in SQLalchemy module.

Fix regular expression & capitalization issues

Refs #1766

diff --git a/pithos/api/functions.py b/pithos/api/functions.py
index 39c03e3..5dfe307 100644 (file)
--- a/pithos/api/functions.py
+++ b/pithos/api/functions.py
@@ -500,7 +500,7 @@ def object_list(request, v_account, v_container):
         included, excluded, opers = parse_filters(keys)
         keys = []
         keys += [format_header_key('X-Object-Meta-' + x) for x in included]
         included, excluded, opers = parse_filters(keys)
         keys = []
         keys += [format_header_key('X-Object-Meta-' + x) for x in included]

-        keys += [format_header_key('!X-Object-Meta-' + x) for x in excluded]
+        keys += ['!'+format_header_key('X-Object-Meta-' + x) for x in excluded]

         keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
     else:
         keys = []
         keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
     else:
         keys = []

diff --git a/pithos/backends/lib/sqlalchemy/node.py b/pithos/backends/lib/sqlalchemy/node.py
index ddf9308..de8a462 100644 (file)
--- a/pithos/backends/lib/sqlalchemy/node.py
+++ b/pithos/backends/lib/sqlalchemy/node.py
@@ -35,12 +35,14 @@ from time import time
 from sqlalchemy import Table, Integer, BigInteger, DECIMAL, Column, String, MetaData, ForeignKey
 from sqlalchemy.types import Text
 from sqlalchemy.schema import Index, Sequence
 from sqlalchemy import Table, Integer, BigInteger, DECIMAL, Column, String, MetaData, ForeignKey
 from sqlalchemy.types import Text
 from sqlalchemy.schema import Index, Sequence

-from sqlalchemy.sql import func, and_, or_, null, select, bindparam, text
+from sqlalchemy.sql import func, and_, or_, not_, null, select, bindparam, text

 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.engine.reflection import Inspector
 
 from dbworker import DBWorker
 
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.engine.reflection import Inspector
 
 from dbworker import DBWorker
 

+from pithos.lib.filter import parse_filters
+

 ROOTNODE  = 0
 
 ( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8)
 ROOTNODE  = 0
 
 ( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8)

@@ -82,7 +84,6 @@ def strprevling(prefix):
         s += unichr(c-1) + unichr(0xffff)
     return s
 
         s += unichr(c-1) + unichr(0xffff)
     return s
 

-

 _propnames = {
     'serial'    : 0,
     'node'      : 1,
 _propnames = {
     'serial'    : 0,
     'node'      : 1,

@@ -752,7 +753,7 @@ class Node(DBWorker):
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,

-                            except_cluster=0, pathq=[], filterq=None):
+                            except_cluster=0, pathq=[], filterq=[]):

         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.
         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.

@@ -827,7 +828,14 @@ class Node(DBWorker):
             s = s.where(or_(*conj))
         
         if filterq:
             s = s.where(or_(*conj))
         
         if filterq:

-            s = s.where(a.c.key.in_(filterq.split(',')))
+            included, excluded, opers = parse_filters(filterq)
+            if included:
+                s = s.where(a.c.key.in_(x for x in included))
+            if excluded:
+                s = s.where(not_(a.c.key.in_(x for x in excluded)))
+            if opers:
+                for k, o, v in opers:
+                    s = s.where(or_(a.c.key == k and a.c.value.op(o)(v)))

         
         s = s.order_by(n.c.path)
         
         
         s = s.order_by(n.c.path)
         

diff --git a/pithos/backends/lib/sqlite/node.py b/pithos/backends/lib/sqlite/node.py
index e4e0b73..f46d4cf 100644 (file)
--- a/pithos/backends/lib/sqlite/node.py
+++ b/pithos/backends/lib/sqlite/node.py
@@ -597,7 +597,7 @@ class Node(DBWorker):
         
         subqlist = []
         append = subqlist.append
         
         subqlist = []
         append = subqlist.append

-        included, excluded, opers = parse_filters(filterq.split(','))
+        included, excluded, opers = parse_filters(filterq)

         args = []
         
         if included:
         args = []
         
         if included:

@@ -622,8 +622,7 @@ class Node(DBWorker):
         if not subqlist:
             return None, None
         
         if not subqlist:
             return None, None
         

-        subq = ' ' + ' and '.join(subqlist)
-        
+        subq = ' and ' + ' and '.join(subqlist)

         return subq, args
     
     def _construct_paths(self, pathq):
         return subq, args
     
     def _construct_paths(self, pathq):

@@ -665,7 +664,7 @@ class Node(DBWorker):
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,

-                            except_cluster=0, pathq=[], filterq=None):
+                            except_cluster=0, pathq=[], filterq=[]):

         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.
         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.

diff --git a/pithos/backends/modular.py b/pithos/backends/modular.py
index 12a08f0..585e7bf 100644 (file)
--- a/pithos/backends/modular.py
+++ b/pithos/backends/modular.py
@@ -799,9 +799,8 @@ class ModularBackend(BaseBackend):
         prefix = cont_prefix + prefix
         start = cont_prefix + marker if marker else None
         before = until if until is not None else inf
         prefix = cont_prefix + prefix
         start = cont_prefix + marker if marker else None
         before = until if until is not None else inf

-        filterq = ','.join(keys) if keys else None

         
         

-        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, filterq)
+        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, keys)

         objects.extend([(p, None) for p in prefixes] if virtual else [])
         objects.sort(key=lambda x: x[0])
         objects = [(x[0][len(cont_prefix):], x[1]) for x in objects]
         objects.extend([(p, None) for p in prefixes] if virtual else [])
         objects.sort(key=lambda x: x[0])
         objects = [(x[0][len(cont_prefix):], x[1]) for x in objects]

diff --git a/pithos/lib/filter.py b/pithos/lib/filter.py
index 0f64180..df935dd 100644 (file)
--- a/pithos/lib/filter.py
+++ b/pithos/lib/filter.py
@@ -34,7 +34,7 @@
 import re
 
 
 import re
 
 

-_regexfilter = re.compile('(!?)\s*([\w-]+)\s*(=|!=|<=|>=|<|>)?\s*(.*)$', re.UNICODE)
+_regexfilter = re.compile('(!?)\s*(.+)\s*(=|!=|<=|>=|<|>)?\s*(.*)$', re.UNICODE)

 
 
 def parse_filters(terms):
 
 
 def parse_filters(terms):

diff --git a/pithos/tools/pithos-test b/pithos/tools/pithos-test
index e6d454b..6c7fbcf 100755 (executable)
--- a/pithos/tools/pithos-test
+++ b/pithos/tools/pithos-test
@@ -1684,6 +1684,9 @@ class ListSharing(BaseTestCase):
         self.assertTrue('o2' not in my_shared_objects)
     
 class TestGreek(BaseTestCase):
         self.assertTrue('o2' not in my_shared_objects)
     
 class TestGreek(BaseTestCase):

+    def tearDown(self):
+        pass
+    

     def test_create_container(self):
         self.client.create_container('φάκελος')
         self.assert_container_exists('φάκελος')
     def test_create_container(self):
         self.client.create_container('φάκελος')
         self.assert_container_exists('φάκελος')

@@ -1783,9 +1786,31 @@ class TestGreek(BaseTestCase):
         meta = {'ποσότητα':'μεγάλη'}
         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
         objects = self.client.list_objects('φάκελος', meta='ποιότητα, ποσότητα')
         meta = {'ποσότητα':'μεγάλη'}
         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
         objects = self.client.list_objects('φάκελος', meta='ποιότητα, ποσότητα')

-        self.assertTrue('ο1' in objects)
-        self.assertTrue('ο2' in objects)
-        self.assertTrue('ο3' not in objects)
+        self.assertEquals(objects, ['ο1', 'ο2'])
+        
+        objects = self.client.list_objects('φάκελος', meta='!ποιότητα')
+        self.assertEquals(objects, ['ο2', 'ο3'])
+        
+        objects = self.client.list_objects('φάκελος', meta='!ποιότητα, !ποσότητα')
+        self.assertEquals(objects, ['ο3'])
+        
+        meta = {'ποιότητα':'ΑΒ'}
+        self.client.update_object_metadata('φάκελος', 'ο2', **meta)
+        objects = self.client.list_objects('φάκελος', meta='ποιότητα==ΑΑΑ')
+        self.assertEquals(objects, ['ο1'])
+        objects = self.client.list_objects('φάκελος', meta='ποιότητα!=ΑΑΑ')
+        self.assertEquals(objects, ['ο2'])
+        
+        meta = {'έτος':'2011'}
+        self.client.update_object_metadata('φάκελος', 'ο3', **meta)
+        meta = {'έτος':'2012'}
+        self.client.update_object_metadata('φάκελος', 'ο2', **meta)
+        objects = self.client.list_objects('φάκελος', meta='έτος<2012')
+        self.assertEquals(objects, ['ο1'])
+        objects = self.client.list_objects('φάκελος', meta='έτος<=2012')
+        self.assertEquals(objects, ['ο1', 'ο2'])
+        objects = self.client.list_objects('φάκελος', meta='έτος<2012,έτος!=2012')
+        self.assertEquals(objects, ['ο2'])

     
     def test_groups(self):
         #create a group
     
     def test_groups(self):
         #create a group