Fix SQL for metadata queries in SQLalchemy module.

Fix regular expression & capitalization issues

Refs #1766

diff --git a/pithos/api/functions.py b/pithos/api/functions.py
index 39c03e3..5dfe307 100644 (file)
--- a/pithos/api/functions.py
+++ b/pithos/api/functions.py
@@ -500,7 +500,7 @@ def object_list(request, v_account, v_container):
         included, excluded, opers = parse_filters(keys)
         keys = []
         keys += [format_header_key('X-Object-Meta-' + x) for x in included]
-        keys += [format_header_key('!X-Object-Meta-' + x) for x in excluded]
+        keys += ['!'+format_header_key('X-Object-Meta-' + x) for x in excluded]
         keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
     else:
         keys = []

diff --git a/pithos/backends/lib/sqlalchemy/node.py b/pithos/backends/lib/sqlalchemy/node.py
index ddf9308..de8a462 100644 (file)
--- a/pithos/backends/lib/sqlalchemy/node.py
+++ b/pithos/backends/lib/sqlalchemy/node.py
@@ -35,12 +35,14 @@ from time import time
 from sqlalchemy import Table, Integer, BigInteger, DECIMAL, Column, String, MetaData, ForeignKey
 from sqlalchemy.types import Text
 from sqlalchemy.schema import Index, Sequence
-from sqlalchemy.sql import func, and_, or_, null, select, bindparam, text
+from sqlalchemy.sql import func, and_, or_, not_, null, select, bindparam, text
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.engine.reflection import Inspector
 
 from dbworker import DBWorker
 
+from pithos.lib.filter import parse_filters
+
 ROOTNODE  = 0
 
 ( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8)
@@ -82,7 +84,6 @@ def strprevling(prefix):
         s += unichr(c-1) + unichr(0xffff)
     return s
 
-
 _propnames = {
     'serial'    : 0,
     'node'      : 1,
@@ -752,7 +753,7 @@ class Node(DBWorker):
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,
-                            except_cluster=0, pathq=[], filterq=None):
+                            except_cluster=0, pathq=[], filterq=[]):
         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.
@@ -827,7 +828,14 @@ class Node(DBWorker):
             s = s.where(or_(*conj))
         
         if filterq:
-            s = s.where(a.c.key.in_(filterq.split(',')))
+            included, excluded, opers = parse_filters(filterq)
+            if included:
+                s = s.where(a.c.key.in_(x for x in included))
+            if excluded:
+                s = s.where(not_(a.c.key.in_(x for x in excluded)))
+            if opers:
+                for k, o, v in opers:
+                    s = s.where(or_(a.c.key == k and a.c.value.op(o)(v)))
         
         s = s.order_by(n.c.path)
         

diff --git a/pithos/backends/lib/sqlite/node.py b/pithos/backends/lib/sqlite/node.py
index e4e0b73..f46d4cf 100644 (file)
--- a/pithos/backends/lib/sqlite/node.py
+++ b/pithos/backends/lib/sqlite/node.py
@@ -597,7 +597,7 @@ class Node(DBWorker):
         
         subqlist = []
         append = subqlist.append
-        included, excluded, opers = parse_filters(filterq.split(','))
+        included, excluded, opers = parse_filters(filterq)
         args = []
         
         if included:
@@ -622,8 +622,7 @@ class Node(DBWorker):
         if not subqlist:
             return None, None
         
-        subq = ' ' + ' and '.join(subqlist)
-        
+        subq = ' and ' + ' and '.join(subqlist)
         return subq, args
     
     def _construct_paths(self, pathq):
@@ -665,7 +664,7 @@ class Node(DBWorker):
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,
-                            except_cluster=0, pathq=[], filterq=None):
+                            except_cluster=0, pathq=[], filterq=[]):
         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.

diff --git a/pithos/backends/modular.py b/pithos/backends/modular.py
index 12a08f0..585e7bf 100644 (file)
--- a/pithos/backends/modular.py
+++ b/pithos/backends/modular.py
@@ -799,9 +799,8 @@ class ModularBackend(BaseBackend):
         prefix = cont_prefix + prefix
         start = cont_prefix + marker if marker else None
         before = until if until is not None else inf
-        filterq = ','.join(keys) if keys else None
         
-        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, filterq)
+        objects, prefixes = self.node.latest_version_list(parent, prefix, delimiter, start, limit, before, CLUSTER_DELETED, allowed, keys)
         objects.extend([(p, None) for p in prefixes] if virtual else [])
         objects.sort(key=lambda x: x[0])
         objects = [(x[0][len(cont_prefix):], x[1]) for x in objects]

diff --git a/pithos/lib/filter.py b/pithos/lib/filter.py
index 0f64180..df935dd 100644 (file)
--- a/pithos/lib/filter.py
+++ b/pithos/lib/filter.py
@@ -34,7 +34,7 @@
 import re
 
 
-_regexfilter = re.compile('(!?)\s*([\w-]+)\s*(=|!=|<=|>=|<|>)?\s*(.*)$', re.UNICODE)
+_regexfilter = re.compile('(!?)\s*(.+)\s*(=|!=|<=|>=|<|>)?\s*(.*)$', re.UNICODE)
 
 
 def parse_filters(terms):

diff --git a/pithos/tools/pithos-test b/pithos/tools/pithos-test
index e6d454b..6c7fbcf 100755 (executable)
--- a/pithos/tools/pithos-test
+++ b/pithos/tools/pithos-test
@@ -1684,6 +1684,9 @@ class ListSharing(BaseTestCase):
         self.assertTrue('o2' not in my_shared_objects)
     
 class TestGreek(BaseTestCase):
+    def tearDown(self):
+        pass
+    
     def test_create_container(self):
         self.client.create_container('φάκελος')
         self.assert_container_exists('φάκελος')
@@ -1783,9 +1786,31 @@ class TestGreek(BaseTestCase):
         meta = {'ποσότητα':'μεγάλη'}
         self.client.update_object_metadata('φάκελος', 'ο2', **meta)
         objects = self.client.list_objects('φάκελος', meta='ποιότητα, ποσότητα')
-        self.assertTrue('ο1' in objects)
-        self.assertTrue('ο2' in objects)
-        self.assertTrue('ο3' not in objects)
+        self.assertEquals(objects, ['ο1', 'ο2'])
+        
+        objects = self.client.list_objects('φάκελος', meta='!ποιότητα')
+        self.assertEquals(objects, ['ο2', 'ο3'])
+        
+        objects = self.client.list_objects('φάκελος', meta='!ποιότητα, !ποσότητα')
+        self.assertEquals(objects, ['ο3'])
+        
+        meta = {'ποιότητα':'ΑΒ'}
+        self.client.update_object_metadata('φάκελος', 'ο2', **meta)
+        objects = self.client.list_objects('φάκελος', meta='ποιότητα==ΑΑΑ')
+        self.assertEquals(objects, ['ο1'])
+        objects = self.client.list_objects('φάκελος', meta='ποιότητα!=ΑΑΑ')
+        self.assertEquals(objects, ['ο2'])
+        
+        meta = {'έτος':'2011'}
+        self.client.update_object_metadata('φάκελος', 'ο3', **meta)
+        meta = {'έτος':'2012'}
+        self.client.update_object_metadata('φάκελος', 'ο2', **meta)
+        objects = self.client.list_objects('φάκελος', meta='έτος<2012')
+        self.assertEquals(objects, ['ο1'])
+        objects = self.client.list_objects('φάκελος', meta='έτος<=2012')
+        self.assertEquals(objects, ['ο1', 'ο2'])
+        objects = self.client.list_objects('φάκελος', meta='έτος<2012,έτος!=2012')
+        self.assertEquals(objects, ['ο2'])
     
     def test_groups(self):
         #create a group