Fix metadata search.

[pithos] / pithos / backends / lib / sqlite / node.py

diff --git a/pithos/backends/lib/sqlite/node.py b/pithos/backends/lib/sqlite/node.py
index 6348a3d..87d3dce 100644 (file)
--- a/pithos/backends/lib/sqlite/node.py
+++ b/pithos/backends/lib/sqlite/node.py
@@ -31,16 +31,16 @@
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 
 # interpreted as representing official policies, either expressed
 # or implied, of GRNET S.A.
 

-import re
-

 from time import time
 
 from dbworker import DBWorker
 
 from time import time
 
 from dbworker import DBWorker
 

+from pithos.lib.filter import parse_filters
+

 
 ROOTNODE  = 0
 
 
 ROOTNODE  = 0
 

-( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8)
+( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, UUID, CLUSTER ) = range(9)

 
 inf = float('inf')
 
 
 inf = float('inf')
 

@@ -80,8 +80,6 @@ def strprevling(prefix):
     return s
 
 
     return s
 
 

-_regexfilter = re.compile('(!?)\s*([\w-]+)\s*(==|!=|<=|>=|<|>)?\s*(.*)$', re.UNICODE)
-

 _propnames = {
     'serial'    : 0,
     'node'      : 1,
 _propnames = {
     'serial'    : 0,
     'node'      : 1,

@@ -90,7 +88,8 @@ _propnames = {
     'source'    : 4,
     'mtime'     : 5,
     'muser'     : 6,
     'source'    : 4,
     'mtime'     : 5,
     'muser'     : 6,

-    'cluster'   : 7,
+    'uuid'      : 7,
+    'cluster'   : 8

 }
 
 
 }
 
 

@@ -149,6 +148,7 @@ class Node(DBWorker):
                             source     integer,
                             mtime      integer,
                             muser      text    not null default '',
                             source     integer,
                             mtime      integer,
                             muser      text    not null default '',

+                            uuid       text    not null default '',

                             cluster    integer not null default 0,
                             foreign key (node)
                             references nodes(node)
                             cluster    integer not null default 0,
                             foreign key (node)
                             references nodes(node)

@@ -156,12 +156,15 @@ class Node(DBWorker):
                             on delete cascade ) """)
         execute(""" create index if not exists idx_versions_node_mtime
                     on versions(node, mtime) """)
                             on delete cascade ) """)
         execute(""" create index if not exists idx_versions_node_mtime
                     on versions(node, mtime) """)

+        execute(""" create index if not exists idx_versions_node_uuid
+                    on versions(uuid) """)

         
         execute(""" create table if not exists attributes
                           ( serial integer,
         
         execute(""" create table if not exists attributes
                           ( serial integer,

+                            domain text,

                             key    text,
                             value  text,
                             key    text,
                             value  text,

-                            primary key (serial, key)
+                            primary key (serial, domain, key)

                             foreign key (serial)
                             references versions(serial)
                             on update cascade
                             foreign key (serial)
                             references versions(serial)
                             on update cascade

@@ -204,10 +207,10 @@ class Node(DBWorker):
     def node_get_versions(self, node, keys=(), propnames=_propnames):
         """Return the properties of all versions at node.
            If keys is empty, return all properties in the order
     def node_get_versions(self, node, keys=(), propnames=_propnames):
         """Return the properties of all versions at node.
            If keys is empty, return all properties in the order

-           (serial, node, size, source, mtime, muser, cluster).
+           (serial, node, hash, size, source, mtime, muser, uuid, cluster).

         """
         
         """
         

-        q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+        q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "

              "from versions "
              "where node = ?")
         self.execute(q, (node,))
              "from versions "
              "where node = ?")
         self.execute(q, (node,))

@@ -410,7 +413,7 @@ class Node(DBWorker):
         parent, path = props
         
         # The latest version.
         parent, path = props
         
         # The latest version.

-        q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+        q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "

              "from versions "
              "where serial = (select max(serial) "
                              "from versions "
              "from versions "
              "where serial = (select max(serial) "
                              "from versions "

@@ -460,15 +463,15 @@ class Node(DBWorker):
         mtime = max(mtime, r[2])
         return (count, size, mtime)
     
         mtime = max(mtime, r[2])
         return (count, size, mtime)
     

-    def version_create(self, node, hash, size, source, muser, cluster=0):
+    def version_create(self, node, hash, size, source, muser, uuid, cluster=0):

         """Create a new version from the given properties.
            Return the (serial, mtime) of the new version.
         """
         
         """Create a new version from the given properties.
            Return the (serial, mtime) of the new version.
         """
         

-        q = ("insert into versions (node, hash, size, source, mtime, muser, cluster) "
-             "values (?, ?, ?, ?, ?, ?, ?)")
+        q = ("insert into versions (node, hash, size, source, mtime, muser, uuid, cluster) "
+             "values (?, ?, ?, ?, ?, ?, ?, ?)")

         mtime = time()
         mtime = time()

-        props = (node, hash, size, source, mtime, muser, cluster)
+        props = (node, hash, size, source, mtime, muser, uuid, cluster)

         serial = self.execute(q, props).lastrowid
         self.statistics_update_ancestors(node, 1, size, mtime, cluster)
         return serial, mtime
         serial = self.execute(q, props).lastrowid
         self.statistics_update_ancestors(node, 1, size, mtime, cluster)
         return serial, mtime

@@ -476,11 +479,11 @@ class Node(DBWorker):
     def version_lookup(self, node, before=inf, cluster=0):
         """Lookup the current version of the given node.
            Return a list with its properties:
     def version_lookup(self, node, before=inf, cluster=0):
         """Lookup the current version of the given node.
            Return a list with its properties:

-           (serial, node, hash, size, source, mtime, muser, cluster)
+           (serial, node, hash, size, source, mtime, muser, uuid, cluster)

            or None if the current version is not found in the given cluster.
         """
         
            or None if the current version is not found in the given cluster.
         """
         

-        q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+        q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "

              "from versions "
              "where serial = (select max(serial) "
                              "from versions "
              "from versions "
              "where serial = (select max(serial) "
                              "from versions "

@@ -496,10 +499,10 @@ class Node(DBWorker):
         """Return a sequence of values for the properties of
            the version specified by serial and the keys, in the order given.
            If keys is empty, return all properties in the order
         """Return a sequence of values for the properties of
            the version specified by serial and the keys, in the order given.
            If keys is empty, return all properties in the order

-           (serial, node, hash, size, source, mtime, muser, cluster).
+           (serial, node, hash, size, source, mtime, muser, uuid, cluster).

         """
         
         """
         

-        q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+        q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "

              "from versions "
              "where serial = ?")
         self.execute(q, (serial,))
              "from versions "
              "where serial = ?")
         self.execute(q, (serial,))

@@ -548,7 +551,7 @@ class Node(DBWorker):
         self.execute(q, (serial,))
         return hash
     
         self.execute(q, (serial,))
         return hash
     

-    def attribute_get(self, serial, keys=()):
+    def attribute_get(self, serial, domain, keys=()):

         """Return a list of (key, value) pairs of the version specified by serial.
            If keys is empty, return all attributes.
            Othwerise, return only those specified.
         """Return a list of (key, value) pairs of the version specified by serial.
            If keys is empty, return all attributes.
            Othwerise, return only those specified.

@@ -558,93 +561,78 @@ class Node(DBWorker):
         if keys:
             marks = ','.join('?' for k in keys)
             q = ("select key, value from attributes "
         if keys:
             marks = ','.join('?' for k in keys)
             q = ("select key, value from attributes "

-                 "where key in (%s) and serial = ?" % (marks,))
-            execute(q, keys + (serial,))
+                 "where key in (%s) and serial = ? and domain = ?" % (marks,))
+            execute(q, keys + (serial, domain))

         else:
         else:

-            q = "select key, value from attributes where serial = ?"
-            execute(q, (serial,))
+            q = "select key, value from attributes where serial = ? and domain = ?"
+            execute(q, (serial, domain))

         return self.fetchall()
     
         return self.fetchall()
     

-    def attribute_set(self, serial, items):
+    def attribute_set(self, serial, domain, items):

         """Set the attributes of the version specified by serial.
            Receive attributes as an iterable of (key, value) pairs.
         """
         
         """Set the attributes of the version specified by serial.
            Receive attributes as an iterable of (key, value) pairs.
         """
         

-        q = ("insert or replace into attributes (serial, key, value) "
-             "values (?, ?, ?)")
-        self.executemany(q, ((serial, k, v) for k, v in items))
+        q = ("insert or replace into attributes (serial, domain, key, value) "
+             "values (?, ?, ?, ?)")
+        self.executemany(q, ((serial, domain, k, v) for k, v in items))

     
     

-    def attribute_del(self, serial, keys=()):
+    def attribute_del(self, serial, domain, keys=()):

         """Delete attributes of the version specified by serial.
            If keys is empty, delete all attributes.
            Otherwise delete those specified.
         """
         
         if keys:
         """Delete attributes of the version specified by serial.
            If keys is empty, delete all attributes.
            Otherwise delete those specified.
         """
         
         if keys:

-            q = "delete from attributes where serial = ? and key = ?"
-            self.executemany(q, ((serial, key) for key in keys))
+            q = "delete from attributes where serial = ? and domain = ? and key = ?"
+            self.executemany(q, ((serial, domain, key) for key in keys))

         else:
         else:

-            q = "delete from attributes where serial = ?"
-            self.execute(q, (serial,))
+            q = "delete from attributes where serial = ? and domain = ?"
+            self.execute(q, (serial, domain))

     
     def attribute_copy(self, source, dest):
         q = ("insert or replace into attributes "
     
     def attribute_copy(self, source, dest):
         q = ("insert or replace into attributes "

-             "select ?, key, value from attributes "
+             "select ?, domain, key, value from attributes "

              "where serial = ?")
         self.execute(q, (dest, source))
     
              "where serial = ?")
         self.execute(q, (dest, source))
     

-    def _parse_filters(self, filterq):
-        preterms = filterq.split(',')
-        included = []
-        excluded = []
-        opers = []
-        match = _regexfilter.match
-        for term in preterms:
-            m = match(term)
-            if m is None:
-                continue
-            neg, key, op, value = m.groups()
-            if neg:
-                excluded.append(key)
-            elif not value:
-                included.append(key)
-            elif op:
-                opers.append((key, op, value))
-        
-        return included, excluded, opers
-    
-    def _construct_filters(self, filterq):
-        if not filterq:
+    def _construct_filters(self, domain, filterq):
+        if not domain or not filterq:

             return None, None
         
         subqlist = []
         append = subqlist.append
             return None, None
         
         subqlist = []
         append = subqlist.append

-        included, excluded, opers = self._parse_filters(filterq)
+        included, excluded, opers = parse_filters(filterq)

         args = []
         
         if included:
         args = []
         
         if included:

-            subq = "a.key in ("
-            subq += ','.join(('?' for x in included)) + ")"
+            subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and "
+            subq += "(" + ' or '.join(('key = ?' for x in included)) + ")"
+            subq += ")"
+            args += [domain]

             args += included
             append(subq)
         
         if excluded:
             args += included
             append(subq)
         
         if excluded:

-            subq = "a.key not in ("
-            subq += ','.join(('?' for x in excluded)) + ")"
+            subq = "not exists (select 1 from attributes where serial = v.serial and domain = ? and "
+            subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")"
+            subq += ")"
+            args += [domain]

             args += excluded
             append(subq)
         
         if opers:
             args += excluded
             append(subq)
         
         if opers:

-            t = (("(key = ? and value %s ?)" % (o,)) for k, o, v in opers)
-            subq = "(" + ' or '.join(t) + ")"

             for k, o, v in opers:
             for k, o, v in opers:

-                args += (k, v)
-            append(subq)
+                subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and "
+                subq += "key = ? and value %s ?" % (o,)
+                subq += ")"
+                args += [domain, k, v]
+                append(subq)

         
         if not subqlist:
             return None, None
         
         
         if not subqlist:
             return None, None
         

-        subq = ' ' + ' and '.join(subqlist)
+        subq = ' and ' + ' and '.join(subqlist)

         
         return subq, args
     
         
         return subq, args
     

@@ -659,7 +647,7 @@ class Node(DBWorker):
         
         return subq, args
     
         
         return subq, args
     

-    def latest_attribute_keys(self, parent, before=inf, except_cluster=0, pathq=[]):
+    def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):

         """Return a list with all keys pairs defined
            for all latest versions under parent that
            do not belong to the cluster.
         """Return a list with all keys pairs defined
            for all latest versions under parent that
            do not belong to the cluster.

@@ -669,15 +657,16 @@ class Node(DBWorker):
         q = ("select distinct a.key "
              "from attributes a, versions v, nodes n "
              "where v.serial = (select max(serial) "
         q = ("select distinct a.key "
              "from attributes a, versions v, nodes n "
              "where v.serial = (select max(serial) "

-                              "from versions "
-                              "where node = v.node and mtime < ?) "
+                               "from versions "
+                               "where node = v.node and mtime < ?) "

              "and v.cluster != ? "
              "and v.node in (select node "
              "and v.cluster != ? "
              "and v.node in (select node "

-                           "from nodes "
-                           "where parent = ?) "
+                            "from nodes "
+                            "where parent = ?) "

              "and a.serial = v.serial "
              "and a.serial = v.serial "

+             "and a.domain = ? "

              "and n.node = v.node")
              "and n.node = v.node")

-        args = (before, except_cluster, parent)
+        args = (before, except_cluster, parent, domain)

         subq, subargs = self._construct_paths(pathq)
         if subq is not None:
             q += subq
         subq, subargs = self._construct_paths(pathq)
         if subq is not None:
             q += subq

@@ -687,7 +676,7 @@ class Node(DBWorker):
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,
     
     def latest_version_list(self, parent, prefix='', delimiter=None,
                             start='', limit=10000, before=inf,

-                            except_cluster=0, pathq=[], filterq=None):
+                            except_cluster=0, pathq=[], domain=None, filterq=[]):

         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.
         """Return a (list of (path, serial) tuples, list of common prefixes)
            for the current versions of the paths with the given parent,
            matching the following criteria.

@@ -721,7 +710,7 @@ class Node(DBWorker):
                    
                    key ?op value
                        the attribute with this key satisfies the value
                    
                    key ?op value
                        the attribute with this key satisfies the value

-                       where ?op is one of ==, != <=, >=, <, >.
+                       where ?op is one of =, != <=, >=, <, >.

            
            The list of common prefixes includes the prefixes
            matching up to the first delimiter after prefix,
            
            The list of common prefixes includes the prefixes
            matching up to the first delimiter after prefix,

@@ -741,15 +730,14 @@ class Node(DBWorker):
         nextling = strnextling(prefix)
         
         q = ("select distinct n.path, v.serial "
         nextling = strnextling(prefix)
         
         q = ("select distinct n.path, v.serial "

-             "from attributes a, versions v, nodes n "
+             "from versions v, nodes n "

              "where v.serial = (select max(serial) "
              "where v.serial = (select max(serial) "

-                              "from versions "
-                              "where node = v.node and mtime < ?) "
+                               "from versions "
+                               "where node = v.node and mtime < ?) "

              "and v.cluster != ? "
              "and v.node in (select node "
              "and v.cluster != ? "
              "and v.node in (select node "

-                           "from nodes "
-                           "where parent = ?) "
-             "and a.serial = v.serial "
+                            "from nodes "
+                            "where parent = ?) "

              "and n.node = v.node "
              "and n.path > ? and n.path < ?")
         args = [before, except_cluster, parent, start, nextling]
              "and n.node = v.node "
              "and n.path > ? and n.path < ?")
         args = [before, except_cluster, parent, start, nextling]

@@ -758,7 +746,7 @@ class Node(DBWorker):
         if subq is not None:
             q += subq
             args += subargs
         if subq is not None:
             q += subq
             args += subargs

-        subq, subargs = self._construct_filters(filterq)
+        subq, subargs = self._construct_filters(domain, filterq)

         if subq is not None:
             q += subq
             args += subargs
         if subq is not None:
             q += subq
             args += subargs

@@ -810,3 +798,15 @@ class Node(DBWorker):
             execute(q, args)
         
         return matches, prefixes
             execute(q, args)
         
         return matches, prefixes

+    
+    def latest_uuid(self, uuid):
+        """Return a (path, serial) tuple, for the latest version of the given uuid."""
+        
+        q = ("select n.path, v.serial "
+             "from versions v, nodes n "
+             "where v.serial = (select max(serial) "
+                               "from versions "
+                               "where uuid = ?) "
+             "and n.node = v.node")
+        self.execute(q, (uuid,))
+        return self.fetchone()