Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-backend / pithos / backends / lib / sqlite / node.py @ 73673127

History | View | Annotate | Download (31.5 kB)

1 2e662088 Antony Chazapis
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2 1f2d1b42 Antony Chazapis
# 
3 1f2d1b42 Antony Chazapis
# Redistribution and use in source and binary forms, with or
4 1f2d1b42 Antony Chazapis
# without modification, are permitted provided that the following
5 1f2d1b42 Antony Chazapis
# conditions are met:
6 1f2d1b42 Antony Chazapis
# 
7 1f2d1b42 Antony Chazapis
#   1. Redistributions of source code must retain the above
8 1f2d1b42 Antony Chazapis
#      copyright notice, this list of conditions and the following
9 1f2d1b42 Antony Chazapis
#      disclaimer.
10 1f2d1b42 Antony Chazapis
# 
11 1f2d1b42 Antony Chazapis
#   2. Redistributions in binary form must reproduce the above
12 1f2d1b42 Antony Chazapis
#      copyright notice, this list of conditions and the following
13 1f2d1b42 Antony Chazapis
#      disclaimer in the documentation and/or other materials
14 1f2d1b42 Antony Chazapis
#      provided with the distribution.
15 1f2d1b42 Antony Chazapis
# 
16 1f2d1b42 Antony Chazapis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 1f2d1b42 Antony Chazapis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 1f2d1b42 Antony Chazapis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 1f2d1b42 Antony Chazapis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 1f2d1b42 Antony Chazapis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 1f2d1b42 Antony Chazapis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 1f2d1b42 Antony Chazapis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 1f2d1b42 Antony Chazapis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 1f2d1b42 Antony Chazapis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 1f2d1b42 Antony Chazapis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 1f2d1b42 Antony Chazapis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 1f2d1b42 Antony Chazapis
# POSSIBILITY OF SUCH DAMAGE.
28 1f2d1b42 Antony Chazapis
# 
29 1f2d1b42 Antony Chazapis
# The views and conclusions contained in the software and
30 1f2d1b42 Antony Chazapis
# documentation are those of the authors and should not be
31 1f2d1b42 Antony Chazapis
# interpreted as representing official policies, either expressed
32 1f2d1b42 Antony Chazapis
# or implied, of GRNET S.A.
33 1f2d1b42 Antony Chazapis
34 1f2d1b42 Antony Chazapis
from time import time
35 1f2d1b42 Antony Chazapis
36 1f2d1b42 Antony Chazapis
from dbworker import DBWorker
37 1f2d1b42 Antony Chazapis
38 6e147ecc Antony Chazapis
from pithos.backends.filter import parse_filters
39 1713c946 Antony Chazapis
40 1f2d1b42 Antony Chazapis
41 1f2d1b42 Antony Chazapis
ROOTNODE  = 0
42 1f2d1b42 Antony Chazapis
43 33b4e4a6 Antony Chazapis
( SERIAL, NODE, HASH, SIZE, TYPE, SOURCE, MTIME, MUSER, UUID, CHECKSUM, CLUSTER ) = range(11)
44 1f2d1b42 Antony Chazapis
45 d57eaad4 Antony Chazapis
( MATCH_PREFIX, MATCH_EXACT ) = range(2)
46 1f2d1b42 Antony Chazapis
47 1f2d1b42 Antony Chazapis
inf = float('inf')
48 1f2d1b42 Antony Chazapis
49 1f2d1b42 Antony Chazapis
50 1f2d1b42 Antony Chazapis
def strnextling(prefix):
51 60b8a083 Antony Chazapis
    """Return the first unicode string
52 1f2d1b42 Antony Chazapis
       greater than but not starting with given prefix.
53 1f2d1b42 Antony Chazapis
       strnextling('hello') -> 'hellp'
54 1f2d1b42 Antony Chazapis
    """
55 1f2d1b42 Antony Chazapis
    if not prefix:
56 1f2d1b42 Antony Chazapis
        ## all strings start with the null string,
57 1f2d1b42 Antony Chazapis
        ## therefore we have to approximate strnextling('')
58 1f2d1b42 Antony Chazapis
        ## with the last unicode character supported by python
59 1f2d1b42 Antony Chazapis
        ## 0x10ffff for wide (32-bit unicode) python builds
60 1f2d1b42 Antony Chazapis
        ## 0x00ffff for narrow (16-bit unicode) python builds
61 1f2d1b42 Antony Chazapis
        ## We will not autodetect. 0xffff is safe enough.
62 1f2d1b42 Antony Chazapis
        return unichr(0xffff)
63 1f2d1b42 Antony Chazapis
    s = prefix[:-1]
64 1f2d1b42 Antony Chazapis
    c = ord(prefix[-1])
65 1f2d1b42 Antony Chazapis
    if c >= 0xffff:
66 1f2d1b42 Antony Chazapis
        raise RuntimeError
67 1f2d1b42 Antony Chazapis
    s += unichr(c+1)
68 1f2d1b42 Antony Chazapis
    return s
69 1f2d1b42 Antony Chazapis
70 1f2d1b42 Antony Chazapis
def strprevling(prefix):
71 60b8a083 Antony Chazapis
    """Return an approximation of the last unicode string
72 1f2d1b42 Antony Chazapis
       less than but not starting with given prefix.
73 1f2d1b42 Antony Chazapis
       strprevling(u'hello') -> u'helln\\xffff'
74 1f2d1b42 Antony Chazapis
    """
75 1f2d1b42 Antony Chazapis
    if not prefix:
76 1f2d1b42 Antony Chazapis
        ## There is no prevling for the null string
77 1f2d1b42 Antony Chazapis
        return prefix
78 1f2d1b42 Antony Chazapis
    s = prefix[:-1]
79 1f2d1b42 Antony Chazapis
    c = ord(prefix[-1])
80 1f2d1b42 Antony Chazapis
    if c > 0:
81 1f2d1b42 Antony Chazapis
        s += unichr(c-1) + unichr(0xffff)
82 1f2d1b42 Antony Chazapis
    return s
83 1f2d1b42 Antony Chazapis
84 1f2d1b42 Antony Chazapis
85 1f2d1b42 Antony Chazapis
_propnames = {
86 1f2d1b42 Antony Chazapis
    'serial'    : 0,
87 1f2d1b42 Antony Chazapis
    'node'      : 1,
88 1c2fc0ff Antony Chazapis
    'hash'      : 2,
89 1c2fc0ff Antony Chazapis
    'size'      : 3,
90 66ce2ca5 Antony Chazapis
    'type'      : 4,
91 66ce2ca5 Antony Chazapis
    'source'    : 5,
92 66ce2ca5 Antony Chazapis
    'mtime'     : 6,
93 66ce2ca5 Antony Chazapis
    'muser'     : 7,
94 66ce2ca5 Antony Chazapis
    'uuid'      : 8,
95 33b4e4a6 Antony Chazapis
    'checksum'  : 9,
96 33b4e4a6 Antony Chazapis
    'cluster'   : 10
97 1f2d1b42 Antony Chazapis
}
98 1f2d1b42 Antony Chazapis
99 1f2d1b42 Antony Chazapis
100 1f2d1b42 Antony Chazapis
class Node(DBWorker):
101 60b8a083 Antony Chazapis
    """Nodes store path organization and have multiple versions.
102 60b8a083 Antony Chazapis
       Versions store object history and have multiple attributes.
103 1f2d1b42 Antony Chazapis
       Attributes store metadata.
104 1f2d1b42 Antony Chazapis
    """
105 1f2d1b42 Antony Chazapis
    
106 60b8a083 Antony Chazapis
    # TODO: Provide an interface for included and excluded clusters.
107 c915d3bf Antony Chazapis
    
108 1f2d1b42 Antony Chazapis
    def __init__(self, **params):
109 62f915a1 Antony Chazapis
        DBWorker.__init__(self, **params)
110 1f2d1b42 Antony Chazapis
        execute = self.execute
111 1f2d1b42 Antony Chazapis
        
112 1f2d1b42 Antony Chazapis
        execute(""" pragma foreign_keys = on """)
113 1f2d1b42 Antony Chazapis
        
114 1f2d1b42 Antony Chazapis
        execute(""" create table if not exists nodes
115 1f2d1b42 Antony Chazapis
                          ( node       integer primary key,
116 62f915a1 Antony Chazapis
                            parent     integer default 0,
117 44ad5860 Antony Chazapis
                            path       text    not null default '',
118 1f2d1b42 Antony Chazapis
                            foreign key (parent)
119 1f2d1b42 Antony Chazapis
                            references nodes(node)
120 1f2d1b42 Antony Chazapis
                            on update cascade
121 5e7485da Antony Chazapis
                            on delete cascade ) """)
122 1f2d1b42 Antony Chazapis
        execute(""" create unique index if not exists idx_nodes_path
123 1f2d1b42 Antony Chazapis
                    on nodes(path) """)
124 1f2d1b42 Antony Chazapis
        
125 5e7485da Antony Chazapis
        execute(""" create table if not exists policy
126 5e7485da Antony Chazapis
                          ( node   integer,
127 5e7485da Antony Chazapis
                            key    text,
128 5e7485da Antony Chazapis
                            value  text,
129 5e7485da Antony Chazapis
                            primary key (node, key)
130 5e7485da Antony Chazapis
                            foreign key (node)
131 5e7485da Antony Chazapis
                            references nodes(node)
132 5e7485da Antony Chazapis
                            on update cascade
133 5e7485da Antony Chazapis
                            on delete cascade ) """)
134 5e7485da Antony Chazapis
        
135 1f2d1b42 Antony Chazapis
        execute(""" create table if not exists statistics
136 62f915a1 Antony Chazapis
                          ( node       integer,
137 1f2d1b42 Antony Chazapis
                            population integer not null default 0,
138 1f2d1b42 Antony Chazapis
                            size       integer not null default 0,
139 1f2d1b42 Antony Chazapis
                            mtime      integer,
140 1f2d1b42 Antony Chazapis
                            cluster    integer not null default 0,
141 1f2d1b42 Antony Chazapis
                            primary key (node, cluster)
142 1f2d1b42 Antony Chazapis
                            foreign key (node)
143 1f2d1b42 Antony Chazapis
                            references nodes(node)
144 1f2d1b42 Antony Chazapis
                            on update cascade
145 5e7485da Antony Chazapis
                            on delete cascade ) """)
146 1f2d1b42 Antony Chazapis
        
147 1f2d1b42 Antony Chazapis
        execute(""" create table if not exists versions
148 1f2d1b42 Antony Chazapis
                          ( serial     integer primary key,
149 62f915a1 Antony Chazapis
                            node       integer,
150 1c2fc0ff Antony Chazapis
                            hash       text,
151 1f2d1b42 Antony Chazapis
                            size       integer not null default 0,
152 66ce2ca5 Antony Chazapis
                            type       text    not null default '',
153 1f2d1b42 Antony Chazapis
                            source     integer,
154 1f2d1b42 Antony Chazapis
                            mtime      integer,
155 c1cdc455 Antony Chazapis
                            muser      text    not null default '',
156 25ae8b75 Antony Chazapis
                            uuid       text    not null default '',
157 33b4e4a6 Antony Chazapis
                            checksum   text    not null default '',
158 1f2d1b42 Antony Chazapis
                            cluster    integer not null default 0,
159 1f2d1b42 Antony Chazapis
                            foreign key (node)
160 1f2d1b42 Antony Chazapis
                            references nodes(node)
161 1f2d1b42 Antony Chazapis
                            on update cascade
162 1f2d1b42 Antony Chazapis
                            on delete cascade ) """)
163 9e35988c Antony Chazapis
        execute(""" create index if not exists idx_versions_node_mtime
164 9e35988c Antony Chazapis
                    on versions(node, mtime) """)
165 25ae8b75 Antony Chazapis
        execute(""" create index if not exists idx_versions_node_uuid
166 25ae8b75 Antony Chazapis
                    on versions(uuid) """)
167 1f2d1b42 Antony Chazapis
        
168 1f2d1b42 Antony Chazapis
        execute(""" create table if not exists attributes
169 1f2d1b42 Antony Chazapis
                          ( serial integer,
170 4819d34f Antony Chazapis
                            domain text,
171 1f2d1b42 Antony Chazapis
                            key    text,
172 1f2d1b42 Antony Chazapis
                            value  text,
173 4819d34f Antony Chazapis
                            primary key (serial, domain, key)
174 1f2d1b42 Antony Chazapis
                            foreign key (serial)
175 1f2d1b42 Antony Chazapis
                            references versions(serial)
176 1f2d1b42 Antony Chazapis
                            on update cascade
177 1f2d1b42 Antony Chazapis
                            on delete cascade ) """)
178 1f2d1b42 Antony Chazapis
        
179 1f2d1b42 Antony Chazapis
        q = "insert or ignore into nodes(node, parent) values (?, ?)"
180 1f2d1b42 Antony Chazapis
        execute(q, (ROOTNODE, ROOTNODE))
181 1f2d1b42 Antony Chazapis
    
182 44ad5860 Antony Chazapis
    def node_create(self, parent, path):
183 44ad5860 Antony Chazapis
        """Create a new node from the given properties.
184 44ad5860 Antony Chazapis
           Return the node identifier of the new node.
185 44ad5860 Antony Chazapis
        """
186 44ad5860 Antony Chazapis
        
187 44ad5860 Antony Chazapis
        q = ("insert into nodes (parent, path) "
188 44ad5860 Antony Chazapis
             "values (?, ?)")
189 44ad5860 Antony Chazapis
        props = (parent, path)
190 44ad5860 Antony Chazapis
        return self.execute(q, props).lastrowid
191 44ad5860 Antony Chazapis
    
192 44ad5860 Antony Chazapis
    def node_lookup(self, path):
193 44ad5860 Antony Chazapis
        """Lookup the current node of the given path.
194 44ad5860 Antony Chazapis
           Return None if the path is not found.
195 44ad5860 Antony Chazapis
        """
196 44ad5860 Antony Chazapis
        
197 c1cdc455 Antony Chazapis
        q = "select node from nodes where path = ?"
198 44ad5860 Antony Chazapis
        self.execute(q, (path,))
199 44ad5860 Antony Chazapis
        r = self.fetchone()
200 44ad5860 Antony Chazapis
        if r is not None:
201 44ad5860 Antony Chazapis
            return r[0]
202 44ad5860 Antony Chazapis
        return None
203 44ad5860 Antony Chazapis
    
204 62f915a1 Antony Chazapis
    def node_get_properties(self, node):
205 62f915a1 Antony Chazapis
        """Return the node's (parent, path).
206 62f915a1 Antony Chazapis
           Return None if the node is not found.
207 62f915a1 Antony Chazapis
        """
208 1f2d1b42 Antony Chazapis
        
209 62f915a1 Antony Chazapis
        q = "select parent, path from nodes where node = ?"
210 c1cdc455 Antony Chazapis
        self.execute(q, (node,))
211 62f915a1 Antony Chazapis
        return self.fetchone()
212 1f2d1b42 Antony Chazapis
    
213 676edf89 Antony Chazapis
    def node_get_versions(self, node, keys=(), propnames=_propnames):
214 676edf89 Antony Chazapis
        """Return the properties of all versions at node.
215 676edf89 Antony Chazapis
           If keys is empty, return all properties in the order
216 33b4e4a6 Antony Chazapis
           (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster).
217 676edf89 Antony Chazapis
        """
218 676edf89 Antony Chazapis
        
219 33b4e4a6 Antony Chazapis
        q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster "
220 676edf89 Antony Chazapis
             "from versions "
221 676edf89 Antony Chazapis
             "where node = ?")
222 60b8a083 Antony Chazapis
        self.execute(q, (node,))
223 676edf89 Antony Chazapis
        r = self.fetchall()
224 676edf89 Antony Chazapis
        if r is None:
225 676edf89 Antony Chazapis
            return r
226 676edf89 Antony Chazapis
        
227 676edf89 Antony Chazapis
        if not keys:
228 676edf89 Antony Chazapis
            return r
229 676edf89 Antony Chazapis
        return [[p[propnames[k]] for k in keys if k in propnames] for p in r]
230 676edf89 Antony Chazapis
    
231 c915d3bf Antony Chazapis
    def node_count_children(self, node):
232 44ad5860 Antony Chazapis
        """Return node's child count."""
233 1f2d1b42 Antony Chazapis
        
234 c915d3bf Antony Chazapis
        q = "select count(node) from nodes where parent = ? and node != 0"
235 44ad5860 Antony Chazapis
        self.execute(q, (node,))
236 62f915a1 Antony Chazapis
        r = self.fetchone()
237 44ad5860 Antony Chazapis
        if r is None:
238 44ad5860 Antony Chazapis
            return 0
239 62f915a1 Antony Chazapis
        return r[0]
240 1f2d1b42 Antony Chazapis
    
241 c915d3bf Antony Chazapis
    def node_purge_children(self, parent, before=inf, cluster=0):
242 c915d3bf Antony Chazapis
        """Delete all versions with the specified
243 c915d3bf Antony Chazapis
           parent and cluster, and return
244 813e42e5 Antony Chazapis
           the hashes and size of versions deleted.
245 c915d3bf Antony Chazapis
           Clears out nodes with no remaining versions.
246 c915d3bf Antony Chazapis
        """
247 c915d3bf Antony Chazapis
        
248 c915d3bf Antony Chazapis
        execute = self.execute
249 c915d3bf Antony Chazapis
        q = ("select count(serial), sum(size) from versions "
250 c915d3bf Antony Chazapis
             "where node in (select node "
251 c915d3bf Antony Chazapis
                            "from nodes "
252 c915d3bf Antony Chazapis
                            "where parent = ?) "
253 c915d3bf Antony Chazapis
             "and cluster = ? "
254 c915d3bf Antony Chazapis
             "and mtime <= ?")
255 c915d3bf Antony Chazapis
        args = (parent, cluster, before)
256 c915d3bf Antony Chazapis
        execute(q, args)
257 c915d3bf Antony Chazapis
        nr, size = self.fetchone()
258 c915d3bf Antony Chazapis
        if not nr:
259 813e42e5 Antony Chazapis
            return (), 0
260 60b8a083 Antony Chazapis
        mtime = time()
261 60b8a083 Antony Chazapis
        self.statistics_update(parent, -nr, -size, mtime, cluster)
262 60b8a083 Antony Chazapis
        self.statistics_update_ancestors(parent, -nr, -size, mtime, cluster)
263 c915d3bf Antony Chazapis
        
264 e1f09a6e Antony Chazapis
        q = ("select hash from versions "
265 c915d3bf Antony Chazapis
             "where node in (select node "
266 c915d3bf Antony Chazapis
                            "from nodes "
267 c915d3bf Antony Chazapis
                            "where parent = ?) "
268 c915d3bf Antony Chazapis
             "and cluster = ? "
269 c915d3bf Antony Chazapis
             "and mtime <= ?")
270 c915d3bf Antony Chazapis
        execute(q, args)
271 e1f09a6e Antony Chazapis
        hashes = [r[0] for r in self.fetchall()]
272 c915d3bf Antony Chazapis
        q = ("delete from versions "
273 c915d3bf Antony Chazapis
             "where node in (select node "
274 c915d3bf Antony Chazapis
                            "from nodes "
275 c915d3bf Antony Chazapis
                            "where parent = ?) "
276 c915d3bf Antony Chazapis
             "and cluster = ? "
277 c915d3bf Antony Chazapis
             "and mtime <= ?")
278 c915d3bf Antony Chazapis
        execute(q, args)
279 62f915a1 Antony Chazapis
        q = ("delete from nodes "
280 62f915a1 Antony Chazapis
             "where node in (select node from nodes n "
281 62f915a1 Antony Chazapis
                            "where (select count(serial) "
282 62f915a1 Antony Chazapis
                                   "from versions "
283 62f915a1 Antony Chazapis
                                   "where node = n.node) = 0 "
284 62f915a1 Antony Chazapis
                            "and parent = ?)")
285 62f915a1 Antony Chazapis
        execute(q, (parent,))
286 813e42e5 Antony Chazapis
        return hashes, size
287 c915d3bf Antony Chazapis
    
288 c915d3bf Antony Chazapis
    def node_purge(self, node, before=inf, cluster=0):
289 c915d3bf Antony Chazapis
        """Delete all versions with the specified
290 c915d3bf Antony Chazapis
           node and cluster, and return
291 813e42e5 Antony Chazapis
           the hashes and size of versions deleted.
292 c915d3bf Antony Chazapis
           Clears out the node if it has no remaining versions.
293 c915d3bf Antony Chazapis
        """
294 c915d3bf Antony Chazapis
        
295 c915d3bf Antony Chazapis
        execute = self.execute
296 c915d3bf Antony Chazapis
        q = ("select count(serial), sum(size) from versions "
297 c915d3bf Antony Chazapis
             "where node = ? "
298 c915d3bf Antony Chazapis
             "and cluster = ? "
299 c915d3bf Antony Chazapis
             "and mtime <= ?")
300 c915d3bf Antony Chazapis
        args = (node, cluster, before)
301 c915d3bf Antony Chazapis
        execute(q, args)
302 c915d3bf Antony Chazapis
        nr, size = self.fetchone()
303 c915d3bf Antony Chazapis
        if not nr:
304 813e42e5 Antony Chazapis
            return (), 0
305 60b8a083 Antony Chazapis
        mtime = time()
306 60b8a083 Antony Chazapis
        self.statistics_update_ancestors(node, -nr, -size, mtime, cluster)
307 c915d3bf Antony Chazapis
        
308 e1f09a6e Antony Chazapis
        q = ("select hash from versions "
309 c915d3bf Antony Chazapis
             "where node = ? "
310 c915d3bf Antony Chazapis
             "and cluster = ? "
311 c915d3bf Antony Chazapis
             "and mtime <= ?")
312 c915d3bf Antony Chazapis
        execute(q, args)
313 e1f09a6e Antony Chazapis
        hashes = [r[0] for r in self.fetchall()]
314 c915d3bf Antony Chazapis
        q = ("delete from versions "
315 c915d3bf Antony Chazapis
             "where node = ? "
316 c915d3bf Antony Chazapis
             "and cluster = ? "
317 c915d3bf Antony Chazapis
             "and mtime <= ?")
318 c915d3bf Antony Chazapis
        execute(q, args)
319 62f915a1 Antony Chazapis
        q = ("delete from nodes "
320 62f915a1 Antony Chazapis
             "where node in (select node from nodes n "
321 62f915a1 Antony Chazapis
                            "where (select count(serial) "
322 62f915a1 Antony Chazapis
                                   "from versions "
323 62f915a1 Antony Chazapis
                                   "where node = n.node) = 0 "
324 62f915a1 Antony Chazapis
                            "and node = ?)")
325 62f915a1 Antony Chazapis
        execute(q, (node,))
326 813e42e5 Antony Chazapis
        return hashes, size
327 c915d3bf Antony Chazapis
    
328 c915d3bf Antony Chazapis
    def node_remove(self, node):
329 c915d3bf Antony Chazapis
        """Remove the node specified.
330 c915d3bf Antony Chazapis
           Return false if the node has children or is not found.
331 c915d3bf Antony Chazapis
        """
332 c915d3bf Antony Chazapis
        
333 c1cdc455 Antony Chazapis
        if self.node_count_children(node):
334 c915d3bf Antony Chazapis
            return False
335 c915d3bf Antony Chazapis
        
336 c915d3bf Antony Chazapis
        mtime = time()
337 62f915a1 Antony Chazapis
        q = ("select count(serial), sum(size), cluster "
338 c1cdc455 Antony Chazapis
             "from versions "
339 c1cdc455 Antony Chazapis
             "where node = ? "
340 c1cdc455 Antony Chazapis
             "group by cluster")
341 c915d3bf Antony Chazapis
        self.execute(q, (node,))
342 c915d3bf Antony Chazapis
        for population, size, cluster in self.fetchall():
343 c1cdc455 Antony Chazapis
            self.statistics_update_ancestors(node, -population, -size, mtime, cluster)
344 c915d3bf Antony Chazapis
        
345 c915d3bf Antony Chazapis
        q = "delete from nodes where node = ?"
346 c915d3bf Antony Chazapis
        self.execute(q, (node,))
347 c915d3bf Antony Chazapis
        return True
348 c915d3bf Antony Chazapis
    
349 5e7485da Antony Chazapis
    def policy_get(self, node):
350 5e7485da Antony Chazapis
        q = "select key, value from policy where node = ?"
351 5e7485da Antony Chazapis
        self.execute(q, (node,))
352 5e7485da Antony Chazapis
        return dict(self.fetchall())
353 5e7485da Antony Chazapis
    
354 5e7485da Antony Chazapis
    def policy_set(self, node, policy):
355 5e7485da Antony Chazapis
        q = "insert or replace into policy (node, key, value) values (?, ?, ?)"
356 5e7485da Antony Chazapis
        self.executemany(q, ((node, k, v) for k, v in policy.iteritems()))
357 5e7485da Antony Chazapis
    
358 c1cdc455 Antony Chazapis
    def statistics_get(self, node, cluster=0):
359 c1cdc455 Antony Chazapis
        """Return population, total size and last mtime
360 c1cdc455 Antony Chazapis
           for all versions under node that belong to the cluster.
361 c1cdc455 Antony Chazapis
        """
362 c1cdc455 Antony Chazapis
        
363 62f915a1 Antony Chazapis
        q = ("select population, size, mtime from statistics "
364 c1cdc455 Antony Chazapis
             "where node = ? and cluster = ?")
365 c1cdc455 Antony Chazapis
        self.execute(q, (node, cluster))
366 62f915a1 Antony Chazapis
        return self.fetchone()
367 c1cdc455 Antony Chazapis
    
368 c1cdc455 Antony Chazapis
    def statistics_update(self, node, population, size, mtime, cluster=0):
369 c1cdc455 Antony Chazapis
        """Update the statistics of the given node.
370 c1cdc455 Antony Chazapis
           Statistics keep track the population, total
371 c1cdc455 Antony Chazapis
           size of objects and mtime in the node's namespace.
372 c1cdc455 Antony Chazapis
           May be zero or positive or negative numbers.
373 c1cdc455 Antony Chazapis
        """
374 c1cdc455 Antony Chazapis
        
375 62f915a1 Antony Chazapis
        qs = ("select population, size from statistics "
376 c1cdc455 Antony Chazapis
              "where node = ? and cluster = ?")
377 c1cdc455 Antony Chazapis
        qu = ("insert or replace into statistics (node, population, size, mtime, cluster) "
378 c1cdc455 Antony Chazapis
              "values (?, ?, ?, ?, ?)")
379 c1cdc455 Antony Chazapis
        self.execute(qs, (node, cluster))
380 c1cdc455 Antony Chazapis
        r = self.fetchone()
381 c1cdc455 Antony Chazapis
        if r is None:
382 c1cdc455 Antony Chazapis
            prepopulation, presize = (0, 0)
383 c1cdc455 Antony Chazapis
        else:
384 c1cdc455 Antony Chazapis
            prepopulation, presize = r
385 c1cdc455 Antony Chazapis
        population += prepopulation
386 c1cdc455 Antony Chazapis
        size += presize
387 c1cdc455 Antony Chazapis
        self.execute(qu, (node, population, size, mtime, cluster))
388 c1cdc455 Antony Chazapis
    
389 c1cdc455 Antony Chazapis
    def statistics_update_ancestors(self, node, population, size, mtime, cluster=0):
390 c1cdc455 Antony Chazapis
        """Update the statistics of the given node's parent.
391 c1cdc455 Antony Chazapis
           Then recursively update all parents up to the root.
392 c1cdc455 Antony Chazapis
           Population is not recursive.
393 c1cdc455 Antony Chazapis
        """
394 c1cdc455 Antony Chazapis
        
395 c1cdc455 Antony Chazapis
        while True:
396 c1cdc455 Antony Chazapis
            if node == 0:
397 c1cdc455 Antony Chazapis
                break
398 62f915a1 Antony Chazapis
            props = self.node_get_properties(node)
399 62f915a1 Antony Chazapis
            if props is None:
400 c1cdc455 Antony Chazapis
                break
401 62f915a1 Antony Chazapis
            parent, path = props
402 62f915a1 Antony Chazapis
            self.statistics_update(parent, population, size, mtime, cluster)
403 62f915a1 Antony Chazapis
            node = parent
404 c1cdc455 Antony Chazapis
            population = 0 # Population isn't recursive
405 1f2d1b42 Antony Chazapis
    
406 62f915a1 Antony Chazapis
    def statistics_latest(self, node, before=inf, except_cluster=0):
407 62f915a1 Antony Chazapis
        """Return population, total size and last mtime
408 62f915a1 Antony Chazapis
           for all latest versions under node that
409 62f915a1 Antony Chazapis
           do not belong to the cluster.
410 62f915a1 Antony Chazapis
        """
411 62f915a1 Antony Chazapis
        
412 676edf89 Antony Chazapis
        execute = self.execute
413 676edf89 Antony Chazapis
        fetchone = self.fetchone
414 676edf89 Antony Chazapis
        
415 62f915a1 Antony Chazapis
        # The node.
416 62f915a1 Antony Chazapis
        props = self.node_get_properties(node)
417 62f915a1 Antony Chazapis
        if props is None:
418 62f915a1 Antony Chazapis
            return None
419 62f915a1 Antony Chazapis
        parent, path = props
420 62f915a1 Antony Chazapis
        
421 62f915a1 Antony Chazapis
        # The latest version.
422 33b4e4a6 Antony Chazapis
        q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster "
423 62f915a1 Antony Chazapis
             "from versions "
424 62f915a1 Antony Chazapis
             "where serial = (select max(serial) "
425 62f915a1 Antony Chazapis
                             "from versions "
426 62f915a1 Antony Chazapis
                             "where node = ? and mtime < ?) "
427 62f915a1 Antony Chazapis
             "and cluster != ?")
428 676edf89 Antony Chazapis
        execute(q, (node, before, except_cluster))
429 676edf89 Antony Chazapis
        props = fetchone()
430 62f915a1 Antony Chazapis
        if props is None:
431 62f915a1 Antony Chazapis
            return None
432 62f915a1 Antony Chazapis
        mtime = props[MTIME]
433 62f915a1 Antony Chazapis
        
434 62f915a1 Antony Chazapis
        # First level, just under node (get population).
435 62f915a1 Antony Chazapis
        q = ("select count(serial), sum(size), max(mtime) "
436 62f915a1 Antony Chazapis
             "from versions v "
437 62f915a1 Antony Chazapis
             "where serial = (select max(serial) "
438 62f915a1 Antony Chazapis
                             "from versions "
439 62f915a1 Antony Chazapis
                             "where node = v.node and mtime < ?) "
440 62f915a1 Antony Chazapis
             "and cluster != ? "
441 62f915a1 Antony Chazapis
             "and node in (select node "
442 62f915a1 Antony Chazapis
                          "from nodes "
443 62f915a1 Antony Chazapis
                          "where parent = ?)")
444 676edf89 Antony Chazapis
        execute(q, (before, except_cluster, node))
445 62f915a1 Antony Chazapis
        r = fetchone()
446 62f915a1 Antony Chazapis
        if r is None:
447 62f915a1 Antony Chazapis
            return None
448 62f915a1 Antony Chazapis
        count = r[0]
449 62f915a1 Antony Chazapis
        mtime = max(mtime, r[2])
450 62f915a1 Antony Chazapis
        if count == 0:
451 62f915a1 Antony Chazapis
            return (0, 0, mtime)
452 62f915a1 Antony Chazapis
        
453 62f915a1 Antony Chazapis
        # All children (get size and mtime).
454 92da0e5a Antony Chazapis
        # This is why the full path is stored.
455 62f915a1 Antony Chazapis
        q = ("select count(serial), sum(size), max(mtime) "
456 62f915a1 Antony Chazapis
             "from versions v "
457 62f915a1 Antony Chazapis
             "where serial = (select max(serial) "
458 62f915a1 Antony Chazapis
                             "from versions "
459 62f915a1 Antony Chazapis
                             "where node = v.node and mtime < ?) "
460 62f915a1 Antony Chazapis
             "and cluster != ? "
461 62f915a1 Antony Chazapis
             "and node in (select node "
462 62f915a1 Antony Chazapis
                          "from nodes "
463 7759260d Antony Chazapis
                          "where path like ? escape '\\')")
464 7759260d Antony Chazapis
        execute(q, (before, except_cluster, self.escape_like(path) + '%'))
465 62f915a1 Antony Chazapis
        r = fetchone()
466 62f915a1 Antony Chazapis
        if r is None:
467 62f915a1 Antony Chazapis
            return None
468 62f915a1 Antony Chazapis
        size = r[1] - props[SIZE]
469 62f915a1 Antony Chazapis
        mtime = max(mtime, r[2])
470 62f915a1 Antony Chazapis
        return (count, size, mtime)
471 62f915a1 Antony Chazapis
    
472 33b4e4a6 Antony Chazapis
    def version_create(self, node, hash, size, type, source, muser, uuid, checksum, cluster=0):
473 44ad5860 Antony Chazapis
        """Create a new version from the given properties.
474 44ad5860 Antony Chazapis
           Return the (serial, mtime) of the new version.
475 44ad5860 Antony Chazapis
        """
476 44ad5860 Antony Chazapis
        
477 33b4e4a6 Antony Chazapis
        q = ("insert into versions (node, hash, size, type, source, mtime, muser, uuid, checksum, cluster) "
478 33b4e4a6 Antony Chazapis
             "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
479 44ad5860 Antony Chazapis
        mtime = time()
480 33b4e4a6 Antony Chazapis
        props = (node, hash, size, type, source, mtime, muser, uuid, checksum, cluster)
481 44ad5860 Antony Chazapis
        serial = self.execute(q, props).lastrowid
482 c1cdc455 Antony Chazapis
        self.statistics_update_ancestors(node, 1, size, mtime, cluster)
483 44ad5860 Antony Chazapis
        return serial, mtime
484 44ad5860 Antony Chazapis
    
485 44ad5860 Antony Chazapis
    def version_lookup(self, node, before=inf, cluster=0):
486 44ad5860 Antony Chazapis
        """Lookup the current version of the given node.
487 44ad5860 Antony Chazapis
           Return a list with its properties:
488 33b4e4a6 Antony Chazapis
           (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster)
489 44ad5860 Antony Chazapis
           or None if the current version is not found in the given cluster.
490 44ad5860 Antony Chazapis
        """
491 44ad5860 Antony Chazapis
        
492 33b4e4a6 Antony Chazapis
        q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster "
493 44ad5860 Antony Chazapis
             "from versions "
494 44ad5860 Antony Chazapis
             "where serial = (select max(serial) "
495 44ad5860 Antony Chazapis
                             "from versions "
496 44ad5860 Antony Chazapis
                             "where node = ? and mtime < ?) "
497 44ad5860 Antony Chazapis
             "and cluster = ?")
498 44ad5860 Antony Chazapis
        self.execute(q, (node, before, cluster))
499 44ad5860 Antony Chazapis
        props = self.fetchone()
500 44ad5860 Antony Chazapis
        if props is not None:
501 44ad5860 Antony Chazapis
            return props
502 44ad5860 Antony Chazapis
        return None
503 44ad5860 Antony Chazapis
    
504 1f2d1b42 Antony Chazapis
    def version_get_properties(self, serial, keys=(), propnames=_propnames):
505 1f2d1b42 Antony Chazapis
        """Return a sequence of values for the properties of
506 1f2d1b42 Antony Chazapis
           the version specified by serial and the keys, in the order given.
507 1f2d1b42 Antony Chazapis
           If keys is empty, return all properties in the order
508 33b4e4a6 Antony Chazapis
           (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster).
509 1f2d1b42 Antony Chazapis
        """
510 1f2d1b42 Antony Chazapis
        
511 33b4e4a6 Antony Chazapis
        q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster "
512 62f915a1 Antony Chazapis
             "from versions "
513 1f2d1b42 Antony Chazapis
             "where serial = ?")
514 1f2d1b42 Antony Chazapis
        self.execute(q, (serial,))
515 1f2d1b42 Antony Chazapis
        r = self.fetchone()
516 1f2d1b42 Antony Chazapis
        if r is None:
517 1f2d1b42 Antony Chazapis
            return r
518 1f2d1b42 Antony Chazapis
        
519 1f2d1b42 Antony Chazapis
        if not keys:
520 1f2d1b42 Antony Chazapis
            return r
521 1f2d1b42 Antony Chazapis
        return [r[propnames[k]] for k in keys if k in propnames]
522 1f2d1b42 Antony Chazapis
    
523 33b4e4a6 Antony Chazapis
    def version_put_property(self, serial, key, value):
524 33b4e4a6 Antony Chazapis
        """Set value for the property of version specified by key."""
525 33b4e4a6 Antony Chazapis
        
526 33b4e4a6 Antony Chazapis
        if key not in _propnames:
527 33b4e4a6 Antony Chazapis
            return
528 33b4e4a6 Antony Chazapis
        q = "update versions set %s = ? where serial = ?" % key
529 33b4e4a6 Antony Chazapis
        self.execute(q, (value, serial))
530 33b4e4a6 Antony Chazapis
    
531 44ad5860 Antony Chazapis
    def version_recluster(self, serial, cluster):
532 44ad5860 Antony Chazapis
        """Move the version into another cluster."""
533 1f2d1b42 Antony Chazapis
        
534 62f915a1 Antony Chazapis
        props = self.version_get_properties(serial)
535 62f915a1 Antony Chazapis
        if not props:
536 62f915a1 Antony Chazapis
            return
537 44ad5860 Antony Chazapis
        node = props[NODE]
538 44ad5860 Antony Chazapis
        size = props[SIZE]
539 44ad5860 Antony Chazapis
        oldcluster = props[CLUSTER]
540 44ad5860 Antony Chazapis
        if cluster == oldcluster:
541 44ad5860 Antony Chazapis
            return
542 1f2d1b42 Antony Chazapis
        
543 62f915a1 Antony Chazapis
        mtime = time()
544 c1cdc455 Antony Chazapis
        self.statistics_update_ancestors(node, -1, -size, mtime, oldcluster)
545 c1cdc455 Antony Chazapis
        self.statistics_update_ancestors(node, 1, size, mtime, cluster)
546 c1cdc455 Antony Chazapis
        
547 62f915a1 Antony Chazapis
        q = "update versions set cluster = ? where serial = ?"
548 c915d3bf Antony Chazapis
        self.execute(q, (cluster, serial))
549 1f2d1b42 Antony Chazapis
    
550 62f915a1 Antony Chazapis
    def version_remove(self, serial):
551 62f915a1 Antony Chazapis
        """Remove the serial specified."""
552 1f2d1b42 Antony Chazapis
        
553 5161c672 Antony Chazapis
        props = self.version_get_properties(serial)
554 62f915a1 Antony Chazapis
        if not props:
555 62f915a1 Antony Chazapis
            return
556 62f915a1 Antony Chazapis
        node = props[NODE]
557 5161c672 Antony Chazapis
        hash = props[HASH]
558 62f915a1 Antony Chazapis
        size = props[SIZE]
559 62f915a1 Antony Chazapis
        cluster = props[CLUSTER]
560 62f915a1 Antony Chazapis
        
561 62f915a1 Antony Chazapis
        mtime = time()
562 62f915a1 Antony Chazapis
        self.statistics_update_ancestors(node, -1, -size, mtime, cluster)
563 62f915a1 Antony Chazapis
        
564 62f915a1 Antony Chazapis
        q = "delete from versions where serial = ?"
565 62f915a1 Antony Chazapis
        self.execute(q, (serial,))
566 813e42e5 Antony Chazapis
        return hash, size
567 1f2d1b42 Antony Chazapis
    
568 4819d34f Antony Chazapis
    def attribute_get(self, serial, domain, keys=()):
569 44ad5860 Antony Chazapis
        """Return a list of (key, value) pairs of the version specified by serial.
570 1f2d1b42 Antony Chazapis
           If keys is empty, return all attributes.
571 1f2d1b42 Antony Chazapis
           Othwerise, return only those specified.
572 1f2d1b42 Antony Chazapis
        """
573 1f2d1b42 Antony Chazapis
        
574 1f2d1b42 Antony Chazapis
        execute = self.execute
575 1f2d1b42 Antony Chazapis
        if keys:
576 1f2d1b42 Antony Chazapis
            marks = ','.join('?' for k in keys)
577 1f2d1b42 Antony Chazapis
            q = ("select key, value from attributes "
578 4819d34f Antony Chazapis
                 "where key in (%s) and serial = ? and domain = ?" % (marks,))
579 4819d34f Antony Chazapis
            execute(q, keys + (serial, domain))
580 1f2d1b42 Antony Chazapis
        else:
581 059857e2 Antony Chazapis
            q = "select key, value from attributes where serial = ? and domain = ?"
582 059857e2 Antony Chazapis
            execute(q, (serial, domain))
583 1f2d1b42 Antony Chazapis
        return self.fetchall()
584 1f2d1b42 Antony Chazapis
    
585 4819d34f Antony Chazapis
    def attribute_set(self, serial, domain, items):
586 44ad5860 Antony Chazapis
        """Set the attributes of the version specified by serial.
587 1f2d1b42 Antony Chazapis
           Receive attributes as an iterable of (key, value) pairs.
588 1f2d1b42 Antony Chazapis
        """
589 1f2d1b42 Antony Chazapis
        
590 4819d34f Antony Chazapis
        q = ("insert or replace into attributes (serial, domain, key, value) "
591 4819d34f Antony Chazapis
             "values (?, ?, ?, ?)")
592 4819d34f Antony Chazapis
        self.executemany(q, ((serial, domain, k, v) for k, v in items))
593 1f2d1b42 Antony Chazapis
    
594 4819d34f Antony Chazapis
    def attribute_del(self, serial, domain, keys=()):
595 44ad5860 Antony Chazapis
        """Delete attributes of the version specified by serial.
596 1f2d1b42 Antony Chazapis
           If keys is empty, delete all attributes.
597 1f2d1b42 Antony Chazapis
           Otherwise delete those specified.
598 1f2d1b42 Antony Chazapis
        """
599 1f2d1b42 Antony Chazapis
        
600 1f2d1b42 Antony Chazapis
        if keys:
601 4819d34f Antony Chazapis
            q = "delete from attributes where serial = ? and domain = ? and key = ?"
602 4819d34f Antony Chazapis
            self.executemany(q, ((serial, domain, key) for key in keys))
603 1f2d1b42 Antony Chazapis
        else:
604 4819d34f Antony Chazapis
            q = "delete from attributes where serial = ? and domain = ?"
605 4819d34f Antony Chazapis
            self.execute(q, (serial, domain))
606 1f2d1b42 Antony Chazapis
    
607 44ad5860 Antony Chazapis
    def attribute_copy(self, source, dest):
608 1f2d1b42 Antony Chazapis
        q = ("insert or replace into attributes "
609 4819d34f Antony Chazapis
             "select ?, domain, key, value from attributes "
610 1f2d1b42 Antony Chazapis
             "where serial = ?")
611 1f2d1b42 Antony Chazapis
        self.execute(q, (dest, source))
612 676edf89 Antony Chazapis
    
613 4819d34f Antony Chazapis
    def _construct_filters(self, domain, filterq):
614 4819d34f Antony Chazapis
        if not domain or not filterq:
615 60b8a083 Antony Chazapis
            return None, None
616 60b8a083 Antony Chazapis
        
617 95d47e1a Antony Chazapis
        subqlist = []
618 95d47e1a Antony Chazapis
        append = subqlist.append
619 4819d34f Antony Chazapis
        included, excluded, opers = parse_filters(filterq)
620 95d47e1a Antony Chazapis
        args = []
621 95d47e1a Antony Chazapis
        
622 95d47e1a Antony Chazapis
        if included:
623 c7c0790f chazapis
            subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and "
624 c7c0790f chazapis
            subq += "(" + ' or '.join(('key = ?' for x in included)) + ")"
625 c7c0790f chazapis
            subq += ")"
626 c7c0790f chazapis
            args += [domain]
627 95d47e1a Antony Chazapis
            args += included
628 95d47e1a Antony Chazapis
            append(subq)
629 95d47e1a Antony Chazapis
        
630 95d47e1a Antony Chazapis
        if excluded:
631 c7c0790f chazapis
            subq = "not exists (select 1 from attributes where serial = v.serial and domain = ? and "
632 c7c0790f chazapis
            subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")"
633 c7c0790f chazapis
            subq += ")"
634 c7c0790f chazapis
            args += [domain]
635 95d47e1a Antony Chazapis
            args += excluded
636 95d47e1a Antony Chazapis
            append(subq)
637 95d47e1a Antony Chazapis
        
638 95d47e1a Antony Chazapis
        if opers:
639 95d47e1a Antony Chazapis
            for k, o, v in opers:
640 6b2f11b4 Antony Chazapis
                subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and "
641 6b2f11b4 Antony Chazapis
                subq += "key = ? and value %s ?" % (o,)
642 6b2f11b4 Antony Chazapis
                subq += ")"
643 6b2f11b4 Antony Chazapis
                args += [domain, k, v]
644 6b2f11b4 Antony Chazapis
                append(subq)
645 95d47e1a Antony Chazapis
        
646 95d47e1a Antony Chazapis
        if not subqlist:
647 95d47e1a Antony Chazapis
            return None, None
648 95d47e1a Antony Chazapis
        
649 c7c0790f chazapis
        subq = ' and ' + ' and '.join(subqlist)
650 60b8a083 Antony Chazapis
        
651 60b8a083 Antony Chazapis
        return subq, args
652 60b8a083 Antony Chazapis
    
653 60b8a083 Antony Chazapis
    def _construct_paths(self, pathq):
654 60b8a083 Antony Chazapis
        if not pathq:
655 60b8a083 Antony Chazapis
            return None, None
656 60b8a083 Antony Chazapis
        
657 d57eaad4 Antony Chazapis
        subqlist = []
658 d57eaad4 Antony Chazapis
        args = []
659 d57eaad4 Antony Chazapis
        print pathq
660 d57eaad4 Antony Chazapis
        for path, match in pathq:
661 d57eaad4 Antony Chazapis
            if match == MATCH_PREFIX:
662 d57eaad4 Antony Chazapis
                subqlist.append("n.path like ? escape '\\'")
663 d57eaad4 Antony Chazapis
                args.append(self.escape_like(path) + '%')
664 d57eaad4 Antony Chazapis
            elif match == MATCH_EXACT:
665 d57eaad4 Antony Chazapis
                subqlist.append("n.path = ?")
666 d57eaad4 Antony Chazapis
                args.append(path)
667 d57eaad4 Antony Chazapis
        
668 d57eaad4 Antony Chazapis
        subq = ' and (' + ' or '.join(subqlist) + ')'
669 d57eaad4 Antony Chazapis
        args = tuple(args)
670 60b8a083 Antony Chazapis
        
671 60b8a083 Antony Chazapis
        return subq, args
672 60b8a083 Antony Chazapis
    
673 7ff57991 Antony Chazapis
    def _construct_size(self, sizeq):
674 7ff57991 Antony Chazapis
        if not sizeq or len(sizeq) != 2:
675 7ff57991 Antony Chazapis
            return None, None
676 7ff57991 Antony Chazapis
        
677 7ff57991 Antony Chazapis
        subq = ''
678 7ff57991 Antony Chazapis
        args = []
679 7ff57991 Antony Chazapis
        if sizeq[0]:
680 7ff57991 Antony Chazapis
            subq += " and v.size >= ?"
681 7ff57991 Antony Chazapis
            args += [sizeq[0]]
682 7ff57991 Antony Chazapis
        if sizeq[1]:
683 7ff57991 Antony Chazapis
            subq += " and v.size < ?"
684 7ff57991 Antony Chazapis
            args += [sizeq[1]]
685 7ff57991 Antony Chazapis
        
686 7ff57991 Antony Chazapis
        return subq, args
687 7ff57991 Antony Chazapis
    
688 4819d34f Antony Chazapis
    def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):
689 676edf89 Antony Chazapis
        """Return a list with all keys pairs defined
690 676edf89 Antony Chazapis
           for all latest versions under parent that
691 676edf89 Antony Chazapis
           do not belong to the cluster.
692 676edf89 Antony Chazapis
        """
693 676edf89 Antony Chazapis
        
694 676edf89 Antony Chazapis
        # TODO: Use another table to store before=inf results.
695 60b8a083 Antony Chazapis
        q = ("select distinct a.key "
696 676edf89 Antony Chazapis
             "from attributes a, versions v, nodes n "
697 676edf89 Antony Chazapis
             "where v.serial = (select max(serial) "
698 37bee317 Antony Chazapis
                               "from versions "
699 37bee317 Antony Chazapis
                               "where node = v.node and mtime < ?) "
700 676edf89 Antony Chazapis
             "and v.cluster != ? "
701 676edf89 Antony Chazapis
             "and v.node in (select node "
702 37bee317 Antony Chazapis
                            "from nodes "
703 37bee317 Antony Chazapis
                            "where parent = ?) "
704 676edf89 Antony Chazapis
             "and a.serial = v.serial "
705 4819d34f Antony Chazapis
             "and a.domain = ? "
706 676edf89 Antony Chazapis
             "and n.node = v.node")
707 4819d34f Antony Chazapis
        args = (before, except_cluster, parent, domain)
708 60b8a083 Antony Chazapis
        subq, subargs = self._construct_paths(pathq)
709 60b8a083 Antony Chazapis
        if subq is not None:
710 60b8a083 Antony Chazapis
            q += subq
711 60b8a083 Antony Chazapis
            args += subargs
712 676edf89 Antony Chazapis
        self.execute(q, args)
713 676edf89 Antony Chazapis
        return [r[0] for r in self.fetchall()]
714 60b8a083 Antony Chazapis
    
715 60b8a083 Antony Chazapis
    def latest_version_list(self, parent, prefix='', delimiter=None,
716 60b8a083 Antony Chazapis
                            start='', limit=10000, before=inf,
717 371d907a Antony Chazapis
                            except_cluster=0, pathq=[], domain=None,
718 371d907a Antony Chazapis
                            filterq=[], sizeq=None, all_props=False):
719 60b8a083 Antony Chazapis
        """Return a (list of (path, serial) tuples, list of common prefixes)
720 60b8a083 Antony Chazapis
           for the current versions of the paths with the given parent,
721 60b8a083 Antony Chazapis
           matching the following criteria.
722 60b8a083 Antony Chazapis
           
723 60b8a083 Antony Chazapis
           The property tuple for a version is returned if all
724 60b8a083 Antony Chazapis
           of these conditions are true:
725 60b8a083 Antony Chazapis
                
726 60b8a083 Antony Chazapis
                a. parent matches
727 60b8a083 Antony Chazapis
                
728 60b8a083 Antony Chazapis
                b. path > start
729 60b8a083 Antony Chazapis
                
730 60b8a083 Antony Chazapis
                c. path starts with prefix (and paths in pathq)
731 60b8a083 Antony Chazapis
                
732 60b8a083 Antony Chazapis
                d. version is the max up to before
733 60b8a083 Antony Chazapis
                
734 60b8a083 Antony Chazapis
                e. version is not in cluster
735 60b8a083 Antony Chazapis
                
736 60b8a083 Antony Chazapis
                f. the path does not have the delimiter occuring
737 60b8a083 Antony Chazapis
                   after the prefix, or ends with the delimiter
738 60b8a083 Antony Chazapis
                
739 60b8a083 Antony Chazapis
                g. serial matches the attribute filter query.
740 60b8a083 Antony Chazapis
                   
741 60b8a083 Antony Chazapis
                   A filter query is a comma-separated list of
742 60b8a083 Antony Chazapis
                   terms in one of these three forms:
743 60b8a083 Antony Chazapis
                   
744 60b8a083 Antony Chazapis
                   key
745 60b8a083 Antony Chazapis
                       an attribute with this key must exist
746 60b8a083 Antony Chazapis
                   
747 60b8a083 Antony Chazapis
                   !key
748 60b8a083 Antony Chazapis
                       an attribute with this key must not exist
749 60b8a083 Antony Chazapis
                   
750 60b8a083 Antony Chazapis
                   key ?op value
751 60b8a083 Antony Chazapis
                       the attribute with this key satisfies the value
752 1713c946 Antony Chazapis
                       where ?op is one of =, != <=, >=, <, >.
753 7ff57991 Antony Chazapis
                
754 7ff57991 Antony Chazapis
                h. the size is in the range set by sizeq
755 60b8a083 Antony Chazapis
           
756 60b8a083 Antony Chazapis
           The list of common prefixes includes the prefixes
757 60b8a083 Antony Chazapis
           matching up to the first delimiter after prefix,
758 60b8a083 Antony Chazapis
           and are reported only once, as "virtual directories".
759 60b8a083 Antony Chazapis
           The delimiter is included in the prefixes.
760 60b8a083 Antony Chazapis
           
761 60b8a083 Antony Chazapis
           If arguments are None, then the corresponding matching rule
762 60b8a083 Antony Chazapis
           will always match.
763 60b8a083 Antony Chazapis
           
764 60b8a083 Antony Chazapis
           Limit applies to the first list of tuples returned.
765 371d907a Antony Chazapis
           
766 371d907a Antony Chazapis
           If all_props is True, return all properties after path, not just serial.
767 60b8a083 Antony Chazapis
        """
768 60b8a083 Antony Chazapis
        
769 60b8a083 Antony Chazapis
        execute = self.execute
770 60b8a083 Antony Chazapis
        
771 60b8a083 Antony Chazapis
        if not start or start < prefix:
772 60b8a083 Antony Chazapis
            start = strprevling(prefix)
773 60b8a083 Antony Chazapis
        nextling = strnextling(prefix)
774 60b8a083 Antony Chazapis
        
775 371d907a Antony Chazapis
        q = ("select distinct n.path, %s "
776 c7c0790f chazapis
             "from versions v, nodes n "
777 60b8a083 Antony Chazapis
             "where v.serial = (select max(serial) "
778 37bee317 Antony Chazapis
                               "from versions "
779 37bee317 Antony Chazapis
                               "where node = v.node and mtime < ?) "
780 60b8a083 Antony Chazapis
             "and v.cluster != ? "
781 60b8a083 Antony Chazapis
             "and v.node in (select node "
782 37bee317 Antony Chazapis
                            "from nodes "
783 37bee317 Antony Chazapis
                            "where parent = ?) "
784 60b8a083 Antony Chazapis
             "and n.node = v.node "
785 60b8a083 Antony Chazapis
             "and n.path > ? and n.path < ?")
786 371d907a Antony Chazapis
        if not all_props:
787 371d907a Antony Chazapis
            q = q % "v.serial"
788 371d907a Antony Chazapis
        else:
789 371d907a Antony Chazapis
            q = q % "v.serial, v.node, v.hash, v.size, v.type, v.source, v.mtime, v.muser, v.uuid, v.checksum, v.cluster"
790 60b8a083 Antony Chazapis
        args = [before, except_cluster, parent, start, nextling]
791 60b8a083 Antony Chazapis
        
792 60b8a083 Antony Chazapis
        subq, subargs = self._construct_paths(pathq)
793 60b8a083 Antony Chazapis
        if subq is not None:
794 60b8a083 Antony Chazapis
            q += subq
795 60b8a083 Antony Chazapis
            args += subargs
796 7ff57991 Antony Chazapis
        subq, subargs = self._construct_size(sizeq)
797 7ff57991 Antony Chazapis
        if subq is not None:
798 7ff57991 Antony Chazapis
            q += subq
799 7ff57991 Antony Chazapis
            args += subargs
800 4819d34f Antony Chazapis
        subq, subargs = self._construct_filters(domain, filterq)
801 60b8a083 Antony Chazapis
        if subq is not None:
802 60b8a083 Antony Chazapis
            q += subq
803 60b8a083 Antony Chazapis
            args += subargs
804 60b8a083 Antony Chazapis
        else:
805 60b8a083 Antony Chazapis
            q = q.replace("attributes a, ", "")
806 60b8a083 Antony Chazapis
            q = q.replace("and a.serial = v.serial ", "")
807 60b8a083 Antony Chazapis
        q += " order by n.path"
808 60b8a083 Antony Chazapis
        
809 60b8a083 Antony Chazapis
        if not delimiter:
810 60b8a083 Antony Chazapis
            q += " limit ?"
811 60b8a083 Antony Chazapis
            args.append(limit)
812 60b8a083 Antony Chazapis
            execute(q, args)
813 60b8a083 Antony Chazapis
            return self.fetchall(), ()
814 60b8a083 Antony Chazapis
        
815 60b8a083 Antony Chazapis
        pfz = len(prefix)
816 60b8a083 Antony Chazapis
        dz = len(delimiter)
817 60b8a083 Antony Chazapis
        count = 0
818 60b8a083 Antony Chazapis
        fetchone = self.fetchone
819 60b8a083 Antony Chazapis
        prefixes = []
820 60b8a083 Antony Chazapis
        pappend = prefixes.append
821 60b8a083 Antony Chazapis
        matches = []
822 60b8a083 Antony Chazapis
        mappend = matches.append
823 60b8a083 Antony Chazapis
        
824 60b8a083 Antony Chazapis
        execute(q, args)
825 60b8a083 Antony Chazapis
        while True:
826 60b8a083 Antony Chazapis
            props = fetchone()
827 60b8a083 Antony Chazapis
            if props is None:
828 60b8a083 Antony Chazapis
                break
829 371d907a Antony Chazapis
            path = props[0]
830 371d907a Antony Chazapis
            serial = props[1]
831 60b8a083 Antony Chazapis
            idx = path.find(delimiter, pfz)
832 60b8a083 Antony Chazapis
            
833 60b8a083 Antony Chazapis
            if idx < 0:
834 60b8a083 Antony Chazapis
                mappend(props)
835 60b8a083 Antony Chazapis
                count += 1
836 60b8a083 Antony Chazapis
                if count >= limit:
837 60b8a083 Antony Chazapis
                    break
838 60b8a083 Antony Chazapis
                continue
839 60b8a083 Antony Chazapis
            
840 60b8a083 Antony Chazapis
            if idx + dz == len(path):
841 60b8a083 Antony Chazapis
                mappend(props)
842 60b8a083 Antony Chazapis
                count += 1
843 4cc00e08 Antony Chazapis
                continue # Get one more, in case there is a path.
844 4cc00e08 Antony Chazapis
            pf = path[:idx + dz]
845 4cc00e08 Antony Chazapis
            pappend(pf)
846 60b8a083 Antony Chazapis
            if count >= limit: 
847 60b8a083 Antony Chazapis
                break
848 60b8a083 Antony Chazapis
            
849 60b8a083 Antony Chazapis
            args[3] = strnextling(pf) # New start.
850 60b8a083 Antony Chazapis
            execute(q, args)
851 60b8a083 Antony Chazapis
        
852 60b8a083 Antony Chazapis
        return matches, prefixes
853 37bee317 Antony Chazapis
    
854 37bee317 Antony Chazapis
    def latest_uuid(self, uuid):
855 37bee317 Antony Chazapis
        """Return a (path, serial) tuple, for the latest version of the given uuid."""
856 37bee317 Antony Chazapis
        
857 37bee317 Antony Chazapis
        q = ("select n.path, v.serial "
858 37bee317 Antony Chazapis
             "from versions v, nodes n "
859 37bee317 Antony Chazapis
             "where v.serial = (select max(serial) "
860 37bee317 Antony Chazapis
                               "from versions "
861 37bee317 Antony Chazapis
                               "where uuid = ?) "
862 37bee317 Antony Chazapis
             "and n.node = v.node")
863 37bee317 Antony Chazapis
        self.execute(q, (uuid,))
864 37bee317 Antony Chazapis
        return self.fetchone()