Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-backend / pithos / backends / lib / sqlalchemy / node.py @ e7117103

History | View | Annotate | Download (40 kB)

1 2e662088 Antony Chazapis
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2 d50ed8d4 Sofia Papagiannaki
#
3 4f917833 Sofia Papagiannaki
# Redistribution and use in source and binary forms, with or
4 4f917833 Sofia Papagiannaki
# without modification, are permitted provided that the following
5 4f917833 Sofia Papagiannaki
# conditions are met:
6 d50ed8d4 Sofia Papagiannaki
#
7 4f917833 Sofia Papagiannaki
#   1. Redistributions of source code must retain the above
8 4f917833 Sofia Papagiannaki
#      copyright notice, this list of conditions and the following
9 4f917833 Sofia Papagiannaki
#      disclaimer.
10 d50ed8d4 Sofia Papagiannaki
#
11 4f917833 Sofia Papagiannaki
#   2. Redistributions in binary form must reproduce the above
12 4f917833 Sofia Papagiannaki
#      copyright notice, this list of conditions and the following
13 4f917833 Sofia Papagiannaki
#      disclaimer in the documentation and/or other materials
14 4f917833 Sofia Papagiannaki
#      provided with the distribution.
15 d50ed8d4 Sofia Papagiannaki
#
16 4f917833 Sofia Papagiannaki
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 4f917833 Sofia Papagiannaki
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 4f917833 Sofia Papagiannaki
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 4f917833 Sofia Papagiannaki
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 4f917833 Sofia Papagiannaki
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 4f917833 Sofia Papagiannaki
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 4f917833 Sofia Papagiannaki
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 4f917833 Sofia Papagiannaki
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 4f917833 Sofia Papagiannaki
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 4f917833 Sofia Papagiannaki
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 4f917833 Sofia Papagiannaki
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 4f917833 Sofia Papagiannaki
# POSSIBILITY OF SUCH DAMAGE.
28 d50ed8d4 Sofia Papagiannaki
#
29 4f917833 Sofia Papagiannaki
# The views and conclusions contained in the software and
30 4f917833 Sofia Papagiannaki
# documentation are those of the authors and should not be
31 4f917833 Sofia Papagiannaki
# interpreted as representing official policies, either expressed
32 4f917833 Sofia Papagiannaki
# or implied, of GRNET S.A.
33 4f917833 Sofia Papagiannaki
34 4f917833 Sofia Papagiannaki
from time import time
35 c48acbfd Sofia Papagiannaki
from sqlalchemy import Table, Integer, BigInteger, DECIMAL, Column, String, MetaData, ForeignKey
36 e414f54d Sofia Papagiannaki
from sqlalchemy.types import Text
37 18266c93 Sofia Papagiannaki
from sqlalchemy.schema import Index, Sequence
38 a847494c chazapis
from sqlalchemy.sql import func, and_, or_, not_, null, select, bindparam, text, exists
39 4f1bc0a6 Sofia Papagiannaki
from sqlalchemy.ext.compiler import compiles
40 e414f54d Sofia Papagiannaki
from sqlalchemy.engine.reflection import Inspector
41 6a82f89f Sofia Papagiannaki
from sqlalchemy.exc import NoSuchTableError
42 4f1bc0a6 Sofia Papagiannaki
43 4f917833 Sofia Papagiannaki
from dbworker import DBWorker
44 4f917833 Sofia Papagiannaki
45 6e147ecc Antony Chazapis
from pithos.backends.filter import parse_filters
46 9d41b050 chazapis
47 059857e2 Antony Chazapis
48 d50ed8d4 Sofia Papagiannaki
ROOTNODE = 0
49 4f917833 Sofia Papagiannaki
50 d50ed8d4 Sofia Papagiannaki
(SERIAL, NODE, HASH, SIZE, TYPE, SOURCE, MTIME, MUSER, UUID, CHECKSUM,
51 d50ed8d4 Sofia Papagiannaki
 CLUSTER) = range(11)
52 cf341da4 Antony Chazapis
53 d50ed8d4 Sofia Papagiannaki
(MATCH_PREFIX, MATCH_EXACT) = range(2)
54 4f917833 Sofia Papagiannaki
55 4f917833 Sofia Papagiannaki
inf = float('inf')
56 4f917833 Sofia Papagiannaki
57 4f917833 Sofia Papagiannaki
58 4f917833 Sofia Papagiannaki
def strnextling(prefix):
59 4f917833 Sofia Papagiannaki
    """Return the first unicode string
60 4f917833 Sofia Papagiannaki
       greater than but not starting with given prefix.
61 4f917833 Sofia Papagiannaki
       strnextling('hello') -> 'hellp'
62 4f917833 Sofia Papagiannaki
    """
63 4f917833 Sofia Papagiannaki
    if not prefix:
64 4f917833 Sofia Papagiannaki
        ## all strings start with the null string,
65 4f917833 Sofia Papagiannaki
        ## therefore we have to approximate strnextling('')
66 4f917833 Sofia Papagiannaki
        ## with the last unicode character supported by python
67 4f917833 Sofia Papagiannaki
        ## 0x10ffff for wide (32-bit unicode) python builds
68 4f917833 Sofia Papagiannaki
        ## 0x00ffff for narrow (16-bit unicode) python builds
69 4f917833 Sofia Papagiannaki
        ## We will not autodetect. 0xffff is safe enough.
70 4f917833 Sofia Papagiannaki
        return unichr(0xffff)
71 4f917833 Sofia Papagiannaki
    s = prefix[:-1]
72 4f917833 Sofia Papagiannaki
    c = ord(prefix[-1])
73 4f917833 Sofia Papagiannaki
    if c >= 0xffff:
74 4f917833 Sofia Papagiannaki
        raise RuntimeError
75 d50ed8d4 Sofia Papagiannaki
    s += unichr(c + 1)
76 4f917833 Sofia Papagiannaki
    return s
77 4f917833 Sofia Papagiannaki
78 d50ed8d4 Sofia Papagiannaki
79 4f917833 Sofia Papagiannaki
def strprevling(prefix):
80 4f917833 Sofia Papagiannaki
    """Return an approximation of the last unicode string
81 4f917833 Sofia Papagiannaki
       less than but not starting with given prefix.
82 4f917833 Sofia Papagiannaki
       strprevling(u'hello') -> u'helln\\xffff'
83 4f917833 Sofia Papagiannaki
    """
84 4f917833 Sofia Papagiannaki
    if not prefix:
85 4f917833 Sofia Papagiannaki
        ## There is no prevling for the null string
86 4f917833 Sofia Papagiannaki
        return prefix
87 4f917833 Sofia Papagiannaki
    s = prefix[:-1]
88 4f917833 Sofia Papagiannaki
    c = ord(prefix[-1])
89 4f917833 Sofia Papagiannaki
    if c > 0:
90 d50ed8d4 Sofia Papagiannaki
        s += unichr(c - 1) + unichr(0xffff)
91 4f917833 Sofia Papagiannaki
    return s
92 4f917833 Sofia Papagiannaki
93 4f917833 Sofia Papagiannaki
_propnames = {
94 d50ed8d4 Sofia Papagiannaki
    'serial': 0,
95 d50ed8d4 Sofia Papagiannaki
    'node': 1,
96 d50ed8d4 Sofia Papagiannaki
    'hash': 2,
97 d50ed8d4 Sofia Papagiannaki
    'size': 3,
98 d50ed8d4 Sofia Papagiannaki
    'type': 4,
99 d50ed8d4 Sofia Papagiannaki
    'source': 5,
100 d50ed8d4 Sofia Papagiannaki
    'mtime': 6,
101 d50ed8d4 Sofia Papagiannaki
    'muser': 7,
102 d50ed8d4 Sofia Papagiannaki
    'uuid': 8,
103 d50ed8d4 Sofia Papagiannaki
    'checksum': 9,
104 d50ed8d4 Sofia Papagiannaki
    'cluster': 10
105 4f917833 Sofia Papagiannaki
}
106 4f917833 Sofia Papagiannaki
107 d50ed8d4 Sofia Papagiannaki
108 6a82f89f Sofia Papagiannaki
def create_tables(engine):
109 6a82f89f Sofia Papagiannaki
    metadata = MetaData()
110 d50ed8d4 Sofia Papagiannaki
111 6a82f89f Sofia Papagiannaki
    #create nodes table
112 d50ed8d4 Sofia Papagiannaki
    columns = []
113 6a82f89f Sofia Papagiannaki
    columns.append(Column('node', Integer, primary_key=True))
114 6a82f89f Sofia Papagiannaki
    columns.append(Column('parent', Integer,
115 6a82f89f Sofia Papagiannaki
                          ForeignKey('nodes.node',
116 6a82f89f Sofia Papagiannaki
                                     ondelete='CASCADE',
117 6a82f89f Sofia Papagiannaki
                                     onupdate='CASCADE'),
118 6a82f89f Sofia Papagiannaki
                          autoincrement=False))
119 6a82f89f Sofia Papagiannaki
    columns.append(Column('latest_version', Integer))
120 6a82f89f Sofia Papagiannaki
    columns.append(Column('path', String(2048), default='', nullable=False))
121 6a82f89f Sofia Papagiannaki
    nodes = Table('nodes', metadata, *columns, mysql_engine='InnoDB')
122 6a82f89f Sofia Papagiannaki
    Index('idx_nodes_path', nodes.c.path, unique=True)
123 6a82f89f Sofia Papagiannaki
    Index('idx_nodes_parent', nodes.c.parent)
124 d50ed8d4 Sofia Papagiannaki
125 6a82f89f Sofia Papagiannaki
    #create policy table
126 d50ed8d4 Sofia Papagiannaki
    columns = []
127 6a82f89f Sofia Papagiannaki
    columns.append(Column('node', Integer,
128 6a82f89f Sofia Papagiannaki
                          ForeignKey('nodes.node',
129 6a82f89f Sofia Papagiannaki
                                     ondelete='CASCADE',
130 6a82f89f Sofia Papagiannaki
                                     onupdate='CASCADE'),
131 6a82f89f Sofia Papagiannaki
                          primary_key=True))
132 6a82f89f Sofia Papagiannaki
    columns.append(Column('key', String(128), primary_key=True))
133 6a82f89f Sofia Papagiannaki
    columns.append(Column('value', String(256)))
134 6a82f89f Sofia Papagiannaki
    policy = Table('policy', metadata, *columns, mysql_engine='InnoDB')
135 d50ed8d4 Sofia Papagiannaki
136 6a82f89f Sofia Papagiannaki
    #create statistics table
137 d50ed8d4 Sofia Papagiannaki
    columns = []
138 6a82f89f Sofia Papagiannaki
    columns.append(Column('node', Integer,
139 6a82f89f Sofia Papagiannaki
                          ForeignKey('nodes.node',
140 6a82f89f Sofia Papagiannaki
                                     ondelete='CASCADE',
141 6a82f89f Sofia Papagiannaki
                                     onupdate='CASCADE'),
142 6a82f89f Sofia Papagiannaki
                          primary_key=True))
143 6a82f89f Sofia Papagiannaki
    columns.append(Column('population', Integer, nullable=False, default=0))
144 6a82f89f Sofia Papagiannaki
    columns.append(Column('size', BigInteger, nullable=False, default=0))
145 6a82f89f Sofia Papagiannaki
    columns.append(Column('mtime', DECIMAL(precision=16, scale=6)))
146 6a82f89f Sofia Papagiannaki
    columns.append(Column('cluster', Integer, nullable=False, default=0,
147 6a82f89f Sofia Papagiannaki
                          primary_key=True, autoincrement=False))
148 6a82f89f Sofia Papagiannaki
    statistics = Table('statistics', metadata, *columns, mysql_engine='InnoDB')
149 d50ed8d4 Sofia Papagiannaki
150 6a82f89f Sofia Papagiannaki
    #create versions table
151 d50ed8d4 Sofia Papagiannaki
    columns = []
152 6a82f89f Sofia Papagiannaki
    columns.append(Column('serial', Integer, primary_key=True))
153 6a82f89f Sofia Papagiannaki
    columns.append(Column('node', Integer,
154 6a82f89f Sofia Papagiannaki
                          ForeignKey('nodes.node',
155 6a82f89f Sofia Papagiannaki
                                     ondelete='CASCADE',
156 6a82f89f Sofia Papagiannaki
                                     onupdate='CASCADE')))
157 6a82f89f Sofia Papagiannaki
    columns.append(Column('hash', String(256)))
158 6a82f89f Sofia Papagiannaki
    columns.append(Column('size', BigInteger, nullable=False, default=0))
159 6a82f89f Sofia Papagiannaki
    columns.append(Column('type', String(256), nullable=False, default=''))
160 6a82f89f Sofia Papagiannaki
    columns.append(Column('source', Integer))
161 6a82f89f Sofia Papagiannaki
    columns.append(Column('mtime', DECIMAL(precision=16, scale=6)))
162 6a82f89f Sofia Papagiannaki
    columns.append(Column('muser', String(256), nullable=False, default=''))
163 6a82f89f Sofia Papagiannaki
    columns.append(Column('uuid', String(64), nullable=False, default=''))
164 6a82f89f Sofia Papagiannaki
    columns.append(Column('checksum', String(256), nullable=False, default=''))
165 6a82f89f Sofia Papagiannaki
    columns.append(Column('cluster', Integer, nullable=False, default=0))
166 6a82f89f Sofia Papagiannaki
    versions = Table('versions', metadata, *columns, mysql_engine='InnoDB')
167 6a82f89f Sofia Papagiannaki
    Index('idx_versions_node_mtime', versions.c.node, versions.c.mtime)
168 6a82f89f Sofia Papagiannaki
    Index('idx_versions_node_uuid', versions.c.uuid)
169 d50ed8d4 Sofia Papagiannaki
170 6a82f89f Sofia Papagiannaki
    #create attributes table
171 6a82f89f Sofia Papagiannaki
    columns = []
172 6a82f89f Sofia Papagiannaki
    columns.append(Column('serial', Integer,
173 6a82f89f Sofia Papagiannaki
                          ForeignKey('versions.serial',
174 6a82f89f Sofia Papagiannaki
                                     ondelete='CASCADE',
175 6a82f89f Sofia Papagiannaki
                                     onupdate='CASCADE'),
176 6a82f89f Sofia Papagiannaki
                          primary_key=True))
177 6a82f89f Sofia Papagiannaki
    columns.append(Column('domain', String(256), primary_key=True))
178 6a82f89f Sofia Papagiannaki
    columns.append(Column('key', String(128), primary_key=True))
179 6a82f89f Sofia Papagiannaki
    columns.append(Column('value', String(256)))
180 6a82f89f Sofia Papagiannaki
    attributes = Table('attributes', metadata, *columns, mysql_engine='InnoDB')
181 d50ed8d4 Sofia Papagiannaki
182 6a82f89f Sofia Papagiannaki
    metadata.create_all(engine)
183 6a82f89f Sofia Papagiannaki
    return metadata.sorted_tables
184 4f917833 Sofia Papagiannaki
185 d50ed8d4 Sofia Papagiannaki
186 4f917833 Sofia Papagiannaki
class Node(DBWorker):
187 4f917833 Sofia Papagiannaki
    """Nodes store path organization and have multiple versions.
188 4f917833 Sofia Papagiannaki
       Versions store object history and have multiple attributes.
189 4f917833 Sofia Papagiannaki
       Attributes store metadata.
190 4f917833 Sofia Papagiannaki
    """
191 d50ed8d4 Sofia Papagiannaki
192 4f917833 Sofia Papagiannaki
    # TODO: Provide an interface for included and excluded clusters.
193 d50ed8d4 Sofia Papagiannaki
194 4f917833 Sofia Papagiannaki
    def __init__(self, **params):
195 4f917833 Sofia Papagiannaki
        DBWorker.__init__(self, **params)
196 6a82f89f Sofia Papagiannaki
        try:
197 6a82f89f Sofia Papagiannaki
            metadata = MetaData(self.engine)
198 6a82f89f Sofia Papagiannaki
            self.nodes = Table('nodes', metadata, autoload=True)
199 6a82f89f Sofia Papagiannaki
            self.policy = Table('policy', metadata, autoload=True)
200 6a82f89f Sofia Papagiannaki
            self.statistics = Table('statistics', metadata, autoload=True)
201 6a82f89f Sofia Papagiannaki
            self.versions = Table('versions', metadata, autoload=True)
202 6a82f89f Sofia Papagiannaki
            self.attributes = Table('attributes', metadata, autoload=True)
203 6a82f89f Sofia Papagiannaki
        except NoSuchTableError:
204 6a82f89f Sofia Papagiannaki
            tables = create_tables(self.engine)
205 6a82f89f Sofia Papagiannaki
            map(lambda t: self.__setattr__(t.name, t), tables)
206 d50ed8d4 Sofia Papagiannaki
207 d3be972a Sofia Papagiannaki
        s = self.nodes.select().where(and_(self.nodes.c.node == ROOTNODE,
208 d3be972a Sofia Papagiannaki
                                           self.nodes.c.parent == ROOTNODE))
209 9eb713e1 Sofia Papagiannaki
        rp = self.conn.execute(s)
210 9eb713e1 Sofia Papagiannaki
        r = rp.fetchone()
211 9eb713e1 Sofia Papagiannaki
        rp.close()
212 18266c93 Sofia Papagiannaki
        if not r:
213 d50ed8d4 Sofia Papagiannaki
            s = self.nodes.insert(
214 d50ed8d4 Sofia Papagiannaki
            ).values(node=ROOTNODE, parent=ROOTNODE, path='')
215 18266c93 Sofia Papagiannaki
            self.conn.execute(s)
216 d50ed8d4 Sofia Papagiannaki
217 4f917833 Sofia Papagiannaki
    def node_create(self, parent, path):
218 4f917833 Sofia Papagiannaki
        """Create a new node from the given properties.
219 4f917833 Sofia Papagiannaki
           Return the node identifier of the new node.
220 4f917833 Sofia Papagiannaki
        """
221 18266c93 Sofia Papagiannaki
        #TODO catch IntegrityError?
222 18266c93 Sofia Papagiannaki
        s = self.nodes.insert().values(parent=parent, path=path)
223 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
224 18266c93 Sofia Papagiannaki
        inserted_primary_key = r.inserted_primary_key[0]
225 18266c93 Sofia Papagiannaki
        r.close()
226 18266c93 Sofia Papagiannaki
        return inserted_primary_key
227 d50ed8d4 Sofia Papagiannaki
228 4f917833 Sofia Papagiannaki
    def node_lookup(self, path):
229 4f917833 Sofia Papagiannaki
        """Lookup the current node of the given path.
230 4f917833 Sofia Papagiannaki
           Return None if the path is not found.
231 4f917833 Sofia Papagiannaki
        """
232 d50ed8d4 Sofia Papagiannaki
233 6b20cfbc Antony Chazapis
        # Use LIKE for comparison to avoid MySQL problems with trailing spaces.
234 d50ed8d4 Sofia Papagiannaki
        s = select([self.nodes.c.node], self.nodes.c.path.like(
235 d50ed8d4 Sofia Papagiannaki
            self.escape_like(path), escape='\\'))
236 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
237 18266c93 Sofia Papagiannaki
        row = r.fetchone()
238 18266c93 Sofia Papagiannaki
        r.close()
239 18266c93 Sofia Papagiannaki
        if row:
240 18266c93 Sofia Papagiannaki
            return row[0]
241 4f917833 Sofia Papagiannaki
        return None
242 d50ed8d4 Sofia Papagiannaki
243 8221c89d Sofia Papagiannaki
    def node_lookup_bulk(self, paths):
244 8221c89d Sofia Papagiannaki
        """Lookup the current nodes for the given paths.
245 8221c89d Sofia Papagiannaki
           Return () if the path is not found.
246 8221c89d Sofia Papagiannaki
        """
247 d50ed8d4 Sofia Papagiannaki
248 fe3252ab Sofia Papagiannaki
        if not paths:
249 fe3252ab Sofia Papagiannaki
            return ()
250 8221c89d Sofia Papagiannaki
        # Use LIKE for comparison to avoid MySQL problems with trailing spaces.
251 8221c89d Sofia Papagiannaki
        s = select([self.nodes.c.node], self.nodes.c.path.in_(paths))
252 8221c89d Sofia Papagiannaki
        r = self.conn.execute(s)
253 8221c89d Sofia Papagiannaki
        rows = r.fetchall()
254 8221c89d Sofia Papagiannaki
        r.close()
255 8221c89d Sofia Papagiannaki
        return [row[0] for row in rows]
256 d50ed8d4 Sofia Papagiannaki
257 4f917833 Sofia Papagiannaki
    def node_get_properties(self, node):
258 4f917833 Sofia Papagiannaki
        """Return the node's (parent, path).
259 4f917833 Sofia Papagiannaki
           Return None if the node is not found.
260 4f917833 Sofia Papagiannaki
        """
261 d50ed8d4 Sofia Papagiannaki
262 18266c93 Sofia Papagiannaki
        s = select([self.nodes.c.parent, self.nodes.c.path])
263 18266c93 Sofia Papagiannaki
        s = s.where(self.nodes.c.node == node)
264 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
265 18266c93 Sofia Papagiannaki
        l = r.fetchone()
266 18266c93 Sofia Papagiannaki
        r.close()
267 18266c93 Sofia Papagiannaki
        return l
268 d50ed8d4 Sofia Papagiannaki
269 4f917833 Sofia Papagiannaki
    def node_get_versions(self, node, keys=(), propnames=_propnames):
270 4f917833 Sofia Papagiannaki
        """Return the properties of all versions at node.
271 4f917833 Sofia Papagiannaki
           If keys is empty, return all properties in the order
272 f7d5b0cf Antony Chazapis
           (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster).
273 4f917833 Sofia Papagiannaki
        """
274 d50ed8d4 Sofia Papagiannaki
275 1c2fc0ff Antony Chazapis
        s = select([self.versions.c.serial,
276 1c2fc0ff Antony Chazapis
                    self.versions.c.node,
277 1c2fc0ff Antony Chazapis
                    self.versions.c.hash,
278 1c2fc0ff Antony Chazapis
                    self.versions.c.size,
279 cf341da4 Antony Chazapis
                    self.versions.c.type,
280 1c2fc0ff Antony Chazapis
                    self.versions.c.source,
281 1c2fc0ff Antony Chazapis
                    self.versions.c.mtime,
282 1c2fc0ff Antony Chazapis
                    self.versions.c.muser,
283 25ae8b75 Antony Chazapis
                    self.versions.c.uuid,
284 f7d5b0cf Antony Chazapis
                    self.versions.c.checksum,
285 1c2fc0ff Antony Chazapis
                    self.versions.c.cluster], self.versions.c.node == node)
286 cc28f894 Sofia Papagiannaki
        s = s.order_by(self.versions.c.serial)
287 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
288 18266c93 Sofia Papagiannaki
        rows = r.fetchall()
289 9eb713e1 Sofia Papagiannaki
        r.close()
290 18266c93 Sofia Papagiannaki
        if not rows:
291 18266c93 Sofia Papagiannaki
            return rows
292 d50ed8d4 Sofia Papagiannaki
293 4f917833 Sofia Papagiannaki
        if not keys:
294 18266c93 Sofia Papagiannaki
            return rows
295 d50ed8d4 Sofia Papagiannaki
296 18266c93 Sofia Papagiannaki
        return [[p[propnames[k]] for k in keys if k in propnames] for p in rows]
297 d50ed8d4 Sofia Papagiannaki
298 4f917833 Sofia Papagiannaki
    def node_count_children(self, node):
299 4f917833 Sofia Papagiannaki
        """Return node's child count."""
300 d50ed8d4 Sofia Papagiannaki
301 18266c93 Sofia Papagiannaki
        s = select([func.count(self.nodes.c.node)])
302 18266c93 Sofia Papagiannaki
        s = s.where(and_(self.nodes.c.parent == node,
303 18266c93 Sofia Papagiannaki
                         self.nodes.c.node != ROOTNODE))
304 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
305 18266c93 Sofia Papagiannaki
        row = r.fetchone()
306 18266c93 Sofia Papagiannaki
        r.close()
307 18266c93 Sofia Papagiannaki
        return row[0]
308 d50ed8d4 Sofia Papagiannaki
309 4f917833 Sofia Papagiannaki
    def node_purge_children(self, parent, before=inf, cluster=0):
310 4f917833 Sofia Papagiannaki
        """Delete all versions with the specified
311 4f917833 Sofia Papagiannaki
           parent and cluster, and return
312 813e42e5 Antony Chazapis
           the hashes and size of versions deleted.
313 4f917833 Sofia Papagiannaki
           Clears out nodes with no remaining versions.
314 4f917833 Sofia Papagiannaki
        """
315 b43d44ad Sofia Papagiannaki
        #update statistics
316 b43d44ad Sofia Papagiannaki
        c1 = select([self.nodes.c.node],
317 d50ed8d4 Sofia Papagiannaki
                    self.nodes.c.parent == parent)
318 b43d44ad Sofia Papagiannaki
        where_clause = and_(self.versions.c.node.in_(c1),
319 62d938dc Sofia Papagiannaki
                            self.versions.c.cluster == cluster)
320 18266c93 Sofia Papagiannaki
        s = select([func.count(self.versions.c.serial),
321 18266c93 Sofia Papagiannaki
                    func.sum(self.versions.c.size)])
322 18266c93 Sofia Papagiannaki
        s = s.where(where_clause)
323 62d938dc Sofia Papagiannaki
        if before != inf:
324 62d938dc Sofia Papagiannaki
            s = s.where(self.versions.c.mtime <= before)
325 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
326 18266c93 Sofia Papagiannaki
        row = r.fetchone()
327 18266c93 Sofia Papagiannaki
        r.close()
328 18266c93 Sofia Papagiannaki
        if not row:
329 813e42e5 Antony Chazapis
            return (), 0
330 18266c93 Sofia Papagiannaki
        nr, size = row[0], -row[1] if row[1] else 0
331 4f917833 Sofia Papagiannaki
        mtime = time()
332 18266c93 Sofia Papagiannaki
        self.statistics_update(parent, -nr, size, mtime, cluster)
333 18266c93 Sofia Papagiannaki
        self.statistics_update_ancestors(parent, -nr, size, mtime, cluster)
334 d50ed8d4 Sofia Papagiannaki
335 9d4502a8 Sofia Papagiannaki
        s = select([self.versions.c.hash, self.versions.c.serial])
336 18266c93 Sofia Papagiannaki
        s = s.where(where_clause)
337 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
338 9d4502a8 Sofia Papagiannaki
        hashes = []
339 9d4502a8 Sofia Papagiannaki
        serials = []
340 9d4502a8 Sofia Papagiannaki
        for row in r.fetchall():
341 9d4502a8 Sofia Papagiannaki
            hashes += [row[0]]
342 9d4502a8 Sofia Papagiannaki
            serials += [row[1]]
343 18266c93 Sofia Papagiannaki
        r.close()
344 d50ed8d4 Sofia Papagiannaki
345 b43d44ad Sofia Papagiannaki
        #delete versions
346 18266c93 Sofia Papagiannaki
        s = self.versions.delete().where(where_clause)
347 18266c93 Sofia Papagiannaki
        r = self.conn.execute(s)
348 18266c93 Sofia Papagiannaki
        r.close()
349 d50ed8d4 Sofia Papagiannaki
350 18266c93 Sofia Papagiannaki
        #delete nodes
351 b43d44ad Sofia Papagiannaki
        s = select([self.nodes.c.node],
352 d50ed8d4 Sofia Papagiannaki
                   and_(self.nodes.c.parent == parent,
353 d50ed8d4 Sofia Papagiannaki
                        select([func.count(self.versions.c.serial)],
354 d50ed8d4 Sofia Papagiannaki
                               self.versions.c.node == self.nodes.c.node).as_scalar() == 0))
355 70516d86 Sofia Papagiannaki
        rp = self.conn.execute(s)
356 70516d86 Sofia Papagiannaki
        nodes = [r[0] for r in rp.fetchall()]
357 70516d86 Sofia Papagiannaki
        rp.close()
358 fe3252ab Sofia Papagiannaki
        if nodes:
359 fe3252ab Sofia Papagiannaki
            s = self.nodes.delete().where(self.nodes.c.node.in_(nodes))
360 fe3252ab Sofia Papagiannaki
            self.conn.execute(s).close()
361 d50ed8d4 Sofia Papagiannaki
362 9d4502a8 Sofia Papagiannaki
        return hashes, size, serials
363 d50ed8d4 Sofia Papagiannaki
364 4f917833 Sofia Papagiannaki
    def node_purge(self, node, before=inf, cluster=0):
365 4f917833 Sofia Papagiannaki
        """Delete all versions with the specified
366 4f917833 Sofia Papagiannaki
           node and cluster, and return
367 813e42e5 Antony Chazapis
           the hashes and size of versions deleted.
368 4f917833 Sofia Papagiannaki
           Clears out the node if it has no remaining versions.
369 4f917833 Sofia Papagiannaki
        """
370 d50ed8d4 Sofia Papagiannaki
371 b43d44ad Sofia Papagiannaki
        #update statistics
372 b43d44ad Sofia Papagiannaki
        s = select([func.count(self.versions.c.serial),
373 b43d44ad Sofia Papagiannaki
                    func.sum(self.versions.c.size)])
374 b43d44ad Sofia Papagiannaki
        where_clause = and_(self.versions.c.node == node,
375 d50ed8d4 Sofia Papagiannaki
                            self.versions.c.cluster == cluster)
376 b43d44ad Sofia Papagiannaki
        s = s.where(where_clause)
377 62d938dc Sofia Papagiannaki
        if before != inf:
378 62d938dc Sofia Papagiannaki
            s = s.where(self.versions.c.mtime <= before)
379 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
380 b43d44ad Sofia Papagiannaki
        row = r.fetchone()
381 b43d44ad Sofia Papagiannaki
        nr, size = row[0], row[1]
382 b43d44ad Sofia Papagiannaki
        r.close()
383 4f917833 Sofia Papagiannaki
        if not nr:
384 813e42e5 Antony Chazapis
            return (), 0
385 4f917833 Sofia Papagiannaki
        mtime = time()
386 4f917833 Sofia Papagiannaki
        self.statistics_update_ancestors(node, -nr, -size, mtime, cluster)
387 d50ed8d4 Sofia Papagiannaki
388 9d4502a8 Sofia Papagiannaki
        s = select([self.versions.c.hash, self.versions.c.serial])
389 b43d44ad Sofia Papagiannaki
        s = s.where(where_clause)
390 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
391 9d4502a8 Sofia Papagiannaki
        hashes = []
392 9d4502a8 Sofia Papagiannaki
        serials = []
393 9d4502a8 Sofia Papagiannaki
        for row in r.fetchall():
394 9d4502a8 Sofia Papagiannaki
            hashes += [row[0]]
395 9d4502a8 Sofia Papagiannaki
            serials += [row[1]]
396 9eb713e1 Sofia Papagiannaki
        r.close()
397 d50ed8d4 Sofia Papagiannaki
398 b43d44ad Sofia Papagiannaki
        #delete versions
399 b43d44ad Sofia Papagiannaki
        s = self.versions.delete().where(where_clause)
400 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
401 b43d44ad Sofia Papagiannaki
        r.close()
402 d50ed8d4 Sofia Papagiannaki
403 b43d44ad Sofia Papagiannaki
        #delete nodes
404 b43d44ad Sofia Papagiannaki
        s = select([self.nodes.c.node],
405 d50ed8d4 Sofia Papagiannaki
                   and_(self.nodes.c.node == node,
406 d50ed8d4 Sofia Papagiannaki
                        select([func.count(self.versions.c.serial)],
407 d50ed8d4 Sofia Papagiannaki
                               self.versions.c.node == self.nodes.c.node).as_scalar() == 0))
408 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
409 b43d44ad Sofia Papagiannaki
        nodes = r.fetchall()
410 b43d44ad Sofia Papagiannaki
        r.close()
411 fe3252ab Sofia Papagiannaki
        if nodes:
412 fe3252ab Sofia Papagiannaki
            s = self.nodes.delete().where(self.nodes.c.node.in_(nodes))
413 fe3252ab Sofia Papagiannaki
            self.conn.execute(s).close()
414 d50ed8d4 Sofia Papagiannaki
415 9d4502a8 Sofia Papagiannaki
        return hashes, size, serials
416 d50ed8d4 Sofia Papagiannaki
417 4f917833 Sofia Papagiannaki
    def node_remove(self, node):
418 4f917833 Sofia Papagiannaki
        """Remove the node specified.
419 4f917833 Sofia Papagiannaki
           Return false if the node has children or is not found.
420 4f917833 Sofia Papagiannaki
        """
421 d50ed8d4 Sofia Papagiannaki
422 4f917833 Sofia Papagiannaki
        if self.node_count_children(node):
423 4f917833 Sofia Papagiannaki
            return False
424 d50ed8d4 Sofia Papagiannaki
425 4f917833 Sofia Papagiannaki
        mtime = time()
426 b43d44ad Sofia Papagiannaki
        s = select([func.count(self.versions.c.serial),
427 b43d44ad Sofia Papagiannaki
                    func.sum(self.versions.c.size),
428 b43d44ad Sofia Papagiannaki
                    self.versions.c.cluster])
429 b43d44ad Sofia Papagiannaki
        s = s.where(self.versions.c.node == node)
430 b43d44ad Sofia Papagiannaki
        s = s.group_by(self.versions.c.cluster)
431 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
432 b43d44ad Sofia Papagiannaki
        for population, size, cluster in r.fetchall():
433 d50ed8d4 Sofia Papagiannaki
            self.statistics_update_ancestors(
434 d50ed8d4 Sofia Papagiannaki
                node, -population, -size, mtime, cluster)
435 b43d44ad Sofia Papagiannaki
        r.close()
436 d50ed8d4 Sofia Papagiannaki
437 b43d44ad Sofia Papagiannaki
        s = self.nodes.delete().where(self.nodes.c.node == node)
438 b43d44ad Sofia Papagiannaki
        self.conn.execute(s).close()
439 4f917833 Sofia Papagiannaki
        return True
440 d50ed8d4 Sofia Papagiannaki
441 5e7485da Antony Chazapis
    def policy_get(self, node):
442 6a82f89f Sofia Papagiannaki
        s = select([self.policy.c.key, self.policy.c.value],
443 d50ed8d4 Sofia Papagiannaki
                   self.policy.c.node == node)
444 5e7485da Antony Chazapis
        r = self.conn.execute(s)
445 5e7485da Antony Chazapis
        d = dict(r.fetchall())
446 5e7485da Antony Chazapis
        r.close()
447 5e7485da Antony Chazapis
        return d
448 d50ed8d4 Sofia Papagiannaki
449 5e7485da Antony Chazapis
    def policy_set(self, node, policy):
450 5e7485da Antony Chazapis
        #insert or replace
451 5e7485da Antony Chazapis
        for k, v in policy.iteritems():
452 6a82f89f Sofia Papagiannaki
            s = self.policy.update().where(and_(self.policy.c.node == node,
453 d50ed8d4 Sofia Papagiannaki
                                                self.policy.c.key == k))
454 d50ed8d4 Sofia Papagiannaki
            s = s.values(value=v)
455 5e7485da Antony Chazapis
            rp = self.conn.execute(s)
456 5e7485da Antony Chazapis
            rp.close()
457 5e7485da Antony Chazapis
            if rp.rowcount == 0:
458 6a82f89f Sofia Papagiannaki
                s = self.policy.insert()
459 d50ed8d4 Sofia Papagiannaki
                values = {'node': node, 'key': k, 'value': v}
460 5e7485da Antony Chazapis
                r = self.conn.execute(s, values)
461 5e7485da Antony Chazapis
                r.close()
462 d50ed8d4 Sofia Papagiannaki
463 4f917833 Sofia Papagiannaki
    def statistics_get(self, node, cluster=0):
464 4f917833 Sofia Papagiannaki
        """Return population, total size and last mtime
465 4f917833 Sofia Papagiannaki
           for all versions under node that belong to the cluster.
466 4f917833 Sofia Papagiannaki
        """
467 d50ed8d4 Sofia Papagiannaki
468 b43d44ad Sofia Papagiannaki
        s = select([self.statistics.c.population,
469 b43d44ad Sofia Papagiannaki
                    self.statistics.c.size,
470 b43d44ad Sofia Papagiannaki
                    self.statistics.c.mtime])
471 b43d44ad Sofia Papagiannaki
        s = s.where(and_(self.statistics.c.node == node,
472 b43d44ad Sofia Papagiannaki
                         self.statistics.c.cluster == cluster))
473 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
474 b43d44ad Sofia Papagiannaki
        row = r.fetchone()
475 b43d44ad Sofia Papagiannaki
        r.close()
476 b43d44ad Sofia Papagiannaki
        return row
477 d50ed8d4 Sofia Papagiannaki
478 4f917833 Sofia Papagiannaki
    def statistics_update(self, node, population, size, mtime, cluster=0):
479 4f917833 Sofia Papagiannaki
        """Update the statistics of the given node.
480 4f917833 Sofia Papagiannaki
           Statistics keep track the population, total
481 4f917833 Sofia Papagiannaki
           size of objects and mtime in the node's namespace.
482 4f917833 Sofia Papagiannaki
           May be zero or positive or negative numbers.
483 4f917833 Sofia Papagiannaki
        """
484 18266c93 Sofia Papagiannaki
        s = select([self.statistics.c.population, self.statistics.c.size],
485 d50ed8d4 Sofia Papagiannaki
                   and_(self.statistics.c.node == node,
486 d50ed8d4 Sofia Papagiannaki
                        self.statistics.c.cluster == cluster))
487 b43d44ad Sofia Papagiannaki
        rp = self.conn.execute(s)
488 b43d44ad Sofia Papagiannaki
        r = rp.fetchone()
489 b43d44ad Sofia Papagiannaki
        rp.close()
490 18266c93 Sofia Papagiannaki
        if not r:
491 4f917833 Sofia Papagiannaki
            prepopulation, presize = (0, 0)
492 4f917833 Sofia Papagiannaki
        else:
493 4f917833 Sofia Papagiannaki
            prepopulation, presize = r
494 4f917833 Sofia Papagiannaki
        population += prepopulation
495 4f917833 Sofia Papagiannaki
        size += presize
496 d50ed8d4 Sofia Papagiannaki
497 d3be972a Sofia Papagiannaki
        #insert or replace
498 70516d86 Sofia Papagiannaki
        #TODO better upsert
499 d50ed8d4 Sofia Papagiannaki
        u = self.statistics.update().where(and_(self.statistics.c.node == node,
500 d50ed8d4 Sofia Papagiannaki
                                           self.statistics.c.cluster == cluster))
501 d3be972a Sofia Papagiannaki
        u = u.values(population=population, size=size, mtime=mtime)
502 d3be972a Sofia Papagiannaki
        rp = self.conn.execute(u)
503 d3be972a Sofia Papagiannaki
        rp.close()
504 d3be972a Sofia Papagiannaki
        if rp.rowcount == 0:
505 d3be972a Sofia Papagiannaki
            ins = self.statistics.insert()
506 d3be972a Sofia Papagiannaki
            ins = ins.values(node=node, population=population, size=size,
507 d3be972a Sofia Papagiannaki
                             mtime=mtime, cluster=cluster)
508 d3be972a Sofia Papagiannaki
            self.conn.execute(ins).close()
509 d50ed8d4 Sofia Papagiannaki
510 4f917833 Sofia Papagiannaki
    def statistics_update_ancestors(self, node, population, size, mtime, cluster=0):
511 4f917833 Sofia Papagiannaki
        """Update the statistics of the given node's parent.
512 4f917833 Sofia Papagiannaki
           Then recursively update all parents up to the root.
513 4f917833 Sofia Papagiannaki
           Population is not recursive.
514 4f917833 Sofia Papagiannaki
        """
515 d50ed8d4 Sofia Papagiannaki
516 4f917833 Sofia Papagiannaki
        while True:
517 18266c93 Sofia Papagiannaki
            if node == ROOTNODE:
518 4f917833 Sofia Papagiannaki
                break
519 4f917833 Sofia Papagiannaki
            props = self.node_get_properties(node)
520 4f917833 Sofia Papagiannaki
            if props is None:
521 4f917833 Sofia Papagiannaki
                break
522 4f917833 Sofia Papagiannaki
            parent, path = props
523 4f917833 Sofia Papagiannaki
            self.statistics_update(parent, population, size, mtime, cluster)
524 4f917833 Sofia Papagiannaki
            node = parent
525 d50ed8d4 Sofia Papagiannaki
            population = 0  # Population isn't recursive
526 d50ed8d4 Sofia Papagiannaki
527 4f917833 Sofia Papagiannaki
    def statistics_latest(self, node, before=inf, except_cluster=0):
528 4f917833 Sofia Papagiannaki
        """Return population, total size and last mtime
529 4f917833 Sofia Papagiannaki
           for all latest versions under node that
530 4f917833 Sofia Papagiannaki
           do not belong to the cluster.
531 4f917833 Sofia Papagiannaki
        """
532 d50ed8d4 Sofia Papagiannaki
533 4f917833 Sofia Papagiannaki
        # The node.
534 4f917833 Sofia Papagiannaki
        props = self.node_get_properties(node)
535 4f917833 Sofia Papagiannaki
        if props is None:
536 4f917833 Sofia Papagiannaki
            return None
537 4f917833 Sofia Papagiannaki
        parent, path = props
538 d50ed8d4 Sofia Papagiannaki
539 4f917833 Sofia Papagiannaki
        # The latest version.
540 b43d44ad Sofia Papagiannaki
        s = select([self.versions.c.serial,
541 b43d44ad Sofia Papagiannaki
                    self.versions.c.node,
542 1c2fc0ff Antony Chazapis
                    self.versions.c.hash,
543 b43d44ad Sofia Papagiannaki
                    self.versions.c.size,
544 cf341da4 Antony Chazapis
                    self.versions.c.type,
545 d0aacf54 Sofia Papagiannaki
                    self.versions.c.source,
546 b43d44ad Sofia Papagiannaki
                    self.versions.c.mtime,
547 b43d44ad Sofia Papagiannaki
                    self.versions.c.muser,
548 25ae8b75 Antony Chazapis
                    self.versions.c.uuid,
549 f7d5b0cf Antony Chazapis
                    self.versions.c.checksum,
550 b43d44ad Sofia Papagiannaki
                    self.versions.c.cluster])
551 62d938dc Sofia Papagiannaki
        if before != inf:
552 585b75e7 Sofia Papagiannaki
            filtered = select([func.max(self.versions.c.serial)],
553 d50ed8d4 Sofia Papagiannaki
                              self.versions.c.node == node)
554 62d938dc Sofia Papagiannaki
            filtered = filtered.where(self.versions.c.mtime < before)
555 585b75e7 Sofia Papagiannaki
        else:
556 585b75e7 Sofia Papagiannaki
            filtered = select([self.nodes.c.latest_version],
557 d50ed8d4 Sofia Papagiannaki
                              self.versions.c.node == node)
558 b43d44ad Sofia Papagiannaki
        s = s.where(and_(self.versions.c.cluster != except_cluster,
559 62d938dc Sofia Papagiannaki
                         self.versions.c.serial == filtered))
560 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
561 b43d44ad Sofia Papagiannaki
        props = r.fetchone()
562 b43d44ad Sofia Papagiannaki
        r.close()
563 b43d44ad Sofia Papagiannaki
        if not props:
564 4f917833 Sofia Papagiannaki
            return None
565 4f917833 Sofia Papagiannaki
        mtime = props[MTIME]
566 d50ed8d4 Sofia Papagiannaki
567 4f917833 Sofia Papagiannaki
        # First level, just under node (get population).
568 b43d44ad Sofia Papagiannaki
        v = self.versions.alias('v')
569 b43d44ad Sofia Papagiannaki
        s = select([func.count(v.c.serial),
570 b43d44ad Sofia Papagiannaki
                    func.sum(v.c.size),
571 b43d44ad Sofia Papagiannaki
                    func.max(v.c.mtime)])
572 62d938dc Sofia Papagiannaki
        if before != inf:
573 585b75e7 Sofia Papagiannaki
            c1 = select([func.max(self.versions.c.serial)])
574 62d938dc Sofia Papagiannaki
            c1 = c1.where(self.versions.c.mtime < before)
575 585b75e7 Sofia Papagiannaki
            c1.where(self.versions.c.node == v.c.node)
576 585b75e7 Sofia Papagiannaki
        else:
577 585b75e7 Sofia Papagiannaki
            c1 = select([self.nodes.c.latest_version])
578 585b75e7 Sofia Papagiannaki
            c1.where(self.nodes.c.node == v.c.node)
579 b43d44ad Sofia Papagiannaki
        c2 = select([self.nodes.c.node], self.nodes.c.parent == node)
580 585b75e7 Sofia Papagiannaki
        s = s.where(and_(v.c.serial == c1,
581 b43d44ad Sofia Papagiannaki
                         v.c.cluster != except_cluster,
582 b43d44ad Sofia Papagiannaki
                         v.c.node.in_(c2)))
583 b43d44ad Sofia Papagiannaki
        rp = self.conn.execute(s)
584 b43d44ad Sofia Papagiannaki
        r = rp.fetchone()
585 b43d44ad Sofia Papagiannaki
        rp.close()
586 b43d44ad Sofia Papagiannaki
        if not r:
587 4f917833 Sofia Papagiannaki
            return None
588 4f917833 Sofia Papagiannaki
        count = r[0]
589 4f917833 Sofia Papagiannaki
        mtime = max(mtime, r[2])
590 4f917833 Sofia Papagiannaki
        if count == 0:
591 4f917833 Sofia Papagiannaki
            return (0, 0, mtime)
592 d50ed8d4 Sofia Papagiannaki
593 4f917833 Sofia Papagiannaki
        # All children (get size and mtime).
594 92da0e5a Antony Chazapis
        # This is why the full path is stored.
595 b43d44ad Sofia Papagiannaki
        s = select([func.count(v.c.serial),
596 b43d44ad Sofia Papagiannaki
                    func.sum(v.c.size),
597 b43d44ad Sofia Papagiannaki
                    func.max(v.c.mtime)])
598 62d938dc Sofia Papagiannaki
        if before != inf:
599 585b75e7 Sofia Papagiannaki
            c1 = select([func.max(self.versions.c.serial)],
600 d50ed8d4 Sofia Papagiannaki
                        self.versions.c.node == v.c.node)
601 62d938dc Sofia Papagiannaki
            c1 = c1.where(self.versions.c.mtime < before)
602 585b75e7 Sofia Papagiannaki
        else:
603 585b75e7 Sofia Papagiannaki
            c1 = select([self.nodes.c.serial],
604 d50ed8d4 Sofia Papagiannaki
                        self.nodes.c.node == v.c.node)
605 d50ed8d4 Sofia Papagiannaki
        c2 = select([self.nodes.c.node], self.nodes.c.path.like(
606 d50ed8d4 Sofia Papagiannaki
            self.escape_like(path) + '%', escape='\\'))
607 b43d44ad Sofia Papagiannaki
        s = s.where(and_(v.c.serial == c1,
608 b43d44ad Sofia Papagiannaki
                         v.c.cluster != except_cluster,
609 b43d44ad Sofia Papagiannaki
                         v.c.node.in_(c2)))
610 b43d44ad Sofia Papagiannaki
        rp = self.conn.execute(s)
611 b43d44ad Sofia Papagiannaki
        r = rp.fetchone()
612 b43d44ad Sofia Papagiannaki
        rp.close()
613 b43d44ad Sofia Papagiannaki
        if not r:
614 4f917833 Sofia Papagiannaki
            return None
615 4f917833 Sofia Papagiannaki
        size = r[1] - props[SIZE]
616 4f917833 Sofia Papagiannaki
        mtime = max(mtime, r[2])
617 4f917833 Sofia Papagiannaki
        return (count, size, mtime)
618 d50ed8d4 Sofia Papagiannaki
619 585b75e7 Sofia Papagiannaki
    def nodes_set_latest_version(self, node, serial):
620 585b75e7 Sofia Papagiannaki
        s = self.nodes.update().where(self.nodes.c.node == node)
621 d50ed8d4 Sofia Papagiannaki
        s = s.values(latest_version=serial)
622 585b75e7 Sofia Papagiannaki
        self.conn.execute(s).close()
623 d50ed8d4 Sofia Papagiannaki
624 f7d5b0cf Antony Chazapis
    def version_create(self, node, hash, size, type, source, muser, uuid, checksum, cluster=0):
625 4f917833 Sofia Papagiannaki
        """Create a new version from the given properties.
626 4f917833 Sofia Papagiannaki
           Return the (serial, mtime) of the new version.
627 4f917833 Sofia Papagiannaki
        """
628 d50ed8d4 Sofia Papagiannaki
629 4f917833 Sofia Papagiannaki
        mtime = time()
630 d50ed8d4 Sofia Papagiannaki
        s = self.versions.insert(
631 d50ed8d4 Sofia Papagiannaki
        ).values(node=node, hash=hash, size=size, type=type, source=source,
632 d50ed8d4 Sofia Papagiannaki
                 mtime=mtime, muser=muser, uuid=uuid, checksum=checksum, cluster=cluster)
633 b43d44ad Sofia Papagiannaki
        serial = self.conn.execute(s).inserted_primary_key[0]
634 4f917833 Sofia Papagiannaki
        self.statistics_update_ancestors(node, 1, size, mtime, cluster)
635 d50ed8d4 Sofia Papagiannaki
636 585b75e7 Sofia Papagiannaki
        self.nodes_set_latest_version(node, serial)
637 d50ed8d4 Sofia Papagiannaki
638 4f917833 Sofia Papagiannaki
        return serial, mtime
639 d50ed8d4 Sofia Papagiannaki
640 8221c89d Sofia Papagiannaki
    def version_lookup(self, node, before=inf, cluster=0, all_props=True):
641 4f917833 Sofia Papagiannaki
        """Lookup the current version of the given node.
642 4f917833 Sofia Papagiannaki
           Return a list with its properties:
643 d50ed8d4 Sofia Papagiannaki
           (serial, node, hash, size, type, source, mtime,
644 d50ed8d4 Sofia Papagiannaki
            muser, uuid, checksum, cluster)
645 4f917833 Sofia Papagiannaki
           or None if the current version is not found in the given cluster.
646 4f917833 Sofia Papagiannaki
        """
647 d50ed8d4 Sofia Papagiannaki
648 b43d44ad Sofia Papagiannaki
        v = self.versions.alias('v')
649 8221c89d Sofia Papagiannaki
        if not all_props:
650 8221c89d Sofia Papagiannaki
            s = select([v.c.serial])
651 8221c89d Sofia Papagiannaki
        else:
652 8221c89d Sofia Papagiannaki
            s = select([v.c.serial, v.c.node, v.c.hash,
653 8221c89d Sofia Papagiannaki
                        v.c.size, v.c.type, v.c.source,
654 8221c89d Sofia Papagiannaki
                        v.c.mtime, v.c.muser, v.c.uuid,
655 8221c89d Sofia Papagiannaki
                        v.c.checksum, v.c.cluster])
656 62d938dc Sofia Papagiannaki
        if before != inf:
657 585b75e7 Sofia Papagiannaki
            c = select([func.max(self.versions.c.serial)],
658 d50ed8d4 Sofia Papagiannaki
                       self.versions.c.node == node)
659 62d938dc Sofia Papagiannaki
            c = c.where(self.versions.c.mtime < before)
660 585b75e7 Sofia Papagiannaki
        else:
661 585b75e7 Sofia Papagiannaki
            c = select([self.nodes.c.latest_version],
662 d50ed8d4 Sofia Papagiannaki
                       self.nodes.c.node == node)
663 b43d44ad Sofia Papagiannaki
        s = s.where(and_(v.c.serial == c,
664 b43d44ad Sofia Papagiannaki
                         v.c.cluster == cluster))
665 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
666 b43d44ad Sofia Papagiannaki
        props = r.fetchone()
667 b43d44ad Sofia Papagiannaki
        r.close()
668 70516d86 Sofia Papagiannaki
        if props:
669 4f917833 Sofia Papagiannaki
            return props
670 4f917833 Sofia Papagiannaki
        return None
671 d50ed8d4 Sofia Papagiannaki
672 8221c89d Sofia Papagiannaki
    def version_lookup_bulk(self, nodes, before=inf, cluster=0, all_props=True):
673 8221c89d Sofia Papagiannaki
        """Lookup the current versions of the given nodes.
674 8221c89d Sofia Papagiannaki
           Return a list with their properties:
675 8221c89d Sofia Papagiannaki
           (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster).
676 8221c89d Sofia Papagiannaki
        """
677 585b75e7 Sofia Papagiannaki
        if not nodes:
678 585b75e7 Sofia Papagiannaki
            return ()
679 8221c89d Sofia Papagiannaki
        v = self.versions.alias('v')
680 8221c89d Sofia Papagiannaki
        if not all_props:
681 8221c89d Sofia Papagiannaki
            s = select([v.c.serial])
682 8221c89d Sofia Papagiannaki
        else:
683 8221c89d Sofia Papagiannaki
            s = select([v.c.serial, v.c.node, v.c.hash,
684 8221c89d Sofia Papagiannaki
                        v.c.size, v.c.type, v.c.source,
685 8221c89d Sofia Papagiannaki
                        v.c.mtime, v.c.muser, v.c.uuid,
686 8221c89d Sofia Papagiannaki
                        v.c.checksum, v.c.cluster])
687 8221c89d Sofia Papagiannaki
        if before != inf:
688 585b75e7 Sofia Papagiannaki
            c = select([func.max(self.versions.c.serial)],
689 d50ed8d4 Sofia Papagiannaki
                       self.versions.c.node.in_(nodes))
690 8221c89d Sofia Papagiannaki
            c = c.where(self.versions.c.mtime < before)
691 585b75e7 Sofia Papagiannaki
            c = c.group_by(self.versions.c.node)
692 585b75e7 Sofia Papagiannaki
        else:
693 585b75e7 Sofia Papagiannaki
            c = select([self.nodes.c.latest_version],
694 d50ed8d4 Sofia Papagiannaki
                       self.nodes.c.node.in_(nodes))
695 8221c89d Sofia Papagiannaki
        s = s.where(and_(v.c.serial.in_(c),
696 8221c89d Sofia Papagiannaki
                         v.c.cluster == cluster))
697 07867f70 Sofia Papagiannaki
        s = s.order_by(v.c.node)
698 8221c89d Sofia Papagiannaki
        r = self.conn.execute(s)
699 8221c89d Sofia Papagiannaki
        rproxy = r.fetchall()
700 8221c89d Sofia Papagiannaki
        r.close()
701 8221c89d Sofia Papagiannaki
        return (tuple(row.values()) for row in rproxy)
702 d50ed8d4 Sofia Papagiannaki
703 4f917833 Sofia Papagiannaki
    def version_get_properties(self, serial, keys=(), propnames=_propnames):
704 4f917833 Sofia Papagiannaki
        """Return a sequence of values for the properties of
705 4f917833 Sofia Papagiannaki
           the version specified by serial and the keys, in the order given.
706 4f917833 Sofia Papagiannaki
           If keys is empty, return all properties in the order
707 f7d5b0cf Antony Chazapis
           (serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster).
708 4f917833 Sofia Papagiannaki
        """
709 d50ed8d4 Sofia Papagiannaki
710 b43d44ad Sofia Papagiannaki
        v = self.versions.alias()
711 37bee317 Antony Chazapis
        s = select([v.c.serial, v.c.node, v.c.hash,
712 cf341da4 Antony Chazapis
                    v.c.size, v.c.type, v.c.source,
713 cf341da4 Antony Chazapis
                    v.c.mtime, v.c.muser, v.c.uuid,
714 f7d5b0cf Antony Chazapis
                    v.c.checksum, v.c.cluster], v.c.serial == serial)
715 b43d44ad Sofia Papagiannaki
        rp = self.conn.execute(s)
716 b43d44ad Sofia Papagiannaki
        r = rp.fetchone()
717 b43d44ad Sofia Papagiannaki
        rp.close()
718 4f917833 Sofia Papagiannaki
        if r is None:
719 4f917833 Sofia Papagiannaki
            return r
720 d50ed8d4 Sofia Papagiannaki
721 4f917833 Sofia Papagiannaki
        if not keys:
722 4f917833 Sofia Papagiannaki
            return r
723 4f917833 Sofia Papagiannaki
        return [r[propnames[k]] for k in keys if k in propnames]
724 d50ed8d4 Sofia Papagiannaki
725 f7d5b0cf Antony Chazapis
    def version_put_property(self, serial, key, value):
726 f7d5b0cf Antony Chazapis
        """Set value for the property of version specified by key."""
727 d50ed8d4 Sofia Papagiannaki
728 f7d5b0cf Antony Chazapis
        if key not in _propnames:
729 f7d5b0cf Antony Chazapis
            return
730 f7d5b0cf Antony Chazapis
        s = self.versions.update()
731 f7d5b0cf Antony Chazapis
        s = s.where(self.versions.c.serial == serial)
732 f7d5b0cf Antony Chazapis
        s = s.values(**{key: value})
733 f7d5b0cf Antony Chazapis
        self.conn.execute(s).close()
734 d50ed8d4 Sofia Papagiannaki
735 4f917833 Sofia Papagiannaki
    def version_recluster(self, serial, cluster):
736 4f917833 Sofia Papagiannaki
        """Move the version into another cluster."""
737 d50ed8d4 Sofia Papagiannaki
738 4f917833 Sofia Papagiannaki
        props = self.version_get_properties(serial)
739 4f917833 Sofia Papagiannaki
        if not props:
740 4f917833 Sofia Papagiannaki
            return
741 4f917833 Sofia Papagiannaki
        node = props[NODE]
742 4f917833 Sofia Papagiannaki
        size = props[SIZE]
743 4f917833 Sofia Papagiannaki
        oldcluster = props[CLUSTER]
744 4f917833 Sofia Papagiannaki
        if cluster == oldcluster:
745 4f917833 Sofia Papagiannaki
            return
746 d50ed8d4 Sofia Papagiannaki
747 4f917833 Sofia Papagiannaki
        mtime = time()
748 4f917833 Sofia Papagiannaki
        self.statistics_update_ancestors(node, -1, -size, mtime, oldcluster)
749 4f917833 Sofia Papagiannaki
        self.statistics_update_ancestors(node, 1, size, mtime, cluster)
750 d50ed8d4 Sofia Papagiannaki
751 b43d44ad Sofia Papagiannaki
        s = self.versions.update()
752 b43d44ad Sofia Papagiannaki
        s = s.where(self.versions.c.serial == serial)
753 d50ed8d4 Sofia Papagiannaki
        s = s.values(cluster=cluster)
754 b43d44ad Sofia Papagiannaki
        self.conn.execute(s).close()
755 d50ed8d4 Sofia Papagiannaki
756 4f917833 Sofia Papagiannaki
    def version_remove(self, serial):
757 4f917833 Sofia Papagiannaki
        """Remove the serial specified."""
758 d50ed8d4 Sofia Papagiannaki
759 5161c672 Antony Chazapis
        props = self.version_get_properties(serial)
760 4f917833 Sofia Papagiannaki
        if not props:
761 4f917833 Sofia Papagiannaki
            return
762 4f917833 Sofia Papagiannaki
        node = props[NODE]
763 5161c672 Antony Chazapis
        hash = props[HASH]
764 4f917833 Sofia Papagiannaki
        size = props[SIZE]
765 4f917833 Sofia Papagiannaki
        cluster = props[CLUSTER]
766 d50ed8d4 Sofia Papagiannaki
767 4f917833 Sofia Papagiannaki
        mtime = time()
768 4f917833 Sofia Papagiannaki
        self.statistics_update_ancestors(node, -1, -size, mtime, cluster)
769 d50ed8d4 Sofia Papagiannaki
770 b43d44ad Sofia Papagiannaki
        s = self.versions.delete().where(self.versions.c.serial == serial)
771 b43d44ad Sofia Papagiannaki
        self.conn.execute(s).close()
772 d50ed8d4 Sofia Papagiannaki
773 585b75e7 Sofia Papagiannaki
        props = self.version_lookup(node, cluster=cluster, all_props=False)
774 585b75e7 Sofia Papagiannaki
        if props:
775 e7117103 Sofia Papagiannaki
            self.nodes_set_latest_version(node, serial)
776 d50ed8d4 Sofia Papagiannaki
777 813e42e5 Antony Chazapis
        return hash, size
778 d50ed8d4 Sofia Papagiannaki
779 059857e2 Antony Chazapis
    def attribute_get(self, serial, domain, keys=()):
780 4f917833 Sofia Papagiannaki
        """Return a list of (key, value) pairs of the version specified by serial.
781 4f917833 Sofia Papagiannaki
           If keys is empty, return all attributes.
782 4f917833 Sofia Papagiannaki
           Othwerise, return only those specified.
783 4f917833 Sofia Papagiannaki
        """
784 d50ed8d4 Sofia Papagiannaki
785 4f917833 Sofia Papagiannaki
        if keys:
786 b43d44ad Sofia Papagiannaki
            attrs = self.attributes.alias()
787 b43d44ad Sofia Papagiannaki
            s = select([attrs.c.key, attrs.c.value])
788 b43d44ad Sofia Papagiannaki
            s = s.where(and_(attrs.c.key.in_(keys),
789 059857e2 Antony Chazapis
                             attrs.c.serial == serial,
790 059857e2 Antony Chazapis
                             attrs.c.domain == domain))
791 4f917833 Sofia Papagiannaki
        else:
792 b43d44ad Sofia Papagiannaki
            attrs = self.attributes.alias()
793 b43d44ad Sofia Papagiannaki
            s = select([attrs.c.key, attrs.c.value])
794 059857e2 Antony Chazapis
            s = s.where(and_(attrs.c.serial == serial,
795 059857e2 Antony Chazapis
                             attrs.c.domain == domain))
796 b43d44ad Sofia Papagiannaki
        r = self.conn.execute(s)
797 b43d44ad Sofia Papagiannaki
        l = r.fetchall()
798 b43d44ad Sofia Papagiannaki
        r.close()
799 b43d44ad Sofia Papagiannaki
        return l
800 d50ed8d4 Sofia Papagiannaki
801 059857e2 Antony Chazapis
    def attribute_set(self, serial, domain, items):
802 4f917833 Sofia Papagiannaki
        """Set the attributes of the version specified by serial.
803 4f917833 Sofia Papagiannaki
           Receive attributes as an iterable of (key, value) pairs.
804 4f917833 Sofia Papagiannaki
        """
805 70516d86 Sofia Papagiannaki
        #insert or replace
806 70516d86 Sofia Papagiannaki
        #TODO better upsert
807 70516d86 Sofia Papagiannaki
        for k, v in items:
808 70516d86 Sofia Papagiannaki
            s = self.attributes.update()
809 70516d86 Sofia Papagiannaki
            s = s.where(and_(self.attributes.c.serial == serial,
810 059857e2 Antony Chazapis
                             self.attributes.c.domain == domain,
811 70516d86 Sofia Papagiannaki
                             self.attributes.c.key == k))
812 d50ed8d4 Sofia Papagiannaki
            s = s.values(value=v)
813 70516d86 Sofia Papagiannaki
            rp = self.conn.execute(s)
814 70516d86 Sofia Papagiannaki
            rp.close()
815 70516d86 Sofia Papagiannaki
            if rp.rowcount == 0:
816 70516d86 Sofia Papagiannaki
                s = self.attributes.insert()
817 059857e2 Antony Chazapis
                s = s.values(serial=serial, domain=domain, key=k, value=v)
818 70516d86 Sofia Papagiannaki
                self.conn.execute(s).close()
819 d50ed8d4 Sofia Papagiannaki
820 059857e2 Antony Chazapis
    def attribute_del(self, serial, domain, keys=()):
821 4f917833 Sofia Papagiannaki
        """Delete attributes of the version specified by serial.
822 4f917833 Sofia Papagiannaki
           If keys is empty, delete all attributes.
823 4f917833 Sofia Papagiannaki
           Otherwise delete those specified.
824 4f917833 Sofia Papagiannaki
        """
825 d50ed8d4 Sofia Papagiannaki
826 4f917833 Sofia Papagiannaki
        if keys:
827 b43d44ad Sofia Papagiannaki
            #TODO more efficient way to do this?
828 b43d44ad Sofia Papagiannaki
            for key in keys:
829 b43d44ad Sofia Papagiannaki
                s = self.attributes.delete()
830 b43d44ad Sofia Papagiannaki
                s = s.where(and_(self.attributes.c.serial == serial,
831 059857e2 Antony Chazapis
                                 self.attributes.c.domain == domain,
832 b43d44ad Sofia Papagiannaki
                                 self.attributes.c.key == key))
833 b43d44ad Sofia Papagiannaki
                self.conn.execute(s).close()
834 4f917833 Sofia Papagiannaki
        else:
835 b43d44ad Sofia Papagiannaki
            s = self.attributes.delete()
836 059857e2 Antony Chazapis
            s = s.where(and_(self.attributes.c.serial == serial,
837 059857e2 Antony Chazapis
                             self.attributes.c.domain == domain))
838 b43d44ad Sofia Papagiannaki
            self.conn.execute(s).close()
839 d50ed8d4 Sofia Papagiannaki
840 4f917833 Sofia Papagiannaki
    def attribute_copy(self, source, dest):
841 d50ed8d4 Sofia Papagiannaki
        s = select(
842 d50ed8d4 Sofia Papagiannaki
            [dest, self.attributes.c.domain,
843 d50ed8d4 Sofia Papagiannaki
                self.attributes.c.key, self.attributes.c.value],
844 b43d44ad Sofia Papagiannaki
            self.attributes.c.serial == source)
845 1b61dbc0 Sofia Papagiannaki
        rp = self.conn.execute(s)
846 1b61dbc0 Sofia Papagiannaki
        attributes = rp.fetchall()
847 1b61dbc0 Sofia Papagiannaki
        rp.close()
848 059857e2 Antony Chazapis
        for dest, domain, k, v in attributes:
849 dc9e6086 Sofia Papagiannaki
            #insert or replace
850 1b61dbc0 Sofia Papagiannaki
            s = self.attributes.update().where(and_(
851 1b61dbc0 Sofia Papagiannaki
                self.attributes.c.serial == dest,
852 059857e2 Antony Chazapis
                self.attributes.c.domain == domain,
853 1b61dbc0 Sofia Papagiannaki
                self.attributes.c.key == k))
854 1b61dbc0 Sofia Papagiannaki
            rp = self.conn.execute(s, value=v)
855 1b61dbc0 Sofia Papagiannaki
            rp.close()
856 1b61dbc0 Sofia Papagiannaki
            if rp.rowcount == 0:
857 1b61dbc0 Sofia Papagiannaki
                s = self.attributes.insert()
858 d50ed8d4 Sofia Papagiannaki
                values = {'serial': dest, 'domain': domain,
859 d50ed8d4 Sofia Papagiannaki
                          'key': k, 'value': v}
860 1b61dbc0 Sofia Papagiannaki
                self.conn.execute(s, values).close()
861 d50ed8d4 Sofia Papagiannaki
862 059857e2 Antony Chazapis
    def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):
863 4f917833 Sofia Papagiannaki
        """Return a list with all keys pairs defined
864 4f917833 Sofia Papagiannaki
           for all latest versions under parent that
865 4f917833 Sofia Papagiannaki
           do not belong to the cluster.
866 4f917833 Sofia Papagiannaki
        """
867 d50ed8d4 Sofia Papagiannaki
868 4f917833 Sofia Papagiannaki
        # TODO: Use another table to store before=inf results.
869 b43d44ad Sofia Papagiannaki
        a = self.attributes.alias('a')
870 b43d44ad Sofia Papagiannaki
        v = self.versions.alias('v')
871 b43d44ad Sofia Papagiannaki
        n = self.nodes.alias('n')
872 b43d44ad Sofia Papagiannaki
        s = select([a.c.key]).distinct()
873 62d938dc Sofia Papagiannaki
        if before != inf:
874 585b75e7 Sofia Papagiannaki
            filtered = select([func.max(self.versions.c.serial)])
875 62d938dc Sofia Papagiannaki
            filtered = filtered.where(self.versions.c.mtime < before)
876 585b75e7 Sofia Papagiannaki
            filtered = filtered.where(self.versions.c.node == v.c.node)
877 585b75e7 Sofia Papagiannaki
        else:
878 585b75e7 Sofia Papagiannaki
            filtered = select([self.nodes.c.latest_version])
879 585b75e7 Sofia Papagiannaki
            filtered = filtered.where(self.nodes.c.node == v.c.node)
880 585b75e7 Sofia Papagiannaki
        s = s.where(v.c.serial == filtered)
881 b43d44ad Sofia Papagiannaki
        s = s.where(v.c.cluster != except_cluster)
882 b43d44ad Sofia Papagiannaki
        s = s.where(v.c.node.in_(select([self.nodes.c.node],
883 d50ed8d4 Sofia Papagiannaki
                                        self.nodes.c.parent == parent)))
884 b43d44ad Sofia Papagiannaki
        s = s.where(a.c.serial == v.c.serial)
885 059857e2 Antony Chazapis
        s = s.where(a.c.domain == domain)
886 b43d44ad Sofia Papagiannaki
        s = s.where(n.c.node == v.c.node)
887 b43d44ad Sofia Papagiannaki
        conj = []
888 cf341da4 Antony Chazapis
        for path, match in pathq:
889 cf341da4 Antony Chazapis
            if match == MATCH_PREFIX:
890 d50ed8d4 Sofia Papagiannaki
                conj.append(
891 d50ed8d4 Sofia Papagiannaki
                    n.c.path.like(self.escape_like(path) + '%', escape='\\'))
892 cf341da4 Antony Chazapis
            elif match == MATCH_EXACT:
893 cf341da4 Antony Chazapis
                conj.append(n.c.path == path)
894 b43d44ad Sofia Papagiannaki
        if conj:
895 b43d44ad Sofia Papagiannaki
            s = s.where(or_(*conj))
896 b43d44ad Sofia Papagiannaki
        rp = self.conn.execute(s)
897 70516d86 Sofia Papagiannaki
        rows = rp.fetchall()
898 b43d44ad Sofia Papagiannaki
        rp.close()
899 70516d86 Sofia Papagiannaki
        return [r[0] for r in rows]
900 d50ed8d4 Sofia Papagiannaki
901 4f917833 Sofia Papagiannaki
    def latest_version_list(self, parent, prefix='', delimiter=None,
902 4f917833 Sofia Papagiannaki
                            start='', limit=10000, before=inf,
903 371d907a Antony Chazapis
                            except_cluster=0, pathq=[], domain=None,
904 371d907a Antony Chazapis
                            filterq=[], sizeq=None, all_props=False):
905 4f917833 Sofia Papagiannaki
        """Return a (list of (path, serial) tuples, list of common prefixes)
906 4f917833 Sofia Papagiannaki
           for the current versions of the paths with the given parent,
907 4f917833 Sofia Papagiannaki
           matching the following criteria.
908 d50ed8d4 Sofia Papagiannaki

909 4f917833 Sofia Papagiannaki
           The property tuple for a version is returned if all
910 4f917833 Sofia Papagiannaki
           of these conditions are true:
911 d50ed8d4 Sofia Papagiannaki

912 4f917833 Sofia Papagiannaki
                a. parent matches
913 d50ed8d4 Sofia Papagiannaki

914 4f917833 Sofia Papagiannaki
                b. path > start
915 d50ed8d4 Sofia Papagiannaki

916 4f917833 Sofia Papagiannaki
                c. path starts with prefix (and paths in pathq)
917 d50ed8d4 Sofia Papagiannaki

918 4f917833 Sofia Papagiannaki
                d. version is the max up to before
919 d50ed8d4 Sofia Papagiannaki

920 4f917833 Sofia Papagiannaki
                e. version is not in cluster
921 d50ed8d4 Sofia Papagiannaki

922 4f917833 Sofia Papagiannaki
                f. the path does not have the delimiter occuring
923 4f917833 Sofia Papagiannaki
                   after the prefix, or ends with the delimiter
924 d50ed8d4 Sofia Papagiannaki

925 4f917833 Sofia Papagiannaki
                g. serial matches the attribute filter query.
926 d50ed8d4 Sofia Papagiannaki

927 4f917833 Sofia Papagiannaki
                   A filter query is a comma-separated list of
928 4f917833 Sofia Papagiannaki
                   terms in one of these three forms:
929 d50ed8d4 Sofia Papagiannaki

930 4f917833 Sofia Papagiannaki
                   key
931 4f917833 Sofia Papagiannaki
                       an attribute with this key must exist
932 d50ed8d4 Sofia Papagiannaki

933 4f917833 Sofia Papagiannaki
                   !key
934 4f917833 Sofia Papagiannaki
                       an attribute with this key must not exist
935 d50ed8d4 Sofia Papagiannaki

936 4f917833 Sofia Papagiannaki
                   key ?op value
937 4f917833 Sofia Papagiannaki
                       the attribute with this key satisfies the value
938 4f917833 Sofia Papagiannaki
                       where ?op is one of ==, != <=, >=, <, >.
939 d50ed8d4 Sofia Papagiannaki

940 7ff57991 Antony Chazapis
                h. the size is in the range set by sizeq
941 d50ed8d4 Sofia Papagiannaki

942 4f917833 Sofia Papagiannaki
           The list of common prefixes includes the prefixes
943 4f917833 Sofia Papagiannaki
           matching up to the first delimiter after prefix,
944 4f917833 Sofia Papagiannaki
           and are reported only once, as "virtual directories".
945 4f917833 Sofia Papagiannaki
           The delimiter is included in the prefixes.
946 d50ed8d4 Sofia Papagiannaki

947 4f917833 Sofia Papagiannaki
           If arguments are None, then the corresponding matching rule
948 4f917833 Sofia Papagiannaki
           will always match.
949 d50ed8d4 Sofia Papagiannaki

950 4f917833 Sofia Papagiannaki
           Limit applies to the first list of tuples returned.
951 d50ed8d4 Sofia Papagiannaki

952 371d907a Antony Chazapis
           If all_props is True, return all properties after path, not just serial.
953 4f917833 Sofia Papagiannaki
        """
954 d50ed8d4 Sofia Papagiannaki
955 4f917833 Sofia Papagiannaki
        if not start or start < prefix:
956 4f917833 Sofia Papagiannaki
            start = strprevling(prefix)
957 4f917833 Sofia Papagiannaki
        nextling = strnextling(prefix)
958 d50ed8d4 Sofia Papagiannaki
959 b43d44ad Sofia Papagiannaki
        v = self.versions.alias('v')
960 b43d44ad Sofia Papagiannaki
        n = self.nodes.alias('n')
961 371d907a Antony Chazapis
        if not all_props:
962 371d907a Antony Chazapis
            s = select([n.c.path, v.c.serial]).distinct()
963 371d907a Antony Chazapis
        else:
964 371d907a Antony Chazapis
            s = select([n.c.path,
965 371d907a Antony Chazapis
                        v.c.serial, v.c.node, v.c.hash,
966 371d907a Antony Chazapis
                        v.c.size, v.c.type, v.c.source,
967 371d907a Antony Chazapis
                        v.c.mtime, v.c.muser, v.c.uuid,
968 371d907a Antony Chazapis
                        v.c.checksum, v.c.cluster]).distinct()
969 62d938dc Sofia Papagiannaki
        if before != inf:
970 585b75e7 Sofia Papagiannaki
            filtered = select([func.max(self.versions.c.serial)])
971 62d938dc Sofia Papagiannaki
            filtered = filtered.where(self.versions.c.mtime < before)
972 585b75e7 Sofia Papagiannaki
        else:
973 585b75e7 Sofia Papagiannaki
            filtered = select([self.nodes.c.latest_version])
974 d50ed8d4 Sofia Papagiannaki
        s = s.where(
975 d50ed8d4 Sofia Papagiannaki
            v.c.serial == filtered.where(self.nodes.c.node == v.c.node))
976 b43d44ad Sofia Papagiannaki
        s = s.where(v.c.cluster != except_cluster)
977 b43d44ad Sofia Papagiannaki
        s = s.where(v.c.node.in_(select([self.nodes.c.node],
978 d50ed8d4 Sofia Papagiannaki
                                        self.nodes.c.parent == parent)))
979 d50ed8d4 Sofia Papagiannaki
980 b43d44ad Sofia Papagiannaki
        s = s.where(n.c.node == v.c.node)
981 b43d44ad Sofia Papagiannaki
        s = s.where(and_(n.c.path > bindparam('start'), n.c.path < nextling))
982 b43d44ad Sofia Papagiannaki
        conj = []
983 cf341da4 Antony Chazapis
        for path, match in pathq:
984 cf341da4 Antony Chazapis
            if match == MATCH_PREFIX:
985 d50ed8d4 Sofia Papagiannaki
                conj.append(
986 d50ed8d4 Sofia Papagiannaki
                    n.c.path.like(self.escape_like(path) + '%', escape='\\'))
987 cf341da4 Antony Chazapis
            elif match == MATCH_EXACT:
988 cf341da4 Antony Chazapis
                conj.append(n.c.path == path)
989 b43d44ad Sofia Papagiannaki
        if conj:
990 b43d44ad Sofia Papagiannaki
            s = s.where(or_(*conj))
991 d50ed8d4 Sofia Papagiannaki
992 7ff57991 Antony Chazapis
        if sizeq and len(sizeq) == 2:
993 7ff57991 Antony Chazapis
            if sizeq[0]:
994 7ff57991 Antony Chazapis
                s = s.where(v.c.size >= sizeq[0])
995 7ff57991 Antony Chazapis
            if sizeq[1]:
996 7ff57991 Antony Chazapis
                s = s.where(v.c.size < sizeq[1])
997 d50ed8d4 Sofia Papagiannaki
998 9d41b050 chazapis
        if domain and filterq:
999 a847494c chazapis
            a = self.attributes.alias('a')
1000 3d13f97a Sofia Papagiannaki
            included, excluded, opers = parse_filters(filterq)
1001 3d13f97a Sofia Papagiannaki
            if included:
1002 a847494c chazapis
                subs = select([1])
1003 8877529a chazapis
                subs = subs.where(a.c.serial == v.c.serial).correlate(v)
1004 a847494c chazapis
                subs = subs.where(a.c.domain == domain)
1005 a847494c chazapis
                subs = subs.where(or_(*[a.c.key.op('=')(x) for x in included]))
1006 a847494c chazapis
                s = s.where(exists(subs))
1007 3d13f97a Sofia Papagiannaki
            if excluded:
1008 a847494c chazapis
                subs = select([1])
1009 8877529a chazapis
                subs = subs.where(a.c.serial == v.c.serial).correlate(v)
1010 a847494c chazapis
                subs = subs.where(a.c.domain == domain)
1011 a847494c chazapis
                subs = subs.where(or_(*[a.c.key.op('=')(x) for x in excluded]))
1012 a847494c chazapis
                s = s.where(not_(exists(subs)))
1013 9d41b050 chazapis
            if opers:
1014 6b2f11b4 Antony Chazapis
                for k, o, val in opers:
1015 6b2f11b4 Antony Chazapis
                    subs = select([1])
1016 6b2f11b4 Antony Chazapis
                    subs = subs.where(a.c.serial == v.c.serial).correlate(v)
1017 6b2f11b4 Antony Chazapis
                    subs = subs.where(a.c.domain == domain)
1018 d50ed8d4 Sofia Papagiannaki
                    subs = subs.where(
1019 d50ed8d4 Sofia Papagiannaki
                        and_(a.c.key.op('=')(k), a.c.value.op(o)(val)))
1020 6b2f11b4 Antony Chazapis
                    s = s.where(exists(subs))
1021 d50ed8d4 Sofia Papagiannaki
1022 9d41b050 chazapis
        s = s.order_by(n.c.path)
1023 d50ed8d4 Sofia Papagiannaki
1024 4f917833 Sofia Papagiannaki
        if not delimiter:
1025 b43d44ad Sofia Papagiannaki
            s = s.limit(limit)
1026 b43d44ad Sofia Papagiannaki
            rp = self.conn.execute(s, start=start)
1027 9d41b050 chazapis
            r = rp.fetchall()
1028 b43d44ad Sofia Papagiannaki
            rp.close()
1029 b43d44ad Sofia Papagiannaki
            return r, ()
1030 d50ed8d4 Sofia Papagiannaki
1031 4f917833 Sofia Papagiannaki
        pfz = len(prefix)
1032 4f917833 Sofia Papagiannaki
        dz = len(delimiter)
1033 4f917833 Sofia Papagiannaki
        count = 0
1034 4f917833 Sofia Papagiannaki
        prefixes = []
1035 4f917833 Sofia Papagiannaki
        pappend = prefixes.append
1036 4f917833 Sofia Papagiannaki
        matches = []
1037 4f917833 Sofia Papagiannaki
        mappend = matches.append
1038 d50ed8d4 Sofia Papagiannaki
1039 b43d44ad Sofia Papagiannaki
        rp = self.conn.execute(s, start=start)
1040 4f917833 Sofia Papagiannaki
        while True:
1041 b43d44ad Sofia Papagiannaki
            props = rp.fetchone()
1042 4f917833 Sofia Papagiannaki
            if props is None:
1043 4f917833 Sofia Papagiannaki
                break
1044 371d907a Antony Chazapis
            path = props[0]
1045 371d907a Antony Chazapis
            serial = props[1]
1046 4f917833 Sofia Papagiannaki
            idx = path.find(delimiter, pfz)
1047 d50ed8d4 Sofia Papagiannaki
1048 4f917833 Sofia Papagiannaki
            if idx < 0:
1049 4f917833 Sofia Papagiannaki
                mappend(props)
1050 4f917833 Sofia Papagiannaki
                count += 1
1051 4f917833 Sofia Papagiannaki
                if count >= limit:
1052 4f917833 Sofia Papagiannaki
                    break
1053 4f917833 Sofia Papagiannaki
                continue
1054 d50ed8d4 Sofia Papagiannaki
1055 4f917833 Sofia Papagiannaki
            if idx + dz == len(path):
1056 4f917833 Sofia Papagiannaki
                mappend(props)
1057 4f917833 Sofia Papagiannaki
                count += 1
1058 d50ed8d4 Sofia Papagiannaki
                continue  # Get one more, in case there is a path.
1059 d3be972a Sofia Papagiannaki
            pf = path[:idx + dz]
1060 d3be972a Sofia Papagiannaki
            pappend(pf)
1061 d50ed8d4 Sofia Papagiannaki
            if count >= limit:
1062 4f917833 Sofia Papagiannaki
                break
1063 d50ed8d4 Sofia Papagiannaki
1064 d50ed8d4 Sofia Papagiannaki
            rp = self.conn.execute(s, start=strnextling(pf))  # New start.
1065 9eb713e1 Sofia Papagiannaki
        rp.close()
1066 d50ed8d4 Sofia Papagiannaki
1067 4f917833 Sofia Papagiannaki
        return matches, prefixes
1068 d50ed8d4 Sofia Papagiannaki
1069 37bee317 Antony Chazapis
    def latest_uuid(self, uuid):
1070 37bee317 Antony Chazapis
        """Return a (path, serial) tuple, for the latest version of the given uuid."""
1071 d50ed8d4 Sofia Papagiannaki
1072 37bee317 Antony Chazapis
        v = self.versions.alias('v')
1073 37bee317 Antony Chazapis
        n = self.nodes.alias('n')
1074 37bee317 Antony Chazapis
        s = select([n.c.path, v.c.serial])
1075 37bee317 Antony Chazapis
        filtered = select([func.max(self.versions.c.serial)])
1076 37bee317 Antony Chazapis
        s = s.where(v.c.serial == filtered.where(self.versions.c.uuid == uuid))
1077 37bee317 Antony Chazapis
        s = s.where(n.c.node == v.c.node)
1078 d50ed8d4 Sofia Papagiannaki
1079 37bee317 Antony Chazapis
        r = self.conn.execute(s)
1080 37bee317 Antony Chazapis
        l = r.fetchone()
1081 37bee317 Antony Chazapis
        r.close()
1082 37bee317 Antony Chazapis
        return l