Revision d50ed8d4 snf-pithos-backend/pithos/backends/lib/sqlite/node.py
b/snf-pithos-backend/pithos/backends/lib/sqlite/node.py | ||
---|---|---|
1 | 1 |
# Copyright 2011-2012 GRNET S.A. All rights reserved. |
2 |
#
|
|
2 |
# |
|
3 | 3 |
# Redistribution and use in source and binary forms, with or |
4 | 4 |
# without modification, are permitted provided that the following |
5 | 5 |
# conditions are met: |
6 |
#
|
|
6 |
# |
|
7 | 7 |
# 1. Redistributions of source code must retain the above |
8 | 8 |
# copyright notice, this list of conditions and the following |
9 | 9 |
# disclaimer. |
10 |
#
|
|
10 |
# |
|
11 | 11 |
# 2. Redistributions in binary form must reproduce the above |
12 | 12 |
# copyright notice, this list of conditions and the following |
13 | 13 |
# disclaimer in the documentation and/or other materials |
14 | 14 |
# provided with the distribution. |
15 |
#
|
|
15 |
# |
|
16 | 16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
17 | 17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
18 | 18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
... | ... | |
25 | 25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
26 | 26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
27 | 27 |
# POSSIBILITY OF SUCH DAMAGE. |
28 |
#
|
|
28 |
# |
|
29 | 29 |
# The views and conclusions contained in the software and |
30 | 30 |
# documentation are those of the authors and should not be |
31 | 31 |
# interpreted as representing official policies, either expressed |
... | ... | |
38 | 38 |
from pithos.backends.filter import parse_filters |
39 | 39 |
|
40 | 40 |
|
41 |
ROOTNODE = 0
|
|
41 |
ROOTNODE = 0 |
|
42 | 42 |
|
43 |
( SERIAL, NODE, HASH, SIZE, TYPE, SOURCE, MTIME, MUSER, UUID, CHECKSUM, CLUSTER ) = range(11) |
|
43 |
(SERIAL, NODE, HASH, SIZE, TYPE, SOURCE, MTIME, MUSER, UUID, CHECKSUM, |
|
44 |
CLUSTER) = range(11) |
|
44 | 45 |
|
45 |
( MATCH_PREFIX, MATCH_EXACT ) = range(2)
|
|
46 |
(MATCH_PREFIX, MATCH_EXACT) = range(2)
|
|
46 | 47 |
|
47 | 48 |
inf = float('inf') |
48 | 49 |
|
... | ... | |
64 | 65 |
c = ord(prefix[-1]) |
65 | 66 |
if c >= 0xffff: |
66 | 67 |
raise RuntimeError |
67 |
s += unichr(c+1)
|
|
68 |
s += unichr(c + 1)
|
|
68 | 69 |
return s |
69 | 70 |
|
71 |
|
|
70 | 72 |
def strprevling(prefix): |
71 | 73 |
"""Return an approximation of the last unicode string |
72 | 74 |
less than but not starting with given prefix. |
... | ... | |
78 | 80 |
s = prefix[:-1] |
79 | 81 |
c = ord(prefix[-1]) |
80 | 82 |
if c > 0: |
81 |
s += unichr(c-1) + unichr(0xffff)
|
|
83 |
s += unichr(c - 1) + unichr(0xffff)
|
|
82 | 84 |
return s |
83 | 85 |
|
84 | 86 |
|
85 | 87 |
_propnames = { |
86 |
'serial' : 0,
|
|
87 |
'node' : 1,
|
|
88 |
'hash' : 2,
|
|
89 |
'size' : 3,
|
|
90 |
'type' : 4,
|
|
91 |
'source' : 5,
|
|
92 |
'mtime' : 6,
|
|
93 |
'muser' : 7,
|
|
94 |
'uuid' : 8,
|
|
95 |
'checksum' : 9,
|
|
96 |
'cluster' : 10
|
|
88 |
'serial': 0, |
|
89 |
'node': 1, |
|
90 |
'hash': 2, |
|
91 |
'size': 3, |
|
92 |
'type': 4, |
|
93 |
'source': 5, |
|
94 |
'mtime': 6, |
|
95 |
'muser': 7, |
|
96 |
'uuid': 8, |
|
97 |
'checksum': 9, |
|
98 |
'cluster': 10 |
|
97 | 99 |
} |
98 | 100 |
|
99 | 101 |
|
... | ... | |
102 | 104 |
Versions store object history and have multiple attributes. |
103 | 105 |
Attributes store metadata. |
104 | 106 |
""" |
105 |
|
|
107 |
|
|
106 | 108 |
# TODO: Provide an interface for included and excluded clusters. |
107 |
|
|
109 |
|
|
108 | 110 |
def __init__(self, **params): |
109 | 111 |
DBWorker.__init__(self, **params) |
110 | 112 |
execute = self.execute |
111 |
|
|
113 |
|
|
112 | 114 |
execute(""" pragma foreign_keys = on """) |
113 |
|
|
115 |
|
|
114 | 116 |
execute(""" create table if not exists nodes |
115 | 117 |
( node integer primary key, |
116 | 118 |
parent integer default 0, |
... | ... | |
124 | 126 |
on nodes(path) """) |
125 | 127 |
execute(""" create index if not exists idx_nodes_parent |
126 | 128 |
on nodes(parent) """) |
127 |
|
|
129 |
|
|
128 | 130 |
execute(""" create table if not exists policy |
129 | 131 |
( node integer, |
130 | 132 |
key text, |
... | ... | |
134 | 136 |
references nodes(node) |
135 | 137 |
on update cascade |
136 | 138 |
on delete cascade ) """) |
137 |
|
|
139 |
|
|
138 | 140 |
execute(""" create table if not exists statistics |
139 | 141 |
( node integer, |
140 | 142 |
population integer not null default 0, |
... | ... | |
146 | 148 |
references nodes(node) |
147 | 149 |
on update cascade |
148 | 150 |
on delete cascade ) """) |
149 |
|
|
151 |
|
|
150 | 152 |
execute(""" create table if not exists versions |
151 | 153 |
( serial integer primary key, |
152 | 154 |
node integer, |
... | ... | |
167 | 169 |
on versions(node, mtime) """) |
168 | 170 |
execute(""" create index if not exists idx_versions_node_uuid |
169 | 171 |
on versions(uuid) """) |
170 |
|
|
172 |
|
|
171 | 173 |
execute(""" create table if not exists attributes |
172 | 174 |
( serial integer, |
173 | 175 |
domain text, |
... | ... | |
178 | 180 |
references versions(serial) |
179 | 181 |
on update cascade |
180 | 182 |
on delete cascade ) """) |
181 |
|
|
183 |
|
|
182 | 184 |
q = "insert or ignore into nodes(node, parent) values (?, ?)" |
183 | 185 |
execute(q, (ROOTNODE, ROOTNODE)) |
184 |
|
|
186 |
|
|
185 | 187 |
def node_create(self, parent, path): |
186 | 188 |
"""Create a new node from the given properties. |
187 | 189 |
Return the node identifier of the new node. |
188 | 190 |
""" |
189 |
|
|
191 |
|
|
190 | 192 |
q = ("insert into nodes (parent, path) " |
191 | 193 |
"values (?, ?)") |
192 | 194 |
props = (parent, path) |
193 | 195 |
return self.execute(q, props).lastrowid |
194 |
|
|
196 |
|
|
195 | 197 |
def node_lookup(self, path): |
196 | 198 |
"""Lookup the current node of the given path. |
197 | 199 |
Return None if the path is not found. |
198 | 200 |
""" |
199 |
|
|
201 |
|
|
200 | 202 |
q = "select node from nodes where path = ?" |
201 | 203 |
self.execute(q, (path,)) |
202 | 204 |
r = self.fetchone() |
203 | 205 |
if r is not None: |
204 | 206 |
return r[0] |
205 | 207 |
return None |
206 |
|
|
208 |
|
|
207 | 209 |
def node_lookup_bulk(self, paths): |
208 |
"""Lookup the current nodes for the given paths.
|
|
210 |
"""Lookup the current nodes for the given paths.
|
|
209 | 211 |
Return () if the path is not found. |
210 | 212 |
""" |
211 |
|
|
213 |
|
|
212 | 214 |
placeholders = ','.join('?' for path in paths) |
213 | 215 |
q = "select node from nodes where path in (%s)" % placeholders |
214 | 216 |
self.execute(q, paths) |
215 | 217 |
r = self.fetchall() |
216 | 218 |
if r is not None: |
217 |
return [row[0] for row in r]
|
|
219 |
return [row[0] for row in r]
|
|
218 | 220 |
return None |
219 |
|
|
221 |
|
|
220 | 222 |
def node_get_properties(self, node): |
221 | 223 |
"""Return the node's (parent, path). |
222 | 224 |
Return None if the node is not found. |
223 | 225 |
""" |
224 |
|
|
226 |
|
|
225 | 227 |
q = "select parent, path from nodes where node = ?" |
226 | 228 |
self.execute(q, (node,)) |
227 | 229 |
return self.fetchone() |
228 |
|
|
230 |
|
|
229 | 231 |
def node_get_versions(self, node, keys=(), propnames=_propnames): |
230 | 232 |
"""Return the properties of all versions at node. |
231 | 233 |
If keys is empty, return all properties in the order |
232 | 234 |
(serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster). |
233 | 235 |
""" |
234 |
|
|
236 |
|
|
235 | 237 |
q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster " |
236 | 238 |
"from versions " |
237 | 239 |
"where node = ?") |
... | ... | |
239 | 241 |
r = self.fetchall() |
240 | 242 |
if r is None: |
241 | 243 |
return r |
242 |
|
|
244 |
|
|
243 | 245 |
if not keys: |
244 | 246 |
return r |
245 | 247 |
return [[p[propnames[k]] for k in keys if k in propnames] for p in r] |
246 |
|
|
248 |
|
|
247 | 249 |
def node_count_children(self, node): |
248 | 250 |
"""Return node's child count.""" |
249 |
|
|
251 |
|
|
250 | 252 |
q = "select count(node) from nodes where parent = ? and node != 0" |
251 | 253 |
self.execute(q, (node,)) |
252 | 254 |
r = self.fetchone() |
253 | 255 |
if r is None: |
254 | 256 |
return 0 |
255 | 257 |
return r[0] |
256 |
|
|
258 |
|
|
257 | 259 |
def node_purge_children(self, parent, before=inf, cluster=0): |
258 | 260 |
"""Delete all versions with the specified |
259 | 261 |
parent and cluster, and return |
260 | 262 |
the hashes and size of versions deleted. |
261 | 263 |
Clears out nodes with no remaining versions. |
262 | 264 |
""" |
263 |
|
|
265 |
|
|
264 | 266 |
execute = self.execute |
265 | 267 |
q = ("select count(serial), sum(size) from versions " |
266 | 268 |
"where node in (select node " |
267 |
"from nodes "
|
|
268 |
"where parent = ?) "
|
|
269 |
"from nodes " |
|
270 |
"where parent = ?) " |
|
269 | 271 |
"and cluster = ? " |
270 | 272 |
"and mtime <= ?") |
271 | 273 |
args = (parent, cluster, before) |
... | ... | |
276 | 278 |
mtime = time() |
277 | 279 |
self.statistics_update(parent, -nr, -size, mtime, cluster) |
278 | 280 |
self.statistics_update_ancestors(parent, -nr, -size, mtime, cluster) |
279 |
|
|
281 |
|
|
280 | 282 |
q = ("select hash from versions " |
281 | 283 |
"where node in (select node " |
282 |
"from nodes "
|
|
283 |
"where parent = ?) "
|
|
284 |
"from nodes " |
|
285 |
"where parent = ?) " |
|
284 | 286 |
"and cluster = ? " |
285 | 287 |
"and mtime <= ?") |
286 | 288 |
execute(q, args) |
287 | 289 |
hashes = [r[0] for r in self.fetchall()] |
288 | 290 |
q = ("delete from versions " |
289 | 291 |
"where node in (select node " |
290 |
"from nodes "
|
|
291 |
"where parent = ?) "
|
|
292 |
"from nodes " |
|
293 |
"where parent = ?) " |
|
292 | 294 |
"and cluster = ? " |
293 | 295 |
"and mtime <= ?") |
294 | 296 |
execute(q, args) |
295 | 297 |
q = ("delete from nodes " |
296 | 298 |
"where node in (select node from nodes n " |
297 |
"where (select count(serial) "
|
|
298 |
"from versions "
|
|
299 |
"where node = n.node) = 0 "
|
|
300 |
"and parent = ?)")
|
|
299 |
"where (select count(serial) " |
|
300 |
"from versions " |
|
301 |
"where node = n.node) = 0 " |
|
302 |
"and parent = ?)") |
|
301 | 303 |
execute(q, (parent,)) |
302 | 304 |
return hashes, size |
303 |
|
|
305 |
|
|
304 | 306 |
def node_purge(self, node, before=inf, cluster=0): |
305 | 307 |
"""Delete all versions with the specified |
306 | 308 |
node and cluster, and return |
307 | 309 |
the hashes and size of versions deleted. |
308 | 310 |
Clears out the node if it has no remaining versions. |
309 | 311 |
""" |
310 |
|
|
312 |
|
|
311 | 313 |
execute = self.execute |
312 | 314 |
q = ("select count(serial), sum(size) from versions " |
313 | 315 |
"where node = ? " |
... | ... | |
320 | 322 |
return (), 0 |
321 | 323 |
mtime = time() |
322 | 324 |
self.statistics_update_ancestors(node, -nr, -size, mtime, cluster) |
323 |
|
|
325 |
|
|
324 | 326 |
q = ("select hash from versions " |
325 | 327 |
"where node = ? " |
326 | 328 |
"and cluster = ? " |
... | ... | |
334 | 336 |
execute(q, args) |
335 | 337 |
q = ("delete from nodes " |
336 | 338 |
"where node in (select node from nodes n " |
337 |
"where (select count(serial) "
|
|
338 |
"from versions "
|
|
339 |
"where node = n.node) = 0 "
|
|
340 |
"and node = ?)")
|
|
339 |
"where (select count(serial) " |
|
340 |
"from versions " |
|
341 |
"where node = n.node) = 0 " |
|
342 |
"and node = ?)") |
|
341 | 343 |
execute(q, (node,)) |
342 | 344 |
return hashes, size |
343 |
|
|
345 |
|
|
344 | 346 |
def node_remove(self, node): |
345 | 347 |
"""Remove the node specified. |
346 | 348 |
Return false if the node has children or is not found. |
347 | 349 |
""" |
348 |
|
|
350 |
|
|
349 | 351 |
if self.node_count_children(node): |
350 | 352 |
return False |
351 |
|
|
353 |
|
|
352 | 354 |
mtime = time() |
353 | 355 |
q = ("select count(serial), sum(size), cluster " |
354 | 356 |
"from versions " |
... | ... | |
356 | 358 |
"group by cluster") |
357 | 359 |
self.execute(q, (node,)) |
358 | 360 |
for population, size, cluster in self.fetchall(): |
359 |
self.statistics_update_ancestors(node, -population, -size, mtime, cluster) |
|
360 |
|
|
361 |
self.statistics_update_ancestors( |
|
362 |
node, -population, -size, mtime, cluster) |
|
363 |
|
|
361 | 364 |
q = "delete from nodes where node = ?" |
362 | 365 |
self.execute(q, (node,)) |
363 | 366 |
return True |
364 |
|
|
367 |
|
|
365 | 368 |
def policy_get(self, node): |
366 | 369 |
q = "select key, value from policy where node = ?" |
367 | 370 |
self.execute(q, (node,)) |
368 | 371 |
return dict(self.fetchall()) |
369 |
|
|
372 |
|
|
370 | 373 |
def policy_set(self, node, policy): |
371 | 374 |
q = "insert or replace into policy (node, key, value) values (?, ?, ?)" |
372 | 375 |
self.executemany(q, ((node, k, v) for k, v in policy.iteritems())) |
373 |
|
|
376 |
|
|
374 | 377 |
def statistics_get(self, node, cluster=0): |
375 | 378 |
"""Return population, total size and last mtime |
376 | 379 |
for all versions under node that belong to the cluster. |
377 | 380 |
""" |
378 |
|
|
381 |
|
|
379 | 382 |
q = ("select population, size, mtime from statistics " |
380 | 383 |
"where node = ? and cluster = ?") |
381 | 384 |
self.execute(q, (node, cluster)) |
382 | 385 |
return self.fetchone() |
383 |
|
|
386 |
|
|
384 | 387 |
def statistics_update(self, node, population, size, mtime, cluster=0): |
385 | 388 |
"""Update the statistics of the given node. |
386 | 389 |
Statistics keep track the population, total |
387 | 390 |
size of objects and mtime in the node's namespace. |
388 | 391 |
May be zero or positive or negative numbers. |
389 | 392 |
""" |
390 |
|
|
393 |
|
|
391 | 394 |
qs = ("select population, size from statistics " |
392 | 395 |
"where node = ? and cluster = ?") |
393 | 396 |
qu = ("insert or replace into statistics (node, population, size, mtime, cluster) " |
... | ... | |
401 | 404 |
population += prepopulation |
402 | 405 |
size += presize |
403 | 406 |
self.execute(qu, (node, population, size, mtime, cluster)) |
404 |
|
|
407 |
|
|
405 | 408 |
def statistics_update_ancestors(self, node, population, size, mtime, cluster=0): |
406 | 409 |
"""Update the statistics of the given node's parent. |
407 | 410 |
Then recursively update all parents up to the root. |
408 | 411 |
Population is not recursive. |
409 | 412 |
""" |
410 |
|
|
413 |
|
|
411 | 414 |
while True: |
412 | 415 |
if node == 0: |
413 | 416 |
break |
... | ... | |
417 | 420 |
parent, path = props |
418 | 421 |
self.statistics_update(parent, population, size, mtime, cluster) |
419 | 422 |
node = parent |
420 |
population = 0 # Population isn't recursive |
|
421 |
|
|
423 |
population = 0 # Population isn't recursive
|
|
424 |
|
|
422 | 425 |
def statistics_latest(self, node, before=inf, except_cluster=0): |
423 | 426 |
"""Return population, total size and last mtime |
424 | 427 |
for all latest versions under node that |
425 | 428 |
do not belong to the cluster. |
426 | 429 |
""" |
427 |
|
|
430 |
|
|
428 | 431 |
execute = self.execute |
429 | 432 |
fetchone = self.fetchone |
430 |
|
|
433 |
|
|
431 | 434 |
# The node. |
432 | 435 |
props = self.node_get_properties(node) |
433 | 436 |
if props is None: |
434 | 437 |
return None |
435 | 438 |
parent, path = props |
436 |
|
|
439 |
|
|
437 | 440 |
# The latest version. |
438 | 441 |
q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster " |
439 | 442 |
"from versions v " |
440 | 443 |
"where serial = %s " |
441 | 444 |
"and cluster != ?") |
442 |
subq, args = self._construct_latest_version_subquery(node=node, before=before) |
|
445 |
subq, args = self._construct_latest_version_subquery( |
|
446 |
node=node, before=before) |
|
443 | 447 |
execute(q % subq, args + [except_cluster]) |
444 | 448 |
props = fetchone() |
445 | 449 |
if props is None: |
446 | 450 |
return None |
447 | 451 |
mtime = props[MTIME] |
448 |
|
|
452 |
|
|
449 | 453 |
# First level, just under node (get population). |
450 | 454 |
q = ("select count(serial), sum(size), max(mtime) " |
451 | 455 |
"from versions v " |
452 | 456 |
"where serial = %s " |
453 | 457 |
"and cluster != ? " |
454 | 458 |
"and node in (select node " |
455 |
"from nodes " |
|
456 |
"where parent = ?)") |
|
457 |
subq, args = self._construct_latest_version_subquery(node=None, before=before) |
|
459 |
"from nodes " |
|
460 |
"where parent = ?)") |
|
461 |
subq, args = self._construct_latest_version_subquery( |
|
462 |
node=None, before=before) |
|
458 | 463 |
execute(q % subq, args + [except_cluster, node]) |
459 | 464 |
r = fetchone() |
460 | 465 |
if r is None: |
... | ... | |
463 | 468 |
mtime = max(mtime, r[2]) |
464 | 469 |
if count == 0: |
465 | 470 |
return (0, 0, mtime) |
466 |
|
|
471 |
|
|
467 | 472 |
# All children (get size and mtime). |
468 | 473 |
# This is why the full path is stored. |
469 | 474 |
q = ("select count(serial), sum(size), max(mtime) " |
... | ... | |
471 | 476 |
"where serial = %s " |
472 | 477 |
"and cluster != ? " |
473 | 478 |
"and node in (select node " |
474 |
"from nodes " |
|
475 |
"where path like ? escape '\\')") |
|
476 |
subq, args = self._construct_latest_version_subquery(node=None, before=before) |
|
477 |
execute(q % subq, args + [except_cluster, self.escape_like(path) + '%']) |
|
479 |
"from nodes " |
|
480 |
"where path like ? escape '\\')") |
|
481 |
subq, args = self._construct_latest_version_subquery( |
|
482 |
node=None, before=before) |
|
483 |
execute( |
|
484 |
q % subq, args + [except_cluster, self.escape_like(path) + '%']) |
|
478 | 485 |
r = fetchone() |
479 | 486 |
if r is None: |
480 | 487 |
return None |
481 | 488 |
size = r[1] - props[SIZE] |
482 | 489 |
mtime = max(mtime, r[2]) |
483 | 490 |
return (count, size, mtime) |
484 |
|
|
491 |
|
|
485 | 492 |
def nodes_set_latest_version(self, node, serial): |
486 |
q = ("update nodes set latest_version = ? where node = ?")
|
|
493 |
q = ("update nodes set latest_version = ? where node = ?")
|
|
487 | 494 |
props = (serial, node) |
488 | 495 |
self.execute(q, props) |
489 |
|
|
496 |
|
|
490 | 497 |
def version_create(self, node, hash, size, type, source, muser, uuid, checksum, cluster=0): |
491 | 498 |
"""Create a new version from the given properties. |
492 | 499 |
Return the (serial, mtime) of the new version. |
493 | 500 |
""" |
494 |
|
|
501 |
|
|
495 | 502 |
q = ("insert into versions (node, hash, size, type, source, mtime, muser, uuid, checksum, cluster) " |
496 | 503 |
"values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") |
497 | 504 |
mtime = time() |
498 |
props = (node, hash, size, type, source, mtime, muser, uuid, checksum, cluster) |
|
505 |
props = (node, hash, size, type, source, mtime, muser, |
|
506 |
uuid, checksum, cluster) |
|
499 | 507 |
serial = self.execute(q, props).lastrowid |
500 | 508 |
self.statistics_update_ancestors(node, 1, size, mtime, cluster) |
501 |
|
|
509 |
|
|
502 | 510 |
self.nodes_set_latest_version(node, serial) |
503 |
|
|
511 |
|
|
504 | 512 |
return serial, mtime |
505 |
|
|
513 |
|
|
506 | 514 |
def version_lookup(self, node, before=inf, cluster=0, all_props=True): |
507 | 515 |
"""Lookup the current version of the given node. |
508 | 516 |
Return a list with its properties: |
509 |
(serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster) |
|
517 |
(serial, node, hash, size, type, source, mtime, |
|
518 |
muser, uuid, checksum, cluster) |
|
510 | 519 |
or None if the current version is not found in the given cluster. |
511 | 520 |
""" |
512 |
|
|
521 |
|
|
513 | 522 |
q = ("select %s " |
514 | 523 |
"from versions v " |
515 | 524 |
"where serial = %s " |
516 | 525 |
"and cluster = ?") |
517 |
subq, args = self._construct_latest_version_subquery(node=node, before=before) |
|
526 |
subq, args = self._construct_latest_version_subquery( |
|
527 |
node=node, before=before) |
|
518 | 528 |
if not all_props: |
519 | 529 |
q = q % ("serial", subq) |
520 | 530 |
else: |
521 | 531 |
q = q % ("serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster", subq) |
522 |
|
|
532 |
|
|
523 | 533 |
self.execute(q, args + [cluster]) |
524 | 534 |
props = self.fetchone() |
525 | 535 |
if props is not None: |
... | ... | |
531 | 541 |
Return a list with their properties: |
532 | 542 |
(serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster). |
533 | 543 |
""" |
534 |
|
|
544 |
|
|
535 | 545 |
if not nodes: |
536 |
return ()
|
|
546 |
return ()
|
|
537 | 547 |
q = ("select %s " |
538 | 548 |
"from versions " |
539 | 549 |
"where serial in %s " |
540 | 550 |
"and cluster = ? %s") |
541 |
subq, args = self._construct_latest_versions_subquery(nodes=nodes, before = before) |
|
551 |
subq, args = self._construct_latest_versions_subquery( |
|
552 |
nodes=nodes, before=before) |
|
542 | 553 |
if not all_props: |
543 | 554 |
q = q % ("serial", subq, '') |
544 | 555 |
else: |
545 |
q = q % ("serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster", subq, 'order by node')
|
|
546 |
|
|
556 |
q = q % ("serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster", subq, 'order by node') |
|
557 |
|
|
547 | 558 |
args += [cluster] |
548 | 559 |
self.execute(q, args) |
549 | 560 |
return self.fetchall() |
550 |
|
|
561 |
|
|
551 | 562 |
def version_get_properties(self, serial, keys=(), propnames=_propnames): |
552 | 563 |
"""Return a sequence of values for the properties of |
553 | 564 |
the version specified by serial and the keys, in the order given. |
554 | 565 |
If keys is empty, return all properties in the order |
555 | 566 |
(serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster). |
556 | 567 |
""" |
557 |
|
|
568 |
|
|
558 | 569 |
q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster " |
559 | 570 |
"from versions " |
560 | 571 |
"where serial = ?") |
... | ... | |
562 | 573 |
r = self.fetchone() |
563 | 574 |
if r is None: |
564 | 575 |
return r |
565 |
|
|
576 |
|
|
566 | 577 |
if not keys: |
567 | 578 |
return r |
568 | 579 |
return [r[propnames[k]] for k in keys if k in propnames] |
569 |
|
|
580 |
|
|
570 | 581 |
def version_put_property(self, serial, key, value): |
571 | 582 |
"""Set value for the property of version specified by key.""" |
572 |
|
|
583 |
|
|
573 | 584 |
if key not in _propnames: |
574 | 585 |
return |
575 | 586 |
q = "update versions set %s = ? where serial = ?" % key |
576 | 587 |
self.execute(q, (value, serial)) |
577 |
|
|
588 |
|
|
578 | 589 |
def version_recluster(self, serial, cluster): |
579 | 590 |
"""Move the version into another cluster.""" |
580 |
|
|
591 |
|
|
581 | 592 |
props = self.version_get_properties(serial) |
582 | 593 |
if not props: |
583 | 594 |
return |
... | ... | |
586 | 597 |
oldcluster = props[CLUSTER] |
587 | 598 |
if cluster == oldcluster: |
588 | 599 |
return |
589 |
|
|
600 |
|
|
590 | 601 |
mtime = time() |
591 | 602 |
self.statistics_update_ancestors(node, -1, -size, mtime, oldcluster) |
592 | 603 |
self.statistics_update_ancestors(node, 1, size, mtime, cluster) |
593 |
|
|
604 |
|
|
594 | 605 |
q = "update versions set cluster = ? where serial = ?" |
595 | 606 |
self.execute(q, (cluster, serial)) |
596 |
|
|
607 |
|
|
597 | 608 |
def version_remove(self, serial): |
598 | 609 |
"""Remove the serial specified.""" |
599 |
|
|
610 |
|
|
600 | 611 |
props = self.version_get_properties(serial) |
601 | 612 |
if not props: |
602 | 613 |
return |
... | ... | |
604 | 615 |
hash = props[HASH] |
605 | 616 |
size = props[SIZE] |
606 | 617 |
cluster = props[CLUSTER] |
607 |
|
|
618 |
|
|
608 | 619 |
mtime = time() |
609 | 620 |
self.statistics_update_ancestors(node, -1, -size, mtime, cluster) |
610 |
|
|
621 |
|
|
611 | 622 |
q = "delete from versions where serial = ?" |
612 | 623 |
self.execute(q, (serial,)) |
613 |
|
|
624 |
|
|
614 | 625 |
props = self.version_lookup(node, cluster=cluster, all_props=False) |
615 | 626 |
if props: |
616 |
self.nodes_set_latest_version(node, props[0])
|
|
627 |
self.nodes_set_latest_version(node, props[0])
|
|
617 | 628 |
return hash, size |
618 |
|
|
629 |
|
|
619 | 630 |
def attribute_get(self, serial, domain, keys=()): |
620 | 631 |
"""Return a list of (key, value) pairs of the version specified by serial. |
621 | 632 |
If keys is empty, return all attributes. |
622 | 633 |
Othwerise, return only those specified. |
623 | 634 |
""" |
624 |
|
|
635 |
|
|
625 | 636 |
execute = self.execute |
626 | 637 |
if keys: |
627 | 638 |
marks = ','.join('?' for k in keys) |
... | ... | |
632 | 643 |
q = "select key, value from attributes where serial = ? and domain = ?" |
633 | 644 |
execute(q, (serial, domain)) |
634 | 645 |
return self.fetchall() |
635 |
|
|
646 |
|
|
636 | 647 |
def attribute_set(self, serial, domain, items): |
637 | 648 |
"""Set the attributes of the version specified by serial. |
638 | 649 |
Receive attributes as an iterable of (key, value) pairs. |
639 | 650 |
""" |
640 |
|
|
651 |
|
|
641 | 652 |
q = ("insert or replace into attributes (serial, domain, key, value) " |
642 | 653 |
"values (?, ?, ?, ?)") |
643 | 654 |
self.executemany(q, ((serial, domain, k, v) for k, v in items)) |
644 |
|
|
655 |
|
|
645 | 656 |
def attribute_del(self, serial, domain, keys=()): |
646 | 657 |
"""Delete attributes of the version specified by serial. |
647 | 658 |
If keys is empty, delete all attributes. |
648 | 659 |
Otherwise delete those specified. |
649 | 660 |
""" |
650 |
|
|
661 |
|
|
651 | 662 |
if keys: |
652 | 663 |
q = "delete from attributes where serial = ? and domain = ? and key = ?" |
653 | 664 |
self.executemany(q, ((serial, domain, key) for key in keys)) |
654 | 665 |
else: |
655 | 666 |
q = "delete from attributes where serial = ? and domain = ?" |
656 | 667 |
self.execute(q, (serial, domain)) |
657 |
|
|
668 |
|
|
658 | 669 |
def attribute_copy(self, source, dest): |
659 | 670 |
q = ("insert or replace into attributes " |
660 | 671 |
"select ?, domain, key, value from attributes " |
661 | 672 |
"where serial = ?") |
662 | 673 |
self.execute(q, (dest, source)) |
663 |
|
|
674 |
|
|
664 | 675 |
def _construct_filters(self, domain, filterq): |
665 | 676 |
if not domain or not filterq: |
666 | 677 |
return None, None |
667 |
|
|
678 |
|
|
668 | 679 |
subqlist = [] |
669 | 680 |
append = subqlist.append |
670 | 681 |
included, excluded, opers = parse_filters(filterq) |
671 | 682 |
args = [] |
672 |
|
|
683 |
|
|
673 | 684 |
if included: |
674 | 685 |
subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and " |
675 | 686 |
subq += "(" + ' or '.join(('key = ?' for x in included)) + ")" |
... | ... | |
677 | 688 |
args += [domain] |
678 | 689 |
args += included |
679 | 690 |
append(subq) |
680 |
|
|
691 |
|
|
681 | 692 |
if excluded: |
682 | 693 |
subq = "not exists (select 1 from attributes where serial = v.serial and domain = ? and " |
683 | 694 |
subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")" |
... | ... | |
685 | 696 |
args += [domain] |
686 | 697 |
args += excluded |
687 | 698 |
append(subq) |
688 |
|
|
699 |
|
|
689 | 700 |
if opers: |
690 | 701 |
for k, o, v in opers: |
691 | 702 |
subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and " |
... | ... | |
693 | 704 |
subq += ")" |
694 | 705 |
args += [domain, k, v] |
695 | 706 |
append(subq) |
696 |
|
|
707 |
|
|
697 | 708 |
if not subqlist: |
698 | 709 |
return None, None |
699 |
|
|
710 |
|
|
700 | 711 |
subq = ' and ' + ' and '.join(subqlist) |
701 |
|
|
712 |
|
|
702 | 713 |
return subq, args |
703 |
|
|
714 |
|
|
704 | 715 |
def _construct_paths(self, pathq): |
705 | 716 |
if not pathq: |
706 | 717 |
return None, None |
707 |
|
|
718 |
|
|
708 | 719 |
subqlist = [] |
709 | 720 |
args = [] |
710 | 721 |
for path, match in pathq: |
... | ... | |
714 | 725 |
elif match == MATCH_EXACT: |
715 | 726 |
subqlist.append("n.path = ?") |
716 | 727 |
args.append(path) |
717 |
|
|
728 |
|
|
718 | 729 |
subq = ' and (' + ' or '.join(subqlist) + ')' |
719 | 730 |
args = tuple(args) |
720 |
|
|
731 |
|
|
721 | 732 |
return subq, args |
722 |
|
|
733 |
|
|
723 | 734 |
def _construct_size(self, sizeq): |
724 | 735 |
if not sizeq or len(sizeq) != 2: |
725 | 736 |
return None, None |
726 |
|
|
737 |
|
|
727 | 738 |
subq = '' |
728 | 739 |
args = [] |
729 | 740 |
if sizeq[0]: |
... | ... | |
732 | 743 |
if sizeq[1]: |
733 | 744 |
subq += " and v.size < ?" |
734 | 745 |
args += [sizeq[1]] |
735 |
|
|
746 |
|
|
736 | 747 |
return subq, args |
737 |
|
|
748 |
|
|
738 | 749 |
def _construct_versions_nodes_latest_version_subquery(self, before=inf): |
739 | 750 |
if before == inf: |
740 | 751 |
q = ("n.latest_version ") |
741 | 752 |
args = [] |
742 | 753 |
else: |
743 | 754 |
q = ("(select max(serial) " |
744 |
"from versions "
|
|
745 |
"where node = v.node and mtime < ?) ")
|
|
755 |
"from versions "
|
|
756 |
"where node = v.node and mtime < ?) ")
|
|
746 | 757 |
args = [before] |
747 | 758 |
return q, args |
748 |
|
|
759 |
|
|
749 | 760 |
def _construct_latest_version_subquery(self, node=None, before=inf): |
750 | 761 |
where_cond = "node = v.node" |
751 | 762 |
args = [] |
752 | 763 |
if node: |
753 | 764 |
where_cond = "node = ? " |
754 | 765 |
args = [node] |
755 |
|
|
766 |
|
|
756 | 767 |
if before == inf: |
757 | 768 |
q = ("(select latest_version " |
758 |
"from nodes "
|
|
759 |
"where %s) ")
|
|
769 |
"from nodes " |
|
770 |
"where %s) ") |
|
760 | 771 |
else: |
761 | 772 |
q = ("(select max(serial) " |
762 |
"from versions "
|
|
763 |
"where %s and mtime < ?) ")
|
|
773 |
"from versions " |
|
774 |
"where %s and mtime < ?) ") |
|
764 | 775 |
args += [before] |
765 | 776 |
return q % where_cond, args |
766 |
|
|
777 |
|
|
767 | 778 |
def _construct_latest_versions_subquery(self, nodes=(), before=inf): |
768 | 779 |
where_cond = "" |
769 | 780 |
args = [] |
770 | 781 |
if nodes: |
771 | 782 |
where_cond = "node in (%s) " % ','.join('?' for node in nodes) |
772 | 783 |
args = nodes |
773 |
|
|
784 |
|
|
774 | 785 |
if before == inf: |
775 | 786 |
q = ("(select latest_version " |
776 |
"from nodes "
|
|
777 |
"where %s ) ")
|
|
787 |
"from nodes " |
|
788 |
"where %s ) ") |
|
778 | 789 |
else: |
779 | 790 |
q = ("(select max(serial) " |
780 |
"from versions "
|
|
781 |
"where %s and mtime < ? group by node) ")
|
|
791 |
"from versions "
|
|
792 |
"where %s and mtime < ? group by node) ")
|
|
782 | 793 |
args += [before] |
783 | 794 |
return q % where_cond, args |
784 |
|
|
795 |
|
|
785 | 796 |
def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]): |
786 | 797 |
"""Return a list with all keys pairs defined |
787 | 798 |
for all latest versions under parent that |
788 | 799 |
do not belong to the cluster. |
789 | 800 |
""" |
790 |
|
|
801 |
|
|
791 | 802 |
# TODO: Use another table to store before=inf results. |
792 | 803 |
q = ("select distinct a.key " |
793 | 804 |
"from attributes a, versions v, nodes n " |
794 | 805 |
"where v.serial = %s " |
795 | 806 |
"and v.cluster != ? " |
796 | 807 |
"and v.node in (select node " |
797 |
"from nodes "
|
|
798 |
"where parent = ?) "
|
|
808 |
"from nodes " |
|
809 |
"where parent = ?) " |
|
799 | 810 |
"and a.serial = v.serial " |
800 | 811 |
"and a.domain = ? " |
801 | 812 |
"and n.node = v.node") |
802 |
subq, subargs = self._construct_latest_version_subquery(node=None, before=before) |
|
813 |
subq, subargs = self._construct_latest_version_subquery( |
|
814 |
node=None, before=before) |
|
803 | 815 |
args = subargs + [except_cluster, parent, domain] |
804 | 816 |
q = q % subq |
805 | 817 |
subq, subargs = self._construct_paths(pathq) |
... | ... | |
808 | 820 |
args += subargs |
809 | 821 |
self.execute(q, args) |
810 | 822 |
return [r[0] for r in self.fetchall()] |
811 |
|
|
823 |
|
|
812 | 824 |
def latest_version_list(self, parent, prefix='', delimiter=None, |
813 | 825 |
start='', limit=10000, before=inf, |
814 | 826 |
except_cluster=0, pathq=[], domain=None, |
... | ... | |
816 | 828 |
"""Return a (list of (path, serial) tuples, list of common prefixes) |
817 | 829 |
for the current versions of the paths with the given parent, |
818 | 830 |
matching the following criteria. |
819 |
|
|
831 |
|
|
820 | 832 |
The property tuple for a version is returned if all |
821 | 833 |
of these conditions are true: |
822 |
|
|
834 |
|
|
823 | 835 |
a. parent matches |
824 |
|
|
836 |
|
|
825 | 837 |
b. path > start |
826 |
|
|
838 |
|
|
827 | 839 |
c. path starts with prefix (and paths in pathq) |
828 |
|
|
840 |
|
|
829 | 841 |
d. version is the max up to before |
830 |
|
|
842 |
|
|
831 | 843 |
e. version is not in cluster |
832 |
|
|
844 |
|
|
833 | 845 |
f. the path does not have the delimiter occuring |
834 | 846 |
after the prefix, or ends with the delimiter |
835 |
|
|
847 |
|
|
836 | 848 |
g. serial matches the attribute filter query. |
837 |
|
|
849 |
|
|
838 | 850 |
A filter query is a comma-separated list of |
839 | 851 |
terms in one of these three forms: |
840 |
|
|
852 |
|
|
841 | 853 |
key |
842 | 854 |
an attribute with this key must exist |
843 |
|
|
855 |
|
|
844 | 856 |
!key |
845 | 857 |
an attribute with this key must not exist |
846 |
|
|
858 |
|
|
847 | 859 |
key ?op value |
848 | 860 |
the attribute with this key satisfies the value |
849 | 861 |
where ?op is one of =, != <=, >=, <, >. |
850 |
|
|
862 |
|
|
851 | 863 |
h. the size is in the range set by sizeq |
852 |
|
|
864 |
|
|
853 | 865 |
The list of common prefixes includes the prefixes |
854 | 866 |
matching up to the first delimiter after prefix, |
855 | 867 |
and are reported only once, as "virtual directories". |
856 | 868 |
The delimiter is included in the prefixes. |
857 |
|
|
869 |
|
|
858 | 870 |
If arguments are None, then the corresponding matching rule |
859 | 871 |
will always match. |
860 |
|
|
872 |
|
|
861 | 873 |
Limit applies to the first list of tuples returned. |
862 |
|
|
874 |
|
|
863 | 875 |
If all_props is True, return all properties after path, not just serial. |
864 | 876 |
""" |
865 |
|
|
877 |
|
|
866 | 878 |
execute = self.execute |
867 |
|
|
879 |
|
|
868 | 880 |
if not start or start < prefix: |
869 | 881 |
start = strprevling(prefix) |
870 | 882 |
nextling = strnextling(prefix) |
871 |
|
|
883 |
|
|
872 | 884 |
q = ("select distinct n.path, %s " |
873 | 885 |
"from versions v, nodes n " |
874 | 886 |
"where v.serial = %s " |
875 | 887 |
"and v.cluster != ? " |
876 | 888 |
"and v.node in (select node " |
877 |
"from nodes "
|
|
878 |
"where parent = ?) "
|
|
889 |
"from nodes " |
|
890 |
"where parent = ?) " |
|
879 | 891 |
"and n.node = v.node " |
880 | 892 |
"and n.path > ? and n.path < ?") |
881 |
subq, args = self._construct_versions_nodes_latest_version_subquery(before) |
|
893 |
subq, args = self._construct_versions_nodes_latest_version_subquery( |
|
894 |
before) |
|
882 | 895 |
if not all_props: |
883 | 896 |
q = q % ("v.serial", subq) |
884 | 897 |
else: |
885 | 898 |
q = q % ("v.serial, v.node, v.hash, v.size, v.type, v.source, v.mtime, v.muser, v.uuid, v.checksum, v.cluster", subq) |
886 | 899 |
args += [except_cluster, parent, start, nextling] |
887 | 900 |
start_index = len(args) - 2 |
888 |
|
|
901 |
|
|
889 | 902 |
subq, subargs = self._construct_paths(pathq) |
890 | 903 |
if subq is not None: |
891 | 904 |
q += subq |
... | ... | |
902 | 915 |
q = q.replace("attributes a, ", "") |
903 | 916 |
q = q.replace("and a.serial = v.serial ", "") |
904 | 917 |
q += " order by n.path" |
905 |
|
|
918 |
|
|
906 | 919 |
if not delimiter: |
907 | 920 |
q += " limit ?" |
908 | 921 |
args.append(limit) |
909 | 922 |
execute(q, args) |
910 | 923 |
return self.fetchall(), () |
911 |
|
|
924 |
|
|
912 | 925 |
pfz = len(prefix) |
913 | 926 |
dz = len(delimiter) |
914 | 927 |
count = 0 |
... | ... | |
917 | 930 |
pappend = prefixes.append |
918 | 931 |
matches = [] |
919 | 932 |
mappend = matches.append |
920 |
|
|
933 |
|
|
921 | 934 |
execute(q, args) |
922 | 935 |
while True: |
923 | 936 |
props = fetchone() |
... | ... | |
926 | 939 |
path = props[0] |
927 | 940 |
serial = props[1] |
928 | 941 |
idx = path.find(delimiter, pfz) |
929 |
|
|
942 |
|
|
930 | 943 |
if idx < 0: |
931 | 944 |
mappend(props) |
932 | 945 |
count += 1 |
933 | 946 |
if count >= limit: |
934 | 947 |
break |
935 | 948 |
continue |
936 |
|
|
949 |
|
|
937 | 950 |
if idx + dz == len(path): |
938 | 951 |
mappend(props) |
939 | 952 |
count += 1 |
940 |
continue # Get one more, in case there is a path. |
|
953 |
continue # Get one more, in case there is a path.
|
|
941 | 954 |
pf = path[:idx + dz] |
942 | 955 |
pappend(pf) |
943 |
if count >= limit:
|
|
956 |
if count >= limit: |
|
944 | 957 |
break |
945 |
|
|
946 |
args[start_index] = strnextling(pf) # New start. |
|
958 |
|
|
959 |
args[start_index] = strnextling(pf) # New start.
|
|
947 | 960 |
execute(q, args) |
948 |
|
|
961 |
|
|
949 | 962 |
return matches, prefixes |
950 |
|
|
963 |
|
|
951 | 964 |
def latest_uuid(self, uuid): |
952 | 965 |
"""Return a (path, serial) tuple, for the latest version of the given uuid.""" |
953 |
|
|
966 |
|
|
954 | 967 |
q = ("select n.path, v.serial " |
955 | 968 |
"from versions v, nodes n " |
956 | 969 |
"where v.serial = (select max(serial) " |
957 |
"from versions "
|
|
958 |
"where uuid = ?) "
|
|
970 |
"from versions " |
|
971 |
"where uuid = ?) " |
|
959 | 972 |
"and n.node = v.node") |
960 | 973 |
self.execute(q, (uuid,)) |
961 | 974 |
return self.fetchone() |
Also available in: Unified diff