( node integer primary key,
parent integer default 0,
path text not null default '',
+ latest_version integer,
foreign key (parent)
references nodes(node)
on update cascade
on delete cascade ) """)
execute(""" create unique index if not exists idx_nodes_path
on nodes(path) """)
+ execute(""" create index if not exists idx_nodes_parent
+ on nodes(parent) """)
execute(""" create table if not exists policy
( node integer,
"""
placeholders = ','.join('?' for path in paths)
- q = "select path, node from nodes where path in (%s)" % placeholders
+ q = "select node from nodes where path in (%s)" % placeholders
self.execute(q, paths)
- return self.fetchall()
+ r = self.fetchall()
+ if r is not None:
+ return [row[0] for row in r]
+ return None
def node_get_properties(self, node):
"""Return the node's (parent, path).
# The latest version.
q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster "
- "from versions "
- "where serial = (select max(serial) "
- "from versions "
- "where node = ? and mtime < ?) "
+ "from versions v "
+ "where serial = %s "
"and cluster != ?")
- execute(q, (node, before, except_cluster))
+ subq, args = self._construct_latest_version_subquery(node=node, before=before)
+ execute(q % subq, args + [except_cluster])
props = fetchone()
if props is None:
return None
# First level, just under node (get population).
q = ("select count(serial), sum(size), max(mtime) "
"from versions v "
- "where serial = (select max(serial) "
- "from versions "
- "where node = v.node and mtime < ?) "
+ "where serial = %s "
"and cluster != ? "
"and node in (select node "
"from nodes "
"where parent = ?)")
- execute(q, (before, except_cluster, node))
+ subq, args = self._construct_latest_version_subquery(node=None, before=before)
+ execute(q % subq, args + [except_cluster, node])
r = fetchone()
if r is None:
return None
# This is why the full path is stored.
q = ("select count(serial), sum(size), max(mtime) "
"from versions v "
- "where serial = (select max(serial) "
- "from versions "
- "where node = v.node and mtime < ?) "
+ "where serial = %s "
"and cluster != ? "
"and node in (select node "
"from nodes "
"where path like ? escape '\\')")
- execute(q, (before, except_cluster, self.escape_like(path) + '%'))
+ subq, args = self._construct_latest_version_subquery(node=None, before=before)
+ execute(q % subq, args + [except_cluster, self.escape_like(path) + '%'])
r = fetchone()
if r is None:
return None
mtime = max(mtime, r[2])
return (count, size, mtime)
+ def nodes_set_latest_version(self, node, serial=None):
+ if not serial:
+ q = ("select serial from versions where node = ? order_by mtime desc limit 1")
+ self.execute(q, (node,))
+ r = self.fetchone()
+ serial = r[0] of r else None
+ q = ("update nodes set latest_version = ? where node = ?")
+ props = (serial, node)
+ self.execute(q, props)
+
def version_create(self, node, hash, size, type, source, muser, uuid, checksum, cluster=0):
"""Create a new version from the given properties.
Return the (serial, mtime) of the new version.
props = (node, hash, size, type, source, mtime, muser, uuid, checksum, cluster)
serial = self.execute(q, props).lastrowid
self.statistics_update_ancestors(node, 1, size, mtime, cluster)
+
+ self.nodes_set_latest_version(node, serial)
+
return serial, mtime
- def version_lookup(self, node, before=inf, cluster=0):
+ def version_lookup(self, node, before=inf, cluster=0, all_props=True):
"""Lookup the current version of the given node.
Return a list with its properties:
(serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster)
or None if the current version is not found in the given cluster.
"""
- q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster "
- "from versions "
- "where serial = (select max(serial) "
- "from versions "
- "where node = ? and mtime < ?) "
+ q = ("select %s "
+ "from versions v "
+ "where serial = %s "
"and cluster = ?")
- self.execute(q, (node, before, cluster))
+ subq, args = self._construct_latest_version_subquery(node=node, before=before)
+ if not all_props:
+ q = q % ("serial", subq)
+ else:
+ q = q % ("serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster", subq)
+
+ self.execute(q, args + [cluster])
props = self.fetchone()
if props is not None:
return props
return None
- def version_lookup_bulk(self, nodes, before=inf, cluster=0):
+ def version_lookup_bulk(self, nodes, before=inf, cluster=0, all_props=True):
"""Lookup the current versions of the given nodes.
Return a list with their properties:
(serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster).
"""
- placeholders = ','.join('?' for node in nodes)
- q = ("select serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster "
+ if not nodes:
+ return ()
+ q = ("select %s "
"from versions "
- "where serial in (select max(serial) "
- "from versions "
- "where node in (%s) and mtime < ? group by node) "
- "and cluster = ?" % placeholders)
- args = nodes
- args.extend((before, cluster))
+ "where serial in %s "
+ "and cluster = ? %s")
+ subq, args = self._construct_latest_versions_subquery(nodes=nodes, before = before)
+ if not all_props:
+ q = q % ("serial", subq, '')
+ else:
+ q = q % ("serial, node, hash, size, type, source, mtime, muser, uuid, checksum, cluster", subq, 'order by node')
+
+ args += [cluster]
self.execute(q, args)
return self.fetchall()
q = "delete from versions where serial = ?"
self.execute(q, (serial,))
+
+ self.nodes_set_latest_version(node)
+
return hash, size
def attribute_get(self, serial, domain, keys=()):
return subq, args
+ def _construct_versions_nodes_latest_version_subquery(self, before=inf):
+ if before == inf:
+ q = ("n.latest_version ")
+ args = []
+ else:
+ q = ("(select max(serial) "
+ "from versions "
+ "where node = v.node and mtime < ?) ")
+ args = [before]
+ return q, args
+
+ def _construct_latest_version_subquery(self, node=None, before=inf):
+ where_cond = "node = v.node"
+ args = []
+ if node:
+ where_cond = "node = ? "
+ args = [node]
+
+ if before == inf:
+ q = ("(select latest_version "
+ "from nodes "
+ "where %s) ")
+ else:
+ q = ("(select max(serial) "
+ "from versions "
+ "where %s and mtime < ?) ")
+ args += [before]
+ return q % where_cond, args
+
+ def _construct_latest_versions_subquery(self, nodes=(), before=inf):
+ where_cond = ""
+ args = []
+ if nodes:
+ where_cond = "node in (%s) " % ','.join('?' for node in nodes)
+ args = nodes
+
+ if before == inf:
+ q = ("(select latest_version "
+ "from nodes "
+ "where %s ) ")
+ else:
+ q = ("(select max(serial) "
+ "from versions "
+ "where %s and mtime < ? group by node) ")
+ args += [before]
+ return q % where_cond, args
+
def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):
"""Return a list with all keys pairs defined
for all latest versions under parent that
# TODO: Use another table to store before=inf results.
q = ("select distinct a.key "
"from attributes a, versions v, nodes n "
- "where v.serial = (select max(serial) "
- "from versions "
- "where node = v.node and mtime < ?) "
+ "where v.serial = %s "
"and v.cluster != ? "
"and v.node in (select node "
"from nodes "
"and a.serial = v.serial "
"and a.domain = ? "
"and n.node = v.node")
- args = (before, except_cluster, parent, domain)
+ subq, subargs = self._construct_latest_version_subquery(node=None, before=before)
+ args = subargs + [except_cluster, parent, domain]
+ q = q % subq
subq, subargs = self._construct_paths(pathq)
if subq is not None:
q += subq
q = ("select distinct n.path, %s "
"from versions v, nodes n "
- "where v.serial = (select max(serial) "
- "from versions "
- "where node = v.node and mtime < ?) "
+ "where v.serial = %s "
"and v.cluster != ? "
"and v.node in (select node "
"from nodes "
"where parent = ?) "
"and n.node = v.node "
"and n.path > ? and n.path < ?")
+ subq, args = self._construct_versions_nodes_latest_version_subquery(before)
if not all_props:
- q = q % "v.serial"
+ q = q % ("v.serial", subq)
else:
- q = q % "v.serial, v.node, v.hash, v.size, v.type, v.source, v.mtime, v.muser, v.uuid, v.checksum, v.cluster"
- args = [before, except_cluster, parent, start, nextling]
+ q = q % ("v.serial, v.node, v.hash, v.size, v.type, v.source, v.mtime, v.muser, v.uuid, v.checksum, v.cluster", subq)
+ args += [except_cluster, parent, start, nextling]
+ start_index = len(args) - 2
subq, subargs = self._construct_paths(pathq)
if subq is not None:
if count >= limit:
break
- args[3] = strnextling(pf) # New start.
+ args[start_index] = strnextling(pf) # New start.
execute(q, args)
return matches, prefixes
projects / pithos / blobdiff