from dbworker import DBWorker
+from pithos.lib.filter import parse_filters
+
ROOTNODE = 0
-( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8)
+( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, UUID, CLUSTER ) = range(9)
inf = float('inf')
'source' : 4,
'mtime' : 5,
'muser' : 6,
- 'cluster' : 7,
+ 'uuid' : 7,
+ 'cluster' : 8
}
source integer,
mtime integer,
muser text not null default '',
+ uuid text not null default '',
cluster integer not null default 0,
foreign key (node)
references nodes(node)
on delete cascade ) """)
execute(""" create index if not exists idx_versions_node_mtime
on versions(node, mtime) """)
+ execute(""" create index if not exists idx_versions_node_uuid
+ on versions(uuid) """)
execute(""" create table if not exists attributes
( serial integer,
+ domain text,
key text,
value text,
- primary key (serial, key)
+ primary key (serial, domain, key)
foreign key (serial)
references versions(serial)
on update cascade
def node_get_versions(self, node, keys=(), propnames=_propnames):
"""Return the properties of all versions at node.
If keys is empty, return all properties in the order
- (serial, node, size, source, mtime, muser, cluster).
+ (serial, node, hash, size, source, mtime, muser, uuid, cluster).
"""
- q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+ q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "
"from versions "
"where node = ?")
self.execute(q, (node,))
def node_purge_children(self, parent, before=inf, cluster=0):
"""Delete all versions with the specified
parent and cluster, and return
- the serials of versions deleted.
+ the hashes of versions deleted.
Clears out nodes with no remaining versions.
"""
self.statistics_update(parent, -nr, -size, mtime, cluster)
self.statistics_update_ancestors(parent, -nr, -size, mtime, cluster)
- q = ("select serial from versions "
+ q = ("select hash from versions "
"where node in (select node "
"from nodes "
"where parent = ?) "
"and cluster = ? "
"and mtime <= ?")
execute(q, args)
- serials = [r[SERIAL] for r in self.fetchall()]
+ hashes = [r[0] for r in self.fetchall()]
q = ("delete from versions "
"where node in (select node "
"from nodes "
"where node = n.node) = 0 "
"and parent = ?)")
execute(q, (parent,))
- return serials
+ return hashes
def node_purge(self, node, before=inf, cluster=0):
"""Delete all versions with the specified
node and cluster, and return
- the serials of versions deleted.
+ the hashes of versions deleted.
Clears out the node if it has no remaining versions.
"""
mtime = time()
self.statistics_update_ancestors(node, -nr, -size, mtime, cluster)
- q = ("select serial from versions "
+ q = ("select hash from versions "
"where node = ? "
"and cluster = ? "
"and mtime <= ?")
execute(q, args)
- serials = [r[SERIAL] for r in self.fetchall()]
+ hashes = [r[0] for r in self.fetchall()]
q = ("delete from versions "
"where node = ? "
"and cluster = ? "
"where node = n.node) = 0 "
"and node = ?)")
execute(q, (node,))
- return serials
+ return hashes
def node_remove(self, node):
"""Remove the node specified.
parent, path = props
# The latest version.
- q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+ q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "
"from versions "
"where serial = (select max(serial) "
"from versions "
"and cluster != ? "
"and node in (select node "
"from nodes "
- "where path like ?)")
- execute(q, (before, except_cluster, path + '%'))
+ "where path like ? escape '\\')")
+ execute(q, (before, except_cluster, self.escape_like(path) + '%'))
r = fetchone()
if r is None:
return None
mtime = max(mtime, r[2])
return (count, size, mtime)
- def version_create(self, node, hash, size, source, muser, cluster=0):
+ def version_create(self, node, hash, size, source, muser, uuid, cluster=0):
"""Create a new version from the given properties.
Return the (serial, mtime) of the new version.
"""
- q = ("insert into versions (node, hash, size, source, mtime, muser, cluster) "
- "values (?, ?, ?, ?, ?, ?, ?)")
+ q = ("insert into versions (node, hash, size, source, mtime, muser, uuid, cluster) "
+ "values (?, ?, ?, ?, ?, ?, ?, ?)")
mtime = time()
- props = (node, hash, size, source, mtime, muser, cluster)
+ props = (node, hash, size, source, mtime, muser, uuid, cluster)
serial = self.execute(q, props).lastrowid
self.statistics_update_ancestors(node, 1, size, mtime, cluster)
return serial, mtime
def version_lookup(self, node, before=inf, cluster=0):
"""Lookup the current version of the given node.
Return a list with its properties:
- (serial, node, hash, size, source, mtime, muser, cluster)
+ (serial, node, hash, size, source, mtime, muser, uuid, cluster)
or None if the current version is not found in the given cluster.
"""
- q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+ q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "
"from versions "
"where serial = (select max(serial) "
"from versions "
"""Return a sequence of values for the properties of
the version specified by serial and the keys, in the order given.
If keys is empty, return all properties in the order
- (serial, node, hash, size, source, mtime, muser, cluster).
+ (serial, node, hash, size, source, mtime, muser, uuid, cluster).
"""
- q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+ q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "
"from versions "
"where serial = ?")
self.execute(q, (serial,))
def version_remove(self, serial):
"""Remove the serial specified."""
- props = self.node_get_properties(serial)
+ props = self.version_get_properties(serial)
if not props:
return
node = props[NODE]
+ hash = props[HASH]
size = props[SIZE]
cluster = props[CLUSTER]
q = "delete from versions where serial = ?"
self.execute(q, (serial,))
- return True
+ return hash
- def attribute_get(self, serial, keys=()):
+ def attribute_get(self, serial, domain, keys=()):
"""Return a list of (key, value) pairs of the version specified by serial.
If keys is empty, return all attributes.
Othwerise, return only those specified.
if keys:
marks = ','.join('?' for k in keys)
q = ("select key, value from attributes "
- "where key in (%s) and serial = ?" % (marks,))
- execute(q, keys + (serial,))
+ "where key in (%s) and serial = ? and domain = ?" % (marks,))
+ execute(q, keys + (serial, domain))
else:
- q = "select key, value from attributes where serial = ?"
- execute(q, (serial,))
+ q = "select key, value from attributes where serial = ? and domain = ?"
+ execute(q, (serial, domain))
return self.fetchall()
- def attribute_set(self, serial, items):
+ def attribute_set(self, serial, domain, items):
"""Set the attributes of the version specified by serial.
Receive attributes as an iterable of (key, value) pairs.
"""
- q = ("insert or replace into attributes (serial, key, value) "
- "values (?, ?, ?)")
- self.executemany(q, ((serial, k, v) for k, v in items))
+ q = ("insert or replace into attributes (serial, domain, key, value) "
+ "values (?, ?, ?, ?)")
+ self.executemany(q, ((serial, domain, k, v) for k, v in items))
- def attribute_del(self, serial, keys=()):
+ def attribute_del(self, serial, domain, keys=()):
"""Delete attributes of the version specified by serial.
If keys is empty, delete all attributes.
Otherwise delete those specified.
"""
if keys:
- q = "delete from attributes where serial = ? and key = ?"
- self.executemany(q, ((serial, key) for key in keys))
+ q = "delete from attributes where serial = ? and domain = ? and key = ?"
+ self.executemany(q, ((serial, domain, key) for key in keys))
else:
- q = "delete from attributes where serial = ?"
- self.execute(q, (serial,))
+ q = "delete from attributes where serial = ? and domain = ?"
+ self.execute(q, (serial, domain))
def attribute_copy(self, source, dest):
q = ("insert or replace into attributes "
- "select ?, key, value from attributes "
+ "select ?, domain, key, value from attributes "
"where serial = ?")
self.execute(q, (dest, source))
- def _construct_filters(self, filterq):
- if not filterq:
+ def _construct_filters(self, domain, filterq):
+ if not domain or not filterq:
return None, None
- args = filterq.split(',')
- subq = " and a.key in ("
- subq += ','.join(('?' for x in args))
- subq += ")"
+ subqlist = []
+ append = subqlist.append
+ included, excluded, opers = parse_filters(filterq)
+ args = []
+
+ if included:
+ subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and "
+ subq += "(" + ' or '.join(('key = ?' for x in included)) + ")"
+ subq += ")"
+ args += [domain]
+ args += included
+ append(subq)
+
+ if excluded:
+ subq = "not exists (select 1 from attributes where serial = v.serial and domain = ? and "
+ subq += "(" + ' or '.join(('key = ?' for x in excluded)) + ")"
+ subq += ")"
+ args += [domain]
+ args += excluded
+ append(subq)
+
+ if opers:
+ for k, o, v in opers:
+ subq = "exists (select 1 from attributes where serial = v.serial and domain = ? and "
+ subq += "key = ? and value %s ?" % (o,)
+ subq += ")"
+ args += [domain, k, v]
+ append(subq)
+
+ if not subqlist:
+ return None, None
+
+ subq = ' and ' + ' and '.join(subqlist)
return subq, args
return None, None
subq = " and ("
- subq += ' or '.join(('n.path like ?' for x in pathq))
+ subq += ' or '.join(("n.path like ? escape '\\'" for x in pathq))
subq += ")"
- args = tuple([x + '%' for x in pathq])
+ args = tuple([self.escape_like(x) + '%' for x in pathq])
return subq, args
- def latest_attribute_keys(self, parent, before=inf, except_cluster=0, pathq=[]):
+ def latest_attribute_keys(self, parent, domain, before=inf, except_cluster=0, pathq=[]):
"""Return a list with all keys pairs defined
for all latest versions under parent that
do not belong to the cluster.
q = ("select distinct a.key "
"from attributes a, versions v, nodes n "
"where v.serial = (select max(serial) "
- "from versions "
- "where node = v.node and mtime < ?) "
+ "from versions "
+ "where node = v.node and mtime < ?) "
"and v.cluster != ? "
"and v.node in (select node "
- "from nodes "
- "where parent = ?) "
+ "from nodes "
+ "where parent = ?) "
"and a.serial = v.serial "
+ "and a.domain = ? "
"and n.node = v.node")
- args = (before, except_cluster, parent)
+ args = (before, except_cluster, parent, domain)
subq, subargs = self._construct_paths(pathq)
if subq is not None:
q += subq
def latest_version_list(self, parent, prefix='', delimiter=None,
start='', limit=10000, before=inf,
- except_cluster=0, pathq=[], filterq=None):
+ except_cluster=0, pathq=[], domain=None, filterq=[]):
"""Return a (list of (path, serial) tuples, list of common prefixes)
for the current versions of the paths with the given parent,
matching the following criteria.
key ?op value
the attribute with this key satisfies the value
- where ?op is one of ==, != <=, >=, <, >.
+ where ?op is one of =, != <=, >=, <, >.
The list of common prefixes includes the prefixes
matching up to the first delimiter after prefix,
nextling = strnextling(prefix)
q = ("select distinct n.path, v.serial "
- "from attributes a, versions v, nodes n "
+ "from versions v, nodes n "
"where v.serial = (select max(serial) "
- "from versions "
- "where node = v.node and mtime < ?) "
+ "from versions "
+ "where node = v.node and mtime < ?) "
"and v.cluster != ? "
"and v.node in (select node "
- "from nodes "
- "where parent = ?) "
- "and a.serial = v.serial "
+ "from nodes "
+ "where parent = ?) "
"and n.node = v.node "
"and n.path > ? and n.path < ?")
args = [before, except_cluster, parent, start, nextling]
if subq is not None:
q += subq
args += subargs
- subq, subargs = self._construct_filters(filterq)
+ subq, subargs = self._construct_filters(domain, filterq)
if subq is not None:
q += subq
args += subargs
execute(q, args)
return matches, prefixes
+
+ def latest_uuid(self, uuid):
+ """Return a (path, serial) tuple, for the latest version of the given uuid."""
+
+ q = ("select n.path, v.serial "
+ "from versions v, nodes n "
+ "where v.serial = (select max(serial) "
+ "from versions "
+ "where uuid = ?) "
+ "and n.node = v.node")
+ self.execute(q, (uuid,))
+ return self.fetchone()
projects / pithos / blobdiff