From 25ae8b75eba66d514aebfd2f15e91a31ab2110be Mon Sep 17 00:00:00 2001 From: Antony Chazapis Date: Mon, 19 Dec 2011 17:50:54 +0200 Subject: [PATCH] Unique object identifiers in the backend. Refs #1790 --- pithos/backends/base.py | 2 ++ pithos/backends/lib/sqlalchemy/__init__.py | 4 ++-- pithos/backends/lib/sqlalchemy/node.py | 25 +++++++++++++++---------- pithos/backends/lib/sqlite/__init__.py | 4 ++-- pithos/backends/lib/sqlite/node.py | 26 +++++++++++++++----------- pithos/backends/modular.py | 28 +++++++++++++++++----------- 6 files changed, 53 insertions(+), 36 deletions(-) diff --git a/pithos/backends/base.py b/pithos/backends/base.py index 8ac2237..e218308 100644 --- a/pithos/backends/base.py +++ b/pithos/backends/base.py @@ -333,6 +333,8 @@ class BaseBackend(object): 'version': The version identifier 'version_timestamp': The version's modification timestamp + + 'uuid': A unique identifier that persists data or metadata updates and renames Raises: NotAllowedError: Operation not permitted diff --git a/pithos/backends/lib/sqlalchemy/__init__.py b/pithos/backends/lib/sqlalchemy/__init__.py index 092d003..7464f6d 100644 --- a/pithos/backends/lib/sqlalchemy/__init__.py +++ b/pithos/backends/lib/sqlalchemy/__init__.py @@ -32,10 +32,10 @@ # or implied, of GRNET S.A. from dbwrapper import DBWrapper -from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, CLUSTER +from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, UUID, CLUSTER from permissions import Permissions, READ, WRITE __all__ = ["DBWrapper", - "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "CLUSTER", + "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "UUID", "CLUSTER", "Permissions", "READ", "WRITE"] diff --git a/pithos/backends/lib/sqlalchemy/node.py b/pithos/backends/lib/sqlalchemy/node.py index 76502c3..2b535db 100644 --- a/pithos/backends/lib/sqlalchemy/node.py +++ b/pithos/backends/lib/sqlalchemy/node.py @@ -46,7 +46,7 @@ from pithos.lib.filter import parse_filters ROOTNODE = 0 -( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8) +( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, UUID, CLUSTER ) = range(9) inf = float('inf') @@ -93,7 +93,8 @@ _propnames = { 'source' : 4, 'mtime' : 5, 'muser' : 6, - 'cluster' : 7, + 'uuid' : 7, + 'cluster' : 8 } @@ -158,10 +159,11 @@ class Node(DBWorker): columns.append(Column('source', Integer)) columns.append(Column('mtime', DECIMAL(precision=16, scale=6))) columns.append(Column('muser', String(255), nullable=False, default='')) + columns.append(Column('uuid', String(64), nullable=False, default='')) columns.append(Column('cluster', Integer, nullable=False, default=0)) self.versions = Table('versions', metadata, *columns, mysql_engine='InnoDB') - Index('idx_versions_node_mtime', self.versions.c.node, - self.versions.c.mtime) + Index('idx_versions_node_mtime', self.versions.c.node, self.versions.c.mtime) + Index('idx_versions_node_uuid', self.versions.c.uuid) #create attributes table columns = [] @@ -236,7 +238,7 @@ class Node(DBWorker): def node_get_versions(self, node, keys=(), propnames=_propnames): """Return the properties of all versions at node. If keys is empty, return all properties in the order - (serial, node, size, source, mtime, muser, cluster). + (serial, node, hash, size, source, mtime, muser, uuid, cluster). """ s = select([self.versions.c.serial, @@ -246,6 +248,7 @@ class Node(DBWorker): self.versions.c.source, self.versions.c.mtime, self.versions.c.muser, + self.versions.c.uuid, self.versions.c.cluster], self.versions.c.node == node) s = s.order_by(self.versions.c.serial) r = self.conn.execute(s) @@ -497,6 +500,7 @@ class Node(DBWorker): self.versions.c.source, self.versions.c.mtime, self.versions.c.muser, + self.versions.c.uuid, self.versions.c.cluster]) filtered = select([func.max(self.versions.c.serial)], self.versions.c.node == node) @@ -555,14 +559,14 @@ class Node(DBWorker): mtime = max(mtime, r[2]) return (count, size, mtime) - def version_create(self, node, hash, size, source, muser, cluster=0): + def version_create(self, node, hash, size, source, muser, uuid, cluster=0): """Create a new version from the given properties. Return the (serial, mtime) of the new version. """ mtime = time() s = self.versions.insert().values(node=node, hash=hash, size=size, source=source, - mtime=mtime, muser=muser, cluster=cluster) + mtime=mtime, muser=muser, uuid=uuid, cluster=cluster) serial = self.conn.execute(s).inserted_primary_key[0] self.statistics_update_ancestors(node, 1, size, mtime, cluster) return serial, mtime @@ -570,13 +574,14 @@ class Node(DBWorker): def version_lookup(self, node, before=inf, cluster=0): """Lookup the current version of the given node. Return a list with its properties: - (serial, node, hash, size, source, mtime, muser, cluster) + (serial, node, hash, size, source, mtime, muser, uuid, cluster) or None if the current version is not found in the given cluster. """ v = self.versions.alias('v') - s = select([v.c.serial, v.c.node, v.c.hash, v.c.size, - v.c.source, v.c.mtime, v.c.muser, v.c.cluster]) + s = select([v.c.serial, v.c.node, v.c.hash, + v.c.size, v.c.source, v.c.mtime, + v.c.muser, v.c.uuid, v.c.cluster]) c = select([func.max(self.versions.c.serial)], self.versions.c.node == node) if before != inf: diff --git a/pithos/backends/lib/sqlite/__init__.py b/pithos/backends/lib/sqlite/__init__.py index 092d003..7464f6d 100644 --- a/pithos/backends/lib/sqlite/__init__.py +++ b/pithos/backends/lib/sqlite/__init__.py @@ -32,10 +32,10 @@ # or implied, of GRNET S.A. from dbwrapper import DBWrapper -from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, CLUSTER +from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, UUID, CLUSTER from permissions import Permissions, READ, WRITE __all__ = ["DBWrapper", - "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "CLUSTER", + "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "UUID", "CLUSTER", "Permissions", "READ", "WRITE"] diff --git a/pithos/backends/lib/sqlite/node.py b/pithos/backends/lib/sqlite/node.py index 3952327..2c2773c 100644 --- a/pithos/backends/lib/sqlite/node.py +++ b/pithos/backends/lib/sqlite/node.py @@ -40,7 +40,7 @@ from pithos.lib.filter import parse_filters ROOTNODE = 0 -( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8) +( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, UUID, CLUSTER ) = range(9) inf = float('inf') @@ -88,7 +88,8 @@ _propnames = { 'source' : 4, 'mtime' : 5, 'muser' : 6, - 'cluster' : 7, + 'uuid' : 7, + 'cluster' : 8 } @@ -147,6 +148,7 @@ class Node(DBWorker): source integer, mtime integer, muser text not null default '', + uuid text not null default '', cluster integer not null default 0, foreign key (node) references nodes(node) @@ -154,6 +156,8 @@ class Node(DBWorker): on delete cascade ) """) execute(""" create index if not exists idx_versions_node_mtime on versions(node, mtime) """) + execute(""" create index if not exists idx_versions_node_uuid + on versions(uuid) """) execute(""" create table if not exists attributes ( serial integer, @@ -203,10 +207,10 @@ class Node(DBWorker): def node_get_versions(self, node, keys=(), propnames=_propnames): """Return the properties of all versions at node. If keys is empty, return all properties in the order - (serial, node, size, source, mtime, muser, cluster). + (serial, node, hash, size, source, mtime, muser, uuid, cluster). """ - q = ("select serial, node, hash, size, source, mtime, muser, cluster " + q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster " "from versions " "where node = ?") self.execute(q, (node,)) @@ -409,7 +413,7 @@ class Node(DBWorker): parent, path = props # The latest version. - q = ("select serial, node, hash, size, source, mtime, muser, cluster " + q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster " "from versions " "where serial = (select max(serial) " "from versions " @@ -459,15 +463,15 @@ class Node(DBWorker): mtime = max(mtime, r[2]) return (count, size, mtime) - def version_create(self, node, hash, size, source, muser, cluster=0): + def version_create(self, node, hash, size, source, muser, uuid, cluster=0): """Create a new version from the given properties. Return the (serial, mtime) of the new version. """ - q = ("insert into versions (node, hash, size, source, mtime, muser, cluster) " - "values (?, ?, ?, ?, ?, ?, ?)") + q = ("insert into versions (node, hash, size, source, mtime, muser, uuid, cluster) " + "values (?, ?, ?, ?, ?, ?, ?, ?)") mtime = time() - props = (node, hash, size, source, mtime, muser, cluster) + props = (node, hash, size, source, mtime, muser, uuid, cluster) serial = self.execute(q, props).lastrowid self.statistics_update_ancestors(node, 1, size, mtime, cluster) return serial, mtime @@ -475,11 +479,11 @@ class Node(DBWorker): def version_lookup(self, node, before=inf, cluster=0): """Lookup the current version of the given node. Return a list with its properties: - (serial, node, hash, size, source, mtime, muser, cluster) + (serial, node, hash, size, source, mtime, muser, uuid, cluster) or None if the current version is not found in the given cluster. """ - q = ("select serial, node, hash, size, source, mtime, muser, cluster " + q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster " "from versions " "where serial = (select max(serial) " "from versions " diff --git a/pithos/backends/modular.py b/pithos/backends/modular.py index ccb7211..adde42d 100644 --- a/pithos/backends/modular.py +++ b/pithos/backends/modular.py @@ -34,6 +34,7 @@ import sys import os import time +import uuid import logging import binascii @@ -426,7 +427,7 @@ class ModularBackend(BaseBackend): meta = dict(self.node.attribute_get(props[self.SERIAL], domain)) meta.update({'name': name, 'bytes': props[self.SIZE], 'hash':props[self.HASH]}) meta.update({'version': props[self.SERIAL], 'version_timestamp': props[self.MTIME]}) - meta.update({'modified': modified, 'modified_by': props[self.MUSER]}) + meta.update({'modified': modified, 'modified_by': props[self.MUSER], 'uuid': props[self.UUID]}) return meta @backend_method @@ -505,7 +506,7 @@ class ModularBackend(BaseBackend): hashmap = self.store.map_get(binascii.unhexlify(props[self.HASH])) return props[self.SIZE], [binascii.hexlify(x) for x in hashmap] - def _update_object_hash(self, user, account, container, name, size, hash, permissions=None): + def _update_object_hash(self, user, account, container, name, size, hash, permissions, is_copy=False): if permissions is not None and user != account: raise NotAllowedError self._can_write(user, account, container, name) @@ -516,7 +517,7 @@ class ModularBackend(BaseBackend): account_path, account_node = self._lookup_account(account, True) container_path, container_node = self._lookup_container(account, container) path, node = self._put_object_node(container_path, container_node, name) - src_version_id, dest_version_id = self._put_version_duplicate(user, node, size, hash) + src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=size, hash=hash, is_copy=is_copy) # Check quota. size_delta = size # Change with versioning. @@ -554,7 +555,7 @@ class ModularBackend(BaseBackend): self.store.map_put(hash, map) return dest_version_id - def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None): + def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None, is_move=False): self._can_read(user, src_account, src_container, src_name) path, node = self._lookup_object(src_account, src_container, src_name) props = self._get_version(node, src_version) @@ -562,7 +563,8 @@ class ModularBackend(BaseBackend): hash = props[self.HASH] size = props[self.SIZE] - src_v_id, dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, hash, permissions) + is_copy = not is_move and (src_account, src_container, src_name) != (dest_account, dest_container, dest_name) # New uuid. + src_v_id, dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, hash, permissions, is_copy=is_copy) self._put_metadata_duplicate(src_version_id, dest_version_id, dest_domain, dest_meta, replace_meta) return dest_version_id @@ -571,7 +573,7 @@ class ModularBackend(BaseBackend): """Copy an object's data and metadata.""" logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version) - return self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version) + return self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version, False) @backend_method def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta={}, replace_meta=False, permissions=None): @@ -580,7 +582,7 @@ class ModularBackend(BaseBackend): logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions) if user != src_account: raise NotAllowedError - dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, None) + dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, None, True) if (src_account, src_container, src_name) != (dest_account, dest_container, dest_name): self._delete_object(user, src_account, src_container, src_name) return dest_version_id @@ -606,7 +608,7 @@ class ModularBackend(BaseBackend): return path, node = self._lookup_object(account, container, name) - src_version_id, dest_version_id = self._put_version_duplicate(user, node, 0, None, CLUSTER_DELETED) + src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, hash=None, cluster=CLUSTER_DELETED) self._apply_versioning(account, container, src_version_id) self.permissions.access_clear(path) @@ -667,6 +669,9 @@ class ModularBackend(BaseBackend): # Path functions. + def _generate_uuid(): + return str(uuid.uuid4()) + def _put_object_node(self, path, parent, name): path = '/'.join((path, name)) node = self.node.node_lookup(path) @@ -676,7 +681,7 @@ class ModularBackend(BaseBackend): def _put_path(self, user, parent, path): node = self.node.node_create(parent, path) - self.node.version_create(node, None, 0, None, user, CLUSTER_NORMAL) + self.node.version_create(node, None, 0, None, user, self._generate_uuid(), CLUSTER_NORMAL) return node def _lookup_account(self, account, create=True): @@ -736,7 +741,7 @@ class ModularBackend(BaseBackend): raise IndexError('Version does not exist') return props - def _put_version_duplicate(self, user, node, size=None, hash=None, cluster=CLUSTER_NORMAL): + def _put_version_duplicate(self, user, node, size=None, hash=None, cluster=CLUSTER_NORMAL, is_copy=False): """Create a new version of the node.""" props = self.node.version_lookup(node, inf, CLUSTER_NORMAL) @@ -751,10 +756,11 @@ class ModularBackend(BaseBackend): if size is None: hash = src_hash # This way hash can be set to None. size = src_size + uniq = self._generate_uuid() if is_copy or src_version_id is None else props[self.UUID] if src_version_id is not None: self.node.version_recluster(src_version_id, CLUSTER_HISTORY) - dest_version_id, mtime = self.node.version_create(node, hash, size, src_version_id, user, cluster) + dest_version_id, mtime = self.node.version_create(node, hash, size, src_version_id, user, uniq, cluster) return src_version_id, dest_version_id def _put_metadata_duplicate(self, src_version_id, dest_version_id, domain, meta, replace=False): -- 1.7.10.4