Unique object identifiers in the backend.

Refs #1790

diff --git a/pithos/backends/base.py b/pithos/backends/base.py
index 8ac2237..e218308 100644 (file)
--- a/pithos/backends/base.py
+++ b/pithos/backends/base.py
@@ -333,6 +333,8 @@ class BaseBackend(object):
             'version': The version identifier
             
             'version_timestamp': The version's modification timestamp
+            
+            'uuid': A unique identifier that persists data or metadata updates and renames
         
         Raises:
             NotAllowedError: Operation not permitted

diff --git a/pithos/backends/lib/sqlalchemy/__init__.py b/pithos/backends/lib/sqlalchemy/__init__.py
index 092d003..7464f6d 100644 (file)
--- a/pithos/backends/lib/sqlalchemy/__init__.py
+++ b/pithos/backends/lib/sqlalchemy/__init__.py
@@ -32,10 +32,10 @@
 # or implied, of GRNET S.A.
 
 from dbwrapper import DBWrapper
-from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, CLUSTER
+from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, UUID, CLUSTER
 from permissions import Permissions, READ, WRITE
 
 __all__ = ["DBWrapper",
-           "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "CLUSTER",
+           "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "UUID", "CLUSTER",
            "Permissions", "READ", "WRITE"]
 

diff --git a/pithos/backends/lib/sqlalchemy/node.py b/pithos/backends/lib/sqlalchemy/node.py
index 76502c3..2b535db 100644 (file)
--- a/pithos/backends/lib/sqlalchemy/node.py
+++ b/pithos/backends/lib/sqlalchemy/node.py
@@ -46,7 +46,7 @@ from pithos.lib.filter import parse_filters
 
 ROOTNODE  = 0
 
-( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8)
+( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, UUID, CLUSTER ) = range(9)
 
 inf = float('inf')
 
@@ -93,7 +93,8 @@ _propnames = {
     'source'    : 4,
     'mtime'     : 5,
     'muser'     : 6,
-    'cluster'   : 7,
+    'uuid'      : 7,
+    'cluster'   : 8
 }
 
 
@@ -158,10 +159,11 @@ class Node(DBWorker):
         columns.append(Column('source', Integer))
         columns.append(Column('mtime', DECIMAL(precision=16, scale=6)))
         columns.append(Column('muser', String(255), nullable=False, default=''))
+        columns.append(Column('uuid', String(64), nullable=False, default=''))
         columns.append(Column('cluster', Integer, nullable=False, default=0))
         self.versions = Table('versions', metadata, *columns, mysql_engine='InnoDB')
-        Index('idx_versions_node_mtime', self.versions.c.node,
-              self.versions.c.mtime)
+        Index('idx_versions_node_mtime', self.versions.c.node, self.versions.c.mtime)
+        Index('idx_versions_node_uuid', self.versions.c.uuid)
         
         #create attributes table
         columns = []
@@ -236,7 +238,7 @@ class Node(DBWorker):
     def node_get_versions(self, node, keys=(), propnames=_propnames):
         """Return the properties of all versions at node.
            If keys is empty, return all properties in the order
-           (serial, node, size, source, mtime, muser, cluster).
+           (serial, node, hash, size, source, mtime, muser, uuid, cluster).
         """
         
         s = select([self.versions.c.serial,
@@ -246,6 +248,7 @@ class Node(DBWorker):
                     self.versions.c.source,
                     self.versions.c.mtime,
                     self.versions.c.muser,
+                    self.versions.c.uuid,
                     self.versions.c.cluster], self.versions.c.node == node)
         s = s.order_by(self.versions.c.serial)
         r = self.conn.execute(s)
@@ -497,6 +500,7 @@ class Node(DBWorker):
                     self.versions.c.source,
                     self.versions.c.mtime,
                     self.versions.c.muser,
+                    self.versions.c.uuid,
                     self.versions.c.cluster])
         filtered = select([func.max(self.versions.c.serial)],
                             self.versions.c.node == node)
@@ -555,14 +559,14 @@ class Node(DBWorker):
         mtime = max(mtime, r[2])
         return (count, size, mtime)
     
-    def version_create(self, node, hash, size, source, muser, cluster=0):
+    def version_create(self, node, hash, size, source, muser, uuid, cluster=0):
         """Create a new version from the given properties.
            Return the (serial, mtime) of the new version.
         """
         
         mtime = time()
         s = self.versions.insert().values(node=node, hash=hash, size=size, source=source,
-                                          mtime=mtime, muser=muser, cluster=cluster)
+                                          mtime=mtime, muser=muser, uuid=uuid, cluster=cluster)
         serial = self.conn.execute(s).inserted_primary_key[0]
         self.statistics_update_ancestors(node, 1, size, mtime, cluster)
         return serial, mtime
@@ -570,13 +574,14 @@ class Node(DBWorker):
     def version_lookup(self, node, before=inf, cluster=0):
         """Lookup the current version of the given node.
            Return a list with its properties:
-           (serial, node, hash, size, source, mtime, muser, cluster)
+           (serial, node, hash, size, source, mtime, muser, uuid, cluster)
            or None if the current version is not found in the given cluster.
         """
         
         v = self.versions.alias('v')
-        s = select([v.c.serial, v.c.node, v.c.hash, v.c.size,
-                    v.c.source, v.c.mtime, v.c.muser, v.c.cluster])
+        s = select([v.c.serial, v.c.node, v.c.hash,
+                    v.c.size, v.c.source, v.c.mtime,
+                    v.c.muser, v.c.uuid, v.c.cluster])
         c = select([func.max(self.versions.c.serial)],
             self.versions.c.node == node)
         if before != inf:

diff --git a/pithos/backends/lib/sqlite/__init__.py b/pithos/backends/lib/sqlite/__init__.py
index 092d003..7464f6d 100644 (file)
--- a/pithos/backends/lib/sqlite/__init__.py
+++ b/pithos/backends/lib/sqlite/__init__.py
@@ -32,10 +32,10 @@
 # or implied, of GRNET S.A.
 
 from dbwrapper import DBWrapper
-from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, CLUSTER
+from node import Node, ROOTNODE, SERIAL, HASH, SIZE, MTIME, MUSER, UUID, CLUSTER
 from permissions import Permissions, READ, WRITE
 
 __all__ = ["DBWrapper",
-           "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "CLUSTER",
+           "Node", "ROOTNODE", "SERIAL", "HASH", "SIZE", "MTIME", "MUSER", "UUID", "CLUSTER",
            "Permissions", "READ", "WRITE"]
 

diff --git a/pithos/backends/lib/sqlite/node.py b/pithos/backends/lib/sqlite/node.py
index 3952327..2c2773c 100644 (file)
--- a/pithos/backends/lib/sqlite/node.py
+++ b/pithos/backends/lib/sqlite/node.py
@@ -40,7 +40,7 @@ from pithos.lib.filter import parse_filters
 
 ROOTNODE  = 0
 
-( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, CLUSTER ) = range(8)
+( SERIAL, NODE, HASH, SIZE, SOURCE, MTIME, MUSER, UUID, CLUSTER ) = range(9)
 
 inf = float('inf')
 
@@ -88,7 +88,8 @@ _propnames = {
     'source'    : 4,
     'mtime'     : 5,
     'muser'     : 6,
-    'cluster'   : 7,
+    'uuid'      : 7,
+    'cluster'   : 8
 }
 
 
@@ -147,6 +148,7 @@ class Node(DBWorker):
                             source     integer,
                             mtime      integer,
                             muser      text    not null default '',
+                            uuid       text    not null default '',
                             cluster    integer not null default 0,
                             foreign key (node)
                             references nodes(node)
@@ -154,6 +156,8 @@ class Node(DBWorker):
                             on delete cascade ) """)
         execute(""" create index if not exists idx_versions_node_mtime
                     on versions(node, mtime) """)
+        execute(""" create index if not exists idx_versions_node_uuid
+                    on versions(uuid) """)
         
         execute(""" create table if not exists attributes
                           ( serial integer,
@@ -203,10 +207,10 @@ class Node(DBWorker):
     def node_get_versions(self, node, keys=(), propnames=_propnames):
         """Return the properties of all versions at node.
            If keys is empty, return all properties in the order
-           (serial, node, size, source, mtime, muser, cluster).
+           (serial, node, hash, size, source, mtime, muser, uuid, cluster).
         """
         
-        q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+        q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "
              "from versions "
              "where node = ?")
         self.execute(q, (node,))
@@ -409,7 +413,7 @@ class Node(DBWorker):
         parent, path = props
         
         # The latest version.
-        q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+        q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "
              "from versions "
              "where serial = (select max(serial) "
                              "from versions "
@@ -459,15 +463,15 @@ class Node(DBWorker):
         mtime = max(mtime, r[2])
         return (count, size, mtime)
     
-    def version_create(self, node, hash, size, source, muser, cluster=0):
+    def version_create(self, node, hash, size, source, muser, uuid, cluster=0):
         """Create a new version from the given properties.
            Return the (serial, mtime) of the new version.
         """
         
-        q = ("insert into versions (node, hash, size, source, mtime, muser, cluster) "
-             "values (?, ?, ?, ?, ?, ?, ?)")
+        q = ("insert into versions (node, hash, size, source, mtime, muser, uuid, cluster) "
+             "values (?, ?, ?, ?, ?, ?, ?, ?)")
         mtime = time()
-        props = (node, hash, size, source, mtime, muser, cluster)
+        props = (node, hash, size, source, mtime, muser, uuid, cluster)
         serial = self.execute(q, props).lastrowid
         self.statistics_update_ancestors(node, 1, size, mtime, cluster)
         return serial, mtime
@@ -475,11 +479,11 @@ class Node(DBWorker):
     def version_lookup(self, node, before=inf, cluster=0):
         """Lookup the current version of the given node.
            Return a list with its properties:
-           (serial, node, hash, size, source, mtime, muser, cluster)
+           (serial, node, hash, size, source, mtime, muser, uuid, cluster)
            or None if the current version is not found in the given cluster.
         """
         
-        q = ("select serial, node, hash, size, source, mtime, muser, cluster "
+        q = ("select serial, node, hash, size, source, mtime, muser, uuid, cluster "
              "from versions "
              "where serial = (select max(serial) "
                              "from versions "

diff --git a/pithos/backends/modular.py b/pithos/backends/modular.py
index ccb7211..adde42d 100644 (file)
--- a/pithos/backends/modular.py
+++ b/pithos/backends/modular.py
@@ -34,6 +34,7 @@
 import sys
 import os
 import time
+import uuid
 import logging
 import binascii
 
@@ -426,7 +427,7 @@ class ModularBackend(BaseBackend):
         meta = dict(self.node.attribute_get(props[self.SERIAL], domain))
         meta.update({'name': name, 'bytes': props[self.SIZE], 'hash':props[self.HASH]})
         meta.update({'version': props[self.SERIAL], 'version_timestamp': props[self.MTIME]})
-        meta.update({'modified': modified, 'modified_by': props[self.MUSER]})
+        meta.update({'modified': modified, 'modified_by': props[self.MUSER], 'uuid': props[self.UUID]})
         return meta
     
     @backend_method
@@ -505,7 +506,7 @@ class ModularBackend(BaseBackend):
         hashmap = self.store.map_get(binascii.unhexlify(props[self.HASH]))
         return props[self.SIZE], [binascii.hexlify(x) for x in hashmap]
     
-    def _update_object_hash(self, user, account, container, name, size, hash, permissions=None):
+    def _update_object_hash(self, user, account, container, name, size, hash, permissions, is_copy=False):
         if permissions is not None and user != account:
             raise NotAllowedError
         self._can_write(user, account, container, name)
@@ -516,7 +517,7 @@ class ModularBackend(BaseBackend):
         account_path, account_node = self._lookup_account(account, True)
         container_path, container_node = self._lookup_container(account, container)
         path, node = self._put_object_node(container_path, container_node, name)
-        src_version_id, dest_version_id = self._put_version_duplicate(user, node, size, hash)
+        src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=size, hash=hash, is_copy=is_copy)
         
         # Check quota.
         size_delta = size # Change with versioning.
@@ -554,7 +555,7 @@ class ModularBackend(BaseBackend):
         self.store.map_put(hash, map)
         return dest_version_id
     
-    def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None):
+    def _copy_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, dest_domain=None, dest_meta={}, replace_meta=False, permissions=None, src_version=None, is_move=False):
         self._can_read(user, src_account, src_container, src_name)
         path, node = self._lookup_object(src_account, src_container, src_name)
         props = self._get_version(node, src_version)
@@ -562,7 +563,8 @@ class ModularBackend(BaseBackend):
         hash = props[self.HASH]
         size = props[self.SIZE]
         
-        src_v_id, dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, hash, permissions)
+        is_copy = not is_move and (src_account, src_container, src_name) != (dest_account, dest_container, dest_name) # New uuid.
+        src_v_id, dest_version_id = self._update_object_hash(user, dest_account, dest_container, dest_name, size, hash, permissions, is_copy=is_copy)
         self._put_metadata_duplicate(src_version_id, dest_version_id, dest_domain, dest_meta, replace_meta)
         return dest_version_id
     
@@ -571,7 +573,7 @@ class ModularBackend(BaseBackend):
         """Copy an object's data and metadata."""
         
         logger.debug("copy_object: %s %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version)
-        return self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version)
+        return self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, src_version, False)
     
     @backend_method
     def move_object(self, user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta={}, replace_meta=False, permissions=None):
@@ -580,7 +582,7 @@ class ModularBackend(BaseBackend):
         logger.debug("move_object: %s %s %s %s %s %s %s %s %s %s", src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions)
         if user != src_account:
             raise NotAllowedError
-        dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, None)
+        dest_version_id = self._copy_object(user, src_account, src_container, src_name, dest_account, dest_container, dest_name, domain, meta, replace_meta, permissions, None, True)
         if (src_account, src_container, src_name) != (dest_account, dest_container, dest_name):
             self._delete_object(user, src_account, src_container, src_name)
         return dest_version_id
@@ -606,7 +608,7 @@ class ModularBackend(BaseBackend):
             return
         
         path, node = self._lookup_object(account, container, name)
-        src_version_id, dest_version_id = self._put_version_duplicate(user, node, 0, None, CLUSTER_DELETED)
+        src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, hash=None, cluster=CLUSTER_DELETED)
         self._apply_versioning(account, container, src_version_id)
         self.permissions.access_clear(path)
     
@@ -667,6 +669,9 @@ class ModularBackend(BaseBackend):
     
     # Path functions.
     
+    def _generate_uuid():
+        return str(uuid.uuid4())
+    
     def _put_object_node(self, path, parent, name):
         path = '/'.join((path, name))
         node = self.node.node_lookup(path)
@@ -676,7 +681,7 @@ class ModularBackend(BaseBackend):
     
     def _put_path(self, user, parent, path):
         node = self.node.node_create(parent, path)
-        self.node.version_create(node, None, 0, None, user, CLUSTER_NORMAL)
+        self.node.version_create(node, None, 0, None, user, self._generate_uuid(), CLUSTER_NORMAL)
         return node
     
     def _lookup_account(self, account, create=True):
@@ -736,7 +741,7 @@ class ModularBackend(BaseBackend):
                 raise IndexError('Version does not exist')
         return props
     
-    def _put_version_duplicate(self, user, node, size=None, hash=None, cluster=CLUSTER_NORMAL):
+    def _put_version_duplicate(self, user, node, size=None, hash=None, cluster=CLUSTER_NORMAL, is_copy=False):
         """Create a new version of the node."""
         
         props = self.node.version_lookup(node, inf, CLUSTER_NORMAL)
@@ -751,10 +756,11 @@ class ModularBackend(BaseBackend):
         if size is None:
             hash = src_hash # This way hash can be set to None.
             size = src_size
+        uniq = self._generate_uuid() if is_copy or src_version_id is None else props[self.UUID]
         
         if src_version_id is not None:
             self.node.version_recluster(src_version_id, CLUSTER_HISTORY)
-        dest_version_id, mtime = self.node.version_create(node, hash, size, src_version_id, user, cluster)
+        dest_version_id, mtime = self.node.version_create(node, hash, size, src_version_id, user, uniq, cluster)
         return src_version_id, dest_version_id
     
     def _put_metadata_duplicate(self, src_version_id, dest_version_id, domain, meta, replace=False):