By default report size change in object delete/update (do not include older version...

diff --git a/snf-pithos-app/README b/snf-pithos-app/README
index 2fdf996..185b5db 100644 (file)
--- a/snf-pithos-app/README
+++ b/snf-pithos-app/README
@@ -42,6 +42,7 @@ PITHOS_BACKEND_QUEUE_HOSTS       None
 PITHOS_BACKEND_QUEUE_EXCHANGE    pithos
 PITHOS_BACKEND_QUOTA             50 GB (50 * 1024 ** 3)                            Default user quota
 PITHOS_BACKEND_VERSIONING        auto                                              Default versioning policy for containers
+PITHOS_BACKEND_FREE_VERSIONING   True                                              Default versioning debit policy (default free)
 PITHOS_UPDATE_MD5                True                                              Update object checksums when using hashmaps
 PITHOS_SERVICE_TOKEN             ''                                                Service token acquired by the identity provider (astakos)
 ===============================  ================================================  ============================================================

diff --git a/snf-pithos-app/conf/20-snf-pithos-app-settings.conf b/snf-pithos-app/conf/20-snf-pithos-app-settings.conf
index aa3c999..03b03a6 100644 (file)
--- a/snf-pithos-app/conf/20-snf-pithos-app-settings.conf
+++ b/snf-pithos-app/conf/20-snf-pithos-app-settings.conf
@@ -28,6 +28,7 @@ PITHOS_BACKEND_BLOCK_PATH = '/usr/share/synnefo/pithos/data'
 # Default setting for new accounts.
 #PITHOS_BACKEND_QUOTA = 50 * 1024 * 1024 * 1024
 #PITHOS_BACKEND_VERSIONING = 'auto'
+#PITHOS_BACKEND_FREE_VERSIONING = True
 
 # Disable if checksums are not required or are computed asynchronously by an external process.
 #PITHOS_UPDATE_MD5 = True

diff --git a/snf-pithos-app/pithos/api/dispatch.py b/snf-pithos-app/pithos/api/dispatch.py
index 521b467..dbc5719 100644 (file)
--- a/snf-pithos-app/pithos/api/dispatch.py
+++ b/snf-pithos-app/pithos/api/dispatch.py
@@ -1,11 +1,14 @@
-from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
-                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
-                                 BACKEND_BLOCK_UMASK,
-                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
-                                 BACKEND_QUEUE_EXCHANGE,
-                                 BACKEND_QUOTA, BACKEND_VERSIONING)
-from pithos.backends import connect_backend
-from pithos.api.util import hashmap_md5
+# from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
+#                                  BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
+#                                  BACKEND_BLOCK_UMASK,
+#                                  BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
+#                                  BACKEND_QUEUE_EXCHANGE,
+#                                  BACKEND_QUOTA, BACKEND_VERSIONING,
+#                                  BACKEND_FREE_VERSIONING)
+from pithos.api.settings import (BACKEND_QUOTA, BACKEND_VERSIONING)
+
+#from pithos.backends import connect_backend
+from pithos.api.util import hashmap_md5, get_backend
 
 from django.core.mail import send_mail
 from django.utils.translation import ugettext as _
@@ -20,14 +23,16 @@ def update_md5(m):
     if m['resource'] != 'object' or m['details']['action'] != 'object update':
         return
 
-    backend = connect_backend(db_module=BACKEND_DB_MODULE,
-                              db_connection=BACKEND_DB_CONNECTION,
-                              block_module=BACKEND_BLOCK_MODULE,
-                              block_path=BACKEND_BLOCK_PATH,
-                              block_umask=BACKEND_BLOCK_UMASK,
-                              queue_module=BACKEND_QUEUE_MODULE,
-                              queue_hosts=BACKEND_QUEUE_HOSTS,
-                              queue_exchange=BACKEND_QUEUE_EXHANGE)
+#     backend = connect_backend(db_module=BACKEND_DB_MODULE,
+#                               db_connection=BACKEND_DB_CONNECTION,
+#                               block_module=BACKEND_BLOCK_MODULE,
+#                               block_path=BACKEND_BLOCK_PATH,
+#                               block_umask=BACKEND_BLOCK_UMASK,
+#                               queue_module=BACKEND_QUEUE_MODULE,
+#                               queue_hosts=BACKEND_QUEUE_HOSTS,
+#                               queue_exchange=BACKEND_QUEUE_EXHANGE
+#                               free_versioning=BACKEND_FREE_VERSIONING)
+    backend = get_backend()
     backend.default_policy['quota'] = BACKEND_QUOTA
     backend.default_policy['versioning'] = BACKEND_VERSIONING
 

diff --git a/snf-pithos-app/pithos/api/management/commands/storagequota.py b/snf-pithos-app/pithos/api/management/commands/storagequota.py
index 7e6f12d..ef508ba 100644 (file)
--- a/snf-pithos-app/pithos/api/management/commands/storagequota.py
+++ b/snf-pithos-app/pithos/api/management/commands/storagequota.py
@@ -35,14 +35,9 @@ from optparse import make_option
 
 from django.core.management.base import BaseCommand, CommandError
 
-from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
-                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
-                                 BACKEND_BLOCK_UMASK,
-                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
-                                 BACKEND_QUEUE_EXCHANGE,
-                                 BACKEND_QUOTA, BACKEND_VERSIONING)
-from pithos.backends import connect_backend
+from pithos.api.settings import (BACKEND_QUOTA, BACKEND_VERSIONING)
 
+from pithos.api.util import get_backend
 
 class Command(BaseCommand):
     args = "<user>"
@@ -67,14 +62,7 @@ class Command(BaseCommand):
             except ValueError:
                 raise CommandError("Invalid quota")
 
-        backend = connect_backend(db_module=BACKEND_DB_MODULE,
-                                  db_connection=BACKEND_DB_CONNECTION,
-                                  block_module=BACKEND_BLOCK_MODULE,
-                                  block_path=BACKEND_BLOCK_PATH,
-                                  block_umask=BACKEND_BLOCK_UMASK,
-                                  queue_module=BACKEND_QUEUE_MODULE,
-                                  queue_connection=BACKEND_QUEUE_HOSTS,
-                                  queue_exchange=BACKEND_QUEUE_EXCHANGE)
+        backend = get_backend()
         backend.default_policy['quota'] = BACKEND_QUOTA
         backend.default_policy['versioning'] = BACKEND_VERSIONING
         if quota is not None:

diff --git a/snf-pithos-app/pithos/api/settings.py b/snf-pithos-app/pithos/api/settings.py
index 7b78707..028495e 100644 (file)
--- a/snf-pithos-app/pithos/api/settings.py
+++ b/snf-pithos-app/pithos/api/settings.py
@@ -44,6 +44,7 @@ BACKEND_QUEUE_EXCHANGE = getattr(settings, 'PITHOS_BACKEND_QUEUE_EXCHANGE', 'pit
 BACKEND_QUOTA = getattr(
     settings, 'PITHOS_BACKEND_QUOTA', 50 * 1024 * 1024 * 1024)
 BACKEND_VERSIONING = getattr(settings, 'PITHOS_BACKEND_VERSIONING', 'auto')
+BACKEND_FREE_VERSIONING = getattr(settings, 'PITHOS_BACKEND_FREE_VERSIONING', True)
 
 # Update object checksums when using hashmaps.
 UPDATE_MD5 = getattr(settings, 'PITHOS_UPDATE_MD5', True)

diff --git a/snf-pithos-app/pithos/api/util.py b/snf-pithos-app/pithos/api/util.py
index 3625249..362f58e 100644 (file)
--- a/snf-pithos-app/pithos/api/util.py
+++ b/snf-pithos-app/pithos/api/util.py
@@ -62,6 +62,7 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
                                  BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
                                  BACKEND_QUEUE_EXCHANGE,
                                  BACKEND_QUOTA, BACKEND_VERSIONING,
+                                 BACKEND_FREE_VERSIONING,
                                  AUTHENTICATION_URL, AUTHENTICATION_USERS,
                                  SERVICE_TOKEN, COOKIE_NAME)
 
@@ -860,7 +861,9 @@ _pithos_backend_pool = PithosBackendPool(size=POOL_SIZE,
                                          block_path=BACKEND_BLOCK_PATH,
                                          block_umask=BACKEND_BLOCK_UMASK,
                                          queue_module=BACKEND_QUEUE_MODULE,
-                                         queue_connection=BACKEND_QUEUE_CONNECTION)
+                                         queue_hosts=BACKEND_QUEUE_HOSTS,
+                                         queue_exchange=BACKEND_QUEUE_EXCHANGE,
+                                         free_versioning=BACKEND_FREE_VERSIONING)
 
 
 def get_backend():

diff --git a/snf-pithos-backend/pithos/backends/modular.py b/snf-pithos-backend/pithos/backends/modular.py
index 88007e9..1ec45f6 100644 (file)
--- a/snf-pithos-backend/pithos/backends/modular.py
+++ b/snf-pithos-backend/pithos/backends/modular.py
@@ -129,7 +129,8 @@ class ModularBackend(BaseBackend):
     def __init__(self, db_module=None, db_connection=None,
                  block_module=None, block_path=None, block_umask=None,
                  queue_module=None, queue_hosts=None,
-                 queue_exchange=None):
+                 queue_exchange=None,
+                 free_versioning=True):
         db_module = db_module or DEFAULT_DB_MODULE
         db_connection = db_connection or DEFAULT_DB_CONNECTION
         block_module = block_module or DEFAULT_BLOCK_MODULE
@@ -138,9 +139,10 @@ class ModularBackend(BaseBackend):
         #queue_module = queue_module or DEFAULT_QUEUE_MODULE
         #queue_hosts = queue_hosts or DEFAULT_QUEUE_HOSTS
         #queue_exchange = queue_exchange or DEFAULT_QUEUE_EXCHANGE
-               
+        
         self.hash_algorithm = 'sha256'
         self.block_size = 4 * 1024 * 1024  # 4MB
+        self.free_versioning = free_versioning
 
         self.default_policy = {'quota': DEFAULT_QUOTA,
                                'versioning': DEFAULT_VERSIONING}
@@ -171,7 +173,7 @@ class ModularBackend(BaseBackend):
         if queue_module and queue_hosts:
             self.queue_module = load_module(queue_module)
             params = {'hosts': queue_hosts,
-                         'exchange': queue_exchange,
+                      'exchange': queue_exchange,
                       'client_id': QUEUE_CLIENT_ID}
             self.queue = self.queue_module.Queue(**params)
         else:
@@ -489,8 +491,8 @@ class ModularBackend(BaseBackend):
                 self.store.map_delete(h)
             self.node.node_purge_children(node, until, CLUSTER_DELETED)
             self._report_size_change(user, account, -size,
-                                                        {'action':'container purge', 'path': path,
-                                                         'versions': ','.join(str(i) for i in serials)})
+                                     {'action':'container purge', 'path': path,
+                                      'versions': ','.join(str(i) for i in serials)})
             return
 
         if not delimiter:
@@ -503,11 +505,11 @@ class ModularBackend(BaseBackend):
             self.node.node_purge_children(node, inf, CLUSTER_DELETED)
             self.node.node_remove(node)
             self._report_size_change(user, account, -size,
-                                                        {'action': 'container delete',
-                                                         'path': path,
-                                                         'versions': ','.join(str(i) for i in serials)})
+                                     {'action': 'container delete',
+                                      'path': path,
+                                      'versions': ','.join(str(i) for i in serials)})
         else:
-                # remove only contents
+            # remove only contents
             src_names = self._list_objects_no_limit(user, account, container, prefix='', delimiter=None, virtual=False, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False)
             paths = []
             for t in src_names:
@@ -516,11 +518,13 @@ class ModularBackend(BaseBackend):
                 src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED)
                 del_size = self._apply_versioning(
                     account, container, src_version_id)
-                if del_size:
-                    self._report_size_change(user, account, -del_size,
-                                                                {'action': 'object delete',
-                                                                 'path': path,
-                                                                 'versions': ','.join([str(dest_version_id)])})
+                self._report_size_change(
+                       user, account, -del_size, {
+                               'action': 'object delete',
+                               'path': path,
+                       'versions': ','.join([str(dest_version_id)])
+                     }
+                )
                 self._report_object_change(
                     user, account, path, details={'action': 'object delete'})
                 paths.append(path)
@@ -800,19 +804,19 @@ class ModularBackend(BaseBackend):
         self._put_metadata_duplicate(
             src_version_id, dest_version_id, domain, meta, replace_meta)
 
-        # Check quota.
         del_size = self._apply_versioning(account, container, pre_version_id)
         size_delta = size - del_size
-        if size_delta > 0:
-            account_quota = long(self._get_policy(account_node)['quota'])
-            container_quota = long(self._get_policy(container_node)['quota'])
-            if (account_quota > 0 and self._get_statistics(account_node)[1] + size_delta > account_quota) or \
-               (container_quota > 0 and self._get_statistics(container_node)[1] + size_delta > container_quota):
-                # This must be executed in a transaction, so the version is never created if it fails.
-                raise QuotaError
+#        # Check quota.
+#         if size_delta > 0:
+#             account_quota = long(self._get_policy(account_node)['quota'])
+#             container_quota = long(self._get_policy(container_node)['quota'])
+#             if (account_quota > 0 and self._get_statistics(account_node)[1] + size_delta > account_quota) or \
+#                (container_quota > 0 and self._get_statistics(container_node)[1] + size_delta > container_quota):
+#                 # This must be executed in a transaction, so the version is never created if it fails.
+#                 raise QuotaError
         self._report_size_change(user, account, size_delta,
-                                                        {'action': 'object update', 'path': path,
-                                                         'versions': ','.join([str(dest_version_id)])})
+                                 {'action': 'object update', 'path': path,
+                                  'versions': ','.join([str(dest_version_id)])})
 
         if permissions is not None:
             self.permissions.access_set(path, permissions)
@@ -943,17 +947,16 @@ class ModularBackend(BaseBackend):
             except NameError:
                 self.permissions.access_clear(path)
             self._report_size_change(user, account, -size,
-                                                       {'action': 'object purge', 'path': path,
-                                                        'versions': ','.join(str(i) for i in serials)})
+                                    {'action': 'object purge', 'path': path,
+                                     'versions': ','.join(str(i) for i in serials)})
             return
 
         path, node = self._lookup_object(account, container, name)
         src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED)
         del_size = self._apply_versioning(account, container, src_version_id)
-        if del_size:
-            self._report_size_change(user, account, -del_size,
-                                                        {'action': 'object delete', 'path': path,
-                                                         'versions': ','.join([str(dest_version_id)])})
+        self._report_size_change(user, account, -del_size,
+                                 {'action': 'object delete', 'path': path,
+                                  'versions': ','.join([str(dest_version_id)])})
         self._report_object_change(
             user, account, path, details={'action': 'object delete'})
         self.permissions.access_clear(path)
@@ -968,11 +971,10 @@ class ModularBackend(BaseBackend):
                 src_version_id, dest_version_id = self._put_version_duplicate(user, node, size=0, type='', hash=None, checksum='', cluster=CLUSTER_DELETED)
                 del_size = self._apply_versioning(
                     account, container, src_version_id)
-                if del_size:
-                    self._report_size_change(user, account, -del_size,
-                                                                {'action': 'object delete',
-                                                                 'path': path,
-                                                                 'versions': ','.join([str(dest_version_id)])})
+                self._report_size_change(user, account, -del_size,
+                                         {'action': 'object delete',
+                                          'path': path,
+                                          'versions': ','.join([str(dest_version_id)])})
                 self._report_object_change(
                     user, account, path, details={'action': 'object delete'})
                 paths.append(path)
@@ -1230,22 +1232,22 @@ class ModularBackend(BaseBackend):
         logger.debug(
             "_report_size_change: %s %s %s %s", user, account, size, details)
         self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('resource.diskspace',), 
-                                                 account, QUEUE_INSTANCE_ID, 'diskspace',
-                                                 float(size), details))
+                              account, QUEUE_INSTANCE_ID, 'diskspace',
+                              float(size), details))
 
     def _report_object_change(self, user, account, path, details={}):
         details.update({'user': user})
         logger.debug("_report_object_change: %s %s %s %s", user,
                      account, path, details)
         self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('object',),
-                                                 account, QUEUE_INSTANCE_ID, 'object', path, details))
+                              account, QUEUE_INSTANCE_ID, 'object', path, details))
 
     def _report_sharing_change(self, user, account, path, details={}):
         logger.debug("_report_permissions_change: %s %s %s %s",
                      user, account, path, details)
         details.update({'user': user})
         self.messages.append((QUEUE_MESSAGE_KEY_PREFIX % ('sharing',),
-                                                 account, QUEUE_INSTANCE_ID, 'sharing', path, details))
+                              account, QUEUE_INSTANCE_ID, 'sharing', path, details))
 
     # Policy functions.
 
@@ -1285,7 +1287,7 @@ class ModularBackend(BaseBackend):
             return 0
         path, node = self._lookup_container(account, container)
         versioning = self._get_policy(node)['versioning']
-        if versioning != 'auto':
+        if versioning != 'auto' or self.free_versioning:
             hash, size = self.node.version_remove(version_id)
             self.store.map_delete(hash)
             return size

diff --git a/snf-pithos-backend/pithos/backends/util.py b/snf-pithos-backend/pithos/backends/util.py
index e8d9837..925387e 100644 (file)
--- a/snf-pithos-backend/pithos/backends/util.py
+++ b/snf-pithos-backend/pithos/backends/util.py
@@ -43,7 +43,9 @@ USAGE_LIMIT = 500
 class PithosBackendPool(ObjectPool):
     def __init__(self, size=None, db_module=None, db_connection=None,
                  block_module=None, block_path=None, block_umask=None,
-                 queue_module=None, queue_connection=None):
+                 queue_module=None, queue_hosts=None,
+                 queue_exchange=None,
+                 free_versioning=True):
         super(PithosBackendPool, self).__init__(size=size)
         self.db_module = db_module
         self.db_connection = db_connection
@@ -51,8 +53,10 @@ class PithosBackendPool(ObjectPool):
         self.block_path = block_path
         self.block_umask = block_umask
         self.queue_module = queue_module
-        self.queue_connection = queue_connection
-
+        self.queue_hosts = queue_hosts
+        self.queue_exchange = queue_exchange
+        self.free_versioning = free_versioning
+    
     def _pool_create(self):
         backend = connect_backend(db_module=self.db_module,
                                   db_connection=self.db_connection,
@@ -60,7 +64,9 @@ class PithosBackendPool(ObjectPool):
                                   block_path=self.block_path,
                                   block_umask=self.block_umask,
                                   queue_module=self.queue_module,
-                                  queue_connection=self.queue_connection)
+                                  queue_hosts=self.queue_hosts,
+                                  queue_exchange=self.queue_exchange,
+                                  free_versioning=self.free_versioning)
 
         backend._real_close = backend.close
         backend.close = instancemethod(_pooled_backend_close, backend,