cleanup pithos backend pools, new pool api support

[pithos] / snf-pithos-app / pithos / api / util.py

diff --git a/snf-pithos-app/pithos/api/util.py b/snf-pithos-app/pithos/api/util.py
index 28ebed9..f8233f9 100644 (file)
--- a/snf-pithos-app/pithos/api/util.py
+++ b/snf-pithos-app/pithos/api/util.py
@@ -64,7 +64,7 @@ from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
                                     SERVICE_TOKEN, COOKIE_NAME)
 
 from pithos.backends import connect_backend
-from pithos.backends.base import NotAllowedError, QuotaError
+from pithos.backends.base import NotAllowedError, QuotaError, ItemNotExists, VersionNotExists
 
 import logging
 import re
@@ -320,7 +320,7 @@ def split_container_object_string(s):
         raise ValueError
     return s[:pos], s[(pos + 1):]
 
-def copy_or_move_object(request, src_account, src_container, src_name, dest_account, dest_container, dest_name, move=False):
+def copy_or_move_object(request, src_account, src_container, src_name, dest_account, dest_container, dest_name, move=False, delimiter=None):
     """Copy or move an object."""
     
     if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
@@ -331,14 +331,14 @@ def copy_or_move_object(request, src_account, src_container, src_name, dest_acco
         if move:
             version_id = request.backend.move_object(request.user_uniq, src_account, src_container, src_name,
                                                         dest_account, dest_container, dest_name,
-                                                        content_type, 'pithos', meta, False, permissions)
+                                                        content_type, 'pithos', meta, False, permissions, delimiter)
         else:
             version_id = request.backend.copy_object(request.user_uniq, src_account, src_container, src_name,
                                                         dest_account, dest_container, dest_name,
-                                                        content_type, 'pithos', meta, False, permissions, src_version)
+                                                        content_type, 'pithos', meta, False, permissions, src_version, delimiter)
     except NotAllowedError:
         raise Forbidden('Not allowed')
-    except (NameError, IndexError):
+    except (ItemNotExists, VersionNotExists):
         raise ItemNotFound('Container or object does not exist')
     except ValueError:
         raise BadRequest('Invalid sharing header')
@@ -349,7 +349,7 @@ def copy_or_move_object(request, src_account, src_container, src_name, dest_acco
             request.backend.update_object_public(request.user_uniq, dest_account, dest_container, dest_name, public)
         except NotAllowedError:
             raise Forbidden('Not allowed')
-        except NameError:
+        except ItemNotExists:
             raise ItemNotFound('Object does not exist')
     return version_id
 
@@ -660,7 +660,7 @@ class ObjectWrapper(object):
                 self.block_hash = self.hashmaps[self.file_index][self.block_index]
                 try:
                     self.block = self.backend.get_block(self.block_hash)
-                except NameError:
+                except ItemNotExists:
                     raise ItemNotFound('Block does not exist')
             
             # Get the data from the block.
@@ -788,7 +788,8 @@ def simple_list_response(request, l):
     if request.serialization == 'json':
         return json.dumps(l)
 
-def get_backend():
+
+def _get_backend():
     backend = connect_backend(db_module=BACKEND_DB_MODULE,
                               db_connection=BACKEND_DB_CONNECTION,
                               block_module=BACKEND_BLOCK_MODULE,
@@ -800,6 +801,50 @@ def get_backend():
     backend.default_policy['versioning'] = BACKEND_VERSIONING
     return backend
 
+
+def _pooled_backend_close(backend):
+    backend._pool.pool_put(backend)
+
+
+from synnefo.lib.pool import ObjectPool
+from new import instancemethod
+
+USAGE_LIMIT = 500
+POOL_SIZE = 5
+
+class PithosBackendPool(ObjectPool):
+    def _pool_create(self):
+        backend = _get_backend()
+        backend._real_close = backend.close
+        backend.close = instancemethod(_pooled_backend_close, backend,
+                                       type(backend))
+        backend._pool = self
+        backend._use_count = USAGE_LIMIT
+        return backend
+
+    def _pool_verify(self, backend):
+        return 1
+
+    def _pool_cleanup(self, backend):
+        c = backend._use_count - 1
+        if c < 0:
+            backend._real_close()
+            return True
+
+        backend._use_count = c
+        if backend.trans is not None:
+            backend.wrapper.rollback()
+        if backend.messages:
+            backend.messages = []
+        return False
+
+_pithos_backend_pool = PithosBackendPool(size=POOL_SIZE)
+
+
+def get_backend():
+    return _pithos_backend_pool.pool_get()
+
+
 def update_request_headers(request):
     # Handle URL-encoded keys and values.
     meta = dict([(k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
@@ -870,6 +915,7 @@ def request_serialization(request, format_allowed=False):
     
     return 'text'
 
+
 def api_method(http_method=None, format_allowed=False, user_required=True):
     """Decorator function for views that implement an API method."""
     
@@ -886,7 +932,7 @@ def api_method(http_method=None, format_allowed=False, user_required=True):
                         cookie_value = unquote(request.COOKIES.get(COOKIE_NAME, ''))
                         if cookie_value and '|' in cookie_value:
                             token = cookie_value.split('|', 1)[1]
-                    get_user(request, IDENTITY_BASEURL, AUTHENTICATION_USERS, token)
+                    get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS, token)
                     if  getattr(request, 'user', None) is None:
                         raise Unauthorized('Access denied')