Pool pithos backends using ObjectPool

Use ObjectPool functionality from synnefo.lib.pool
to pool backend objects.

This reduces the per-request overhead significantly,
since each backend object is only initialized once,
and connections to the DB are persistent.

Also bump snf-common dependency to reflect need for
updated snf-common with ObjectPool support.

Signed-off-by: Georgios D. Tsoukalas <gtsouk@cslab.ece.ntua.gr>
Signed-off-by: Vangelis Koukis <vkoukis@grnet.gr>

diff --git a/snf-pithos-app/pithos/api/util.py b/snf-pithos-app/pithos/api/util.py
index ebfc037..fc10f42 100644 (file)
--- a/snf-pithos-app/pithos/api/util.py
+++ b/snf-pithos-app/pithos/api/util.py
@@ -788,7 +788,8 @@ def simple_list_response(request, l):
     if request.serialization == 'json':
         return json.dumps(l)
 
     if request.serialization == 'json':
         return json.dumps(l)
 

-def get_backend():
+
+def _get_backend():

     backend = connect_backend(db_module=BACKEND_DB_MODULE,
                               db_connection=BACKEND_DB_CONNECTION,
                               block_module=BACKEND_BLOCK_MODULE,
     backend = connect_backend(db_module=BACKEND_DB_MODULE,
                               db_connection=BACKEND_DB_CONNECTION,
                               block_module=BACKEND_BLOCK_MODULE,

@@ -800,6 +801,43 @@ def get_backend():
     backend.default_policy['versioning'] = BACKEND_VERSIONING
     return backend
 
     backend.default_policy['versioning'] = BACKEND_VERSIONING
     return backend
 

+
+def _pooled_backend_close(backend):
+    backend._pool.pool_put(backend)
+
+
+from synnefo.lib.pool import ObjectPool
+from new import instancemethod
+
+USAGE_LIMIT = 500
+POOL_SIZE = 5
+
+class PithosBackendPool(ObjectPool):
+    def _pool_create(self):
+        backend = _get_backend()
+        backend._real_close = backend.close
+        backend.close = instancemethod(_pooled_backend_close, backend,
+                                       type(backend))
+        backend._pool = self
+        backend._use_count = USAGE_LIMIT
+        return backend
+
+    def _pool_cleanup(self, backend):
+        c = backend._use_count - 1
+        if c < 0:
+            backend._real_close()
+            return True
+
+        backend._use_count = c
+        return False
+
+_pithos_backend_pool = PithosBackendPool(size=POOL_SIZE)
+
+
+def get_backend():
+    return _pithos_backend_pool.pool_get()
+
+

 def update_request_headers(request):
     # Handle URL-encoded keys and values.
     meta = dict([(k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
 def update_request_headers(request):
     # Handle URL-encoded keys and values.
     meta = dict([(k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])

@@ -870,6 +908,7 @@ def request_serialization(request, format_allowed=False):
     
     return 'text'
 
     
     return 'text'
 

+

 def api_method(http_method=None, format_allowed=False, user_required=True):
     """Decorator function for views that implement an API method."""
     
 def api_method(http_method=None, format_allowed=False, user_required=True):
     """Decorator function for views that implement an API method."""
     

diff --git a/snf-pithos-app/setup.py b/snf-pithos-app/setup.py
index 1809df3..c02ca92 100644 (file)
--- a/snf-pithos-app/setup.py
+++ b/snf-pithos-app/setup.py
@@ -65,7 +65,7 @@ CLASSIFIERS = []
 
 # Package requirements
 INSTALL_REQUIRES = [
 
 # Package requirements
 INSTALL_REQUIRES = [

-    'snf-common>0.9.13',
+    'snf-common>0.10.0',

     'snf-pithos-backend>0.9.7',
     'Django>=1.2, <1.3'
 ]
     'snf-pithos-backend>0.9.7',
     'Django>=1.2, <1.3'
 ]