root / snf-common / synnefo / lib / pool / http.py @ 68453d22
History | View | Annotate | Download (4.6 kB)
1 |
# Copyright 2011-2012 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or
|
4 |
# without modification, are permitted provided that the following
|
5 |
# conditions are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above
|
8 |
# copyright notice, this list of conditions and the following
|
9 |
# disclaimer.
|
10 |
#
|
11 |
# 2. Redistributions in binary form must reproduce the above
|
12 |
# copyright notice, this list of conditions and the following
|
13 |
# disclaimer in the documentation and/or other materials
|
14 |
# provided with the distribution.
|
15 |
#
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 |
# POSSIBILITY OF SUCH DAMAGE.
|
28 |
#
|
29 |
# The views and conclusions contained in the software and
|
30 |
# documentation are those of the authors and should not be
|
31 |
# interpreted as representing official policies, either expressed
|
32 |
# or implied, of GRNET S.A.
|
33 |
|
34 |
from synnefo.lib.pool import ObjectPool |
35 |
from select import select |
36 |
|
37 |
from httplib import ( |
38 |
HTTPConnection as http_class,
|
39 |
HTTPSConnection as https_class,
|
40 |
ResponseNotReady |
41 |
) |
42 |
|
43 |
from new import instancemethod |
44 |
|
45 |
import logging |
46 |
|
47 |
log = logging.getLogger(__name__) |
48 |
|
49 |
_pools = {} |
50 |
pool_size = 8
|
51 |
|
52 |
|
53 |
USAGE_LIMIT = 1000
|
54 |
|
55 |
|
56 |
def init_http_pooling(size): |
57 |
global pool_size
|
58 |
pool_size = size |
59 |
|
60 |
|
61 |
def put_http_connection(conn): |
62 |
pool = conn._pool |
63 |
log.debug("HTTP-PUT-BEFORE: putting connection %r back to pool %r", conn, pool)
|
64 |
if pool is None: |
65 |
log.debug("HTTP-PUT: connection %r does not have a pool", conn)
|
66 |
return
|
67 |
# conn._pool = None
|
68 |
pool.pool_put(conn) |
69 |
|
70 |
|
71 |
class HTTPConnectionPool(ObjectPool): |
72 |
|
73 |
_scheme_to_class = { |
74 |
'http' : http_class,
|
75 |
'https' : https_class,
|
76 |
} |
77 |
|
78 |
def __init__(self, scheme, netloc, size=None): |
79 |
log.debug("INIT-POOL: Initializing pool of size %d, scheme: %s, netloc: %s",
|
80 |
size, scheme, netloc) |
81 |
ObjectPool.__init__(self, size=size)
|
82 |
|
83 |
connection_class = self._scheme_to_class.get(scheme, None) |
84 |
if connection_class is None: |
85 |
m = 'Unsupported scheme: %s' % (scheme,)
|
86 |
raise ValueError(m) |
87 |
|
88 |
self.connection_class = connection_class
|
89 |
self.scheme = scheme
|
90 |
self.netloc = netloc
|
91 |
|
92 |
def _pool_create(self): |
93 |
log.debug("CREATE-HTTP-BEFORE from pool %r", self) |
94 |
conn = self.connection_class(self.netloc) |
95 |
conn._use_counter = USAGE_LIMIT |
96 |
conn._pool = self
|
97 |
conn._real_close = conn.close |
98 |
conn.close = instancemethod(put_http_connection, conn, type(conn))
|
99 |
return conn
|
100 |
|
101 |
def _pool_verify(self, conn): |
102 |
log.debug("VERIFY-HTTP")
|
103 |
if conn is None: |
104 |
return False |
105 |
sock = conn.sock |
106 |
if sock is None: |
107 |
return True |
108 |
if select((conn.sock,), (), (), 0)[0]: |
109 |
return False |
110 |
return True |
111 |
|
112 |
def _pool_cleanup(self, conn): |
113 |
log.debug("CLEANUP-HTTP")
|
114 |
# every connection can be used a finite number of times
|
115 |
conn._use_counter -= 1
|
116 |
|
117 |
# see httplib source for connection states documentation
|
118 |
if conn._use_counter > 0 and conn._HTTPConnection__state == 'Idle': |
119 |
try:
|
120 |
conn.getresponse() |
121 |
except ResponseNotReady:
|
122 |
log.debug("CLEANUP-HTTP: Not closing connection. Will reuse.")
|
123 |
return False |
124 |
|
125 |
log.debug("CLEANUP-HTTP: Closing connection. Will not reuse.")
|
126 |
conn._real_close() |
127 |
return True |
128 |
|
129 |
|
130 |
def get_http_connection(netloc=None, scheme='http', pool_size=pool_size): |
131 |
log.debug("HTTP-GET: Getting HTTP connection")
|
132 |
if netloc is None: |
133 |
m = "netloc cannot be None"
|
134 |
raise ValueError(m) |
135 |
# does the pool need to be created?
|
136 |
if netloc not in _pools: |
137 |
log.debug("HTTP-GET: Creating pool for netloc %s", netloc)
|
138 |
pool = HTTPConnectionPool(scheme, netloc, size=pool_size) |
139 |
_pools[netloc] = pool |
140 |
|
141 |
obj = _pools[netloc].pool_get() |
142 |
log.debug("HTTP-GET: Returning object %r", obj)
|
143 |
return obj
|