Revision 4ab1af1a snf-common/synnefo/lib/pool/http.py

b/snf-common/synnefo/lib/pool/http.py
31 31
# interpreted as representing official policies, either expressed
32 32
# or implied, of GRNET S.A.
33 33

  
34
from synnefo.lib.pool import ObjectPool
34
from synnefo.lib.pool import ObjectPool, PooledObject
35 35
from select import select
36 36

  
37 37
from httplib import (
......
40 40
    ResponseNotReady,
41 41
)
42 42

  
43
from new import instancemethod
43
from threading import Lock
44 44

  
45 45
import logging
46 46

  
47 47
log = logging.getLogger(__name__)
48 48

  
49 49
_pools = {}
50
pool_size = 8
51

  
50
_pools_mutex = Lock()
52 51

  
52
default_pool_size = 100
53 53
USAGE_LIMIT = 1000
54 54

  
55 55

  
56 56
def init_http_pooling(size):
57
    global pool_size
58
    pool_size = size
59

  
60

  
61
def put_http_connection(conn):
62
    pool = conn._pool
63
    log.debug("HTTP-PUT-BEFORE: putting connection %r back to pool %r",
64
              conn, pool)
65
    if pool is None:
66
        log.debug("HTTP-PUT: connection %r does not have a pool", conn)
67
        return
68
    conn._pool = None
69
    pool.pool_put(conn)
57
    global default_pool_size
58
    default_pool_size = size
70 59

  
71 60

  
72 61
class HTTPConnectionPool(ObjectPool):
......
93 82
    def _pool_create(self):
94 83
        log.debug("CREATE-HTTP-BEFORE from pool %r", self)
95 84
        conn = self.connection_class(self.netloc)
96
        conn._use_counter = USAGE_LIMIT
97
        conn._pool = self
98
        conn._real_close = conn.close
99
        conn.close = instancemethod(put_http_connection, conn, type(conn))
85
        conn._pool_use_counter = USAGE_LIMIT
100 86
        return conn
101 87

  
102 88
    def _pool_verify(self, conn):
103 89
        log.debug("VERIFY-HTTP")
104
        # _pool verify is called at every pool_get().
105
        # Make sure this connection obj is associated with the proper pool.
106
        # The association is broken by put_http_connection(), to prevent
107
        # a connection object from being returned to the pool twice,
108
        # on duplicate invocations of conn.close().
109 90
        if conn is None:
110 91
            return False
111
        if not conn._pool:
112
            conn._pool = self
113 92
        sock = conn.sock
114 93
        if sock is None:
115 94
            return True
......
120 99
    def _pool_cleanup(self, conn):
121 100
        log.debug("CLEANUP-HTTP")
122 101
        # every connection can be used a finite number of times
123
        conn._use_counter -= 1
102
        conn._pool_use_counter -= 1
124 103

  
125 104
        # see httplib source for connection states documentation
126
        if conn._use_counter > 0 and conn._HTTPConnection__state == 'Idle':
105
        if (conn._pool_use_counter > 0 and
106
            conn._HTTPConnection__state == 'Idle'):
127 107
            try:
128 108
                conn.getresponse()
129 109
            except ResponseNotReady:
......
131 111
                return False
132 112

  
133 113
        log.debug("CLEANUP-HTTP: Closing connection. Will not reuse.")
134
        conn._real_close()
114
        conn.close()
135 115
        return True
136 116

  
137 117

  
138
def get_http_connection(netloc=None, scheme='http', pool_size=pool_size):
139
    log.debug("HTTP-GET: Getting HTTP connection")
140
    if netloc is None:
141
        m = "netloc cannot be None"
142
        raise ValueError(m)
143
    # does the pool need to be created?
144
    # ensure distinct pools are created for every (scheme, netloc) combination
145
    key = (scheme, netloc)
146
    if key not in _pools:
147
        log.debug("HTTP-GET: Creating pool for key %s", key)
148
        pool = HTTPConnectionPool(scheme, netloc, size=pool_size)
149
        _pools[key] = pool
150

  
151
    obj = _pools[key].pool_get()
152
    log.debug("HTTP-GET: Returning object %r", obj)
153
    return obj
118
class PooledHTTPConnection(PooledObject):
119

  
120
    _pool_log_prefix = "HTTP"
121
    _pool_class = HTTPConnectionPool
122

  
123
    def __init__(self, netloc, scheme='http', pool=None, **kw):
124
        kw['netloc'] = netloc
125
        kw['scheme'] = scheme
126
        kw['pool'] = pool
127
        super(PooledHTTPConnection, self).__init__(**kw)
128

  
129
    def get_pool(self):
130
        kwargs = self._pool_kwargs
131
        pool = kwargs.pop('pool', None)
132
        if pool is not None:
133
            return pool
134

  
135
        # pool was not given, find one from the global registry
136
        scheme = kwargs['scheme']
137
        netloc = kwargs['netloc']
138
        size = kwargs.get('size', default_pool_size)
139
        # ensure distinct pools for every (scheme, netloc) combination
140
        key = (scheme, netloc)
141
        with _pools_mutex:
142
            if key not in _pools:
143
                log.debug("HTTP-GET: Creating pool for key %s", key)
144
                pool = HTTPConnectionPool(scheme, netloc, size=size)
145
                _pools[key] = pool
146
            else:
147
                pool = _pools[key]
148

  
149
        return pool

Also available in: Unified diff