Statistics
| Branch: | Tag: | Revision:

root / snf-common / synnefo / lib / pool / http.py @ 4ab1af1a

History | View | Annotate | Download (4.7 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from synnefo.lib.pool import ObjectPool, PooledObject
35
from select import select
36

    
37
from httplib import (
38
    HTTPConnection as http_class,
39
    HTTPSConnection as https_class,
40
    ResponseNotReady,
41
)
42

    
43
from threading import Lock
44

    
45
import logging
46

    
47
log = logging.getLogger(__name__)
48

    
49
_pools = {}
50
_pools_mutex = Lock()
51

    
52
default_pool_size = 100
53
USAGE_LIMIT = 1000
54

    
55

    
56
def init_http_pooling(size):
57
    global default_pool_size
58
    default_pool_size = size
59

    
60

    
61
class HTTPConnectionPool(ObjectPool):
62

    
63
    _scheme_to_class = {
64
        'http': http_class,
65
        'https': https_class,
66
    }
67

    
68
    def __init__(self, scheme, netloc, size=None):
69
        log.debug("INIT-POOL: Initializing pool of size %d, scheme: %s, "
70
                  "netloc: %s", size, scheme, netloc)
71
        ObjectPool.__init__(self, size=size)
72

    
73
        connection_class = self._scheme_to_class.get(scheme, None)
74
        if connection_class is None:
75
            m = 'Unsupported scheme: %s' % (scheme,)
76
            raise ValueError(m)
77

    
78
        self.connection_class = connection_class
79
        self.scheme = scheme
80
        self.netloc = netloc
81

    
82
    def _pool_create(self):
83
        log.debug("CREATE-HTTP-BEFORE from pool %r", self)
84
        conn = self.connection_class(self.netloc)
85
        conn._pool_use_counter = USAGE_LIMIT
86
        return conn
87

    
88
    def _pool_verify(self, conn):
89
        log.debug("VERIFY-HTTP")
90
        if conn is None:
91
            return False
92
        sock = conn.sock
93
        if sock is None:
94
            return True
95
        if select((conn.sock,), (), (), 0)[0]:
96
            return False
97
        return True
98

    
99
    def _pool_cleanup(self, conn):
100
        log.debug("CLEANUP-HTTP")
101
        # every connection can be used a finite number of times
102
        conn._pool_use_counter -= 1
103

    
104
        # see httplib source for connection states documentation
105
        if (conn._pool_use_counter > 0 and
106
            conn._HTTPConnection__state == 'Idle'):
107
            try:
108
                conn.getresponse()
109
            except ResponseNotReady:
110
                log.debug("CLEANUP-HTTP: Not closing connection. Will reuse.")
111
                return False
112

    
113
        log.debug("CLEANUP-HTTP: Closing connection. Will not reuse.")
114
        conn.close()
115
        return True
116

    
117

    
118
class PooledHTTPConnection(PooledObject):
119

    
120
    _pool_log_prefix = "HTTP"
121
    _pool_class = HTTPConnectionPool
122

    
123
    def __init__(self, netloc, scheme='http', pool=None, **kw):
124
        kw['netloc'] = netloc
125
        kw['scheme'] = scheme
126
        kw['pool'] = pool
127
        super(PooledHTTPConnection, self).__init__(**kw)
128

    
129
    def get_pool(self):
130
        kwargs = self._pool_kwargs
131
        pool = kwargs.pop('pool', None)
132
        if pool is not None:
133
            return pool
134

    
135
        # pool was not given, find one from the global registry
136
        scheme = kwargs['scheme']
137
        netloc = kwargs['netloc']
138
        size = kwargs.get('size', default_pool_size)
139
        # ensure distinct pools for every (scheme, netloc) combination
140
        key = (scheme, netloc)
141
        with _pools_mutex:
142
            if key not in _pools:
143
                log.debug("HTTP-GET: Creating pool for key %s", key)
144
                pool = HTTPConnectionPool(scheme, netloc, size=size)
145
                _pools[key] = pool
146
            else:
147
                pool = _pools[key]
148

    
149
        return pool