root / lib / http / client.py @ 4ba4fe14
History | View | Annotate | Download (12.3 kB)
1 | 02cab3e7 | Michael Hanselmann | #
|
---|---|---|---|
2 | 02cab3e7 | Michael Hanselmann | #
|
3 | 02cab3e7 | Michael Hanselmann | |
4 | 33231500 | Michael Hanselmann | # Copyright (C) 2007, 2008, 2010 Google Inc.
|
5 | 02cab3e7 | Michael Hanselmann | #
|
6 | 02cab3e7 | Michael Hanselmann | # This program is free software; you can redistribute it and/or modify
|
7 | 02cab3e7 | Michael Hanselmann | # it under the terms of the GNU General Public License as published by
|
8 | 02cab3e7 | Michael Hanselmann | # the Free Software Foundation; either version 2 of the License, or
|
9 | 02cab3e7 | Michael Hanselmann | # (at your option) any later version.
|
10 | 02cab3e7 | Michael Hanselmann | #
|
11 | 02cab3e7 | Michael Hanselmann | # This program is distributed in the hope that it will be useful, but
|
12 | 02cab3e7 | Michael Hanselmann | # WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 | 02cab3e7 | Michael Hanselmann | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14 | 02cab3e7 | Michael Hanselmann | # General Public License for more details.
|
15 | 02cab3e7 | Michael Hanselmann | #
|
16 | 02cab3e7 | Michael Hanselmann | # You should have received a copy of the GNU General Public License
|
17 | 02cab3e7 | Michael Hanselmann | # along with this program; if not, write to the Free Software
|
18 | 02cab3e7 | Michael Hanselmann | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
19 | 02cab3e7 | Michael Hanselmann | # 02110-1301, USA.
|
20 | 02cab3e7 | Michael Hanselmann | |
21 | 02cab3e7 | Michael Hanselmann | """HTTP client module.
|
22 | 02cab3e7 | Michael Hanselmann |
|
23 | 02cab3e7 | Michael Hanselmann | """
|
24 | 02cab3e7 | Michael Hanselmann | |
25 | 33231500 | Michael Hanselmann | import logging |
26 | 33231500 | Michael Hanselmann | import pycurl |
27 | 33231500 | Michael Hanselmann | from cStringIO import StringIO |
28 | 6c881c52 | Iustin Pop | |
29 | 02cab3e7 | Michael Hanselmann | from ganeti import http |
30 | 33231500 | Michael Hanselmann | from ganeti import compat |
31 | 02cab3e7 | Michael Hanselmann | |
32 | 02cab3e7 | Michael Hanselmann | |
33 | 02cab3e7 | Michael Hanselmann | class HttpClientRequest(object): |
34 | 02cab3e7 | Michael Hanselmann | def __init__(self, host, port, method, path, headers=None, post_data=None, |
35 | 33231500 | Michael Hanselmann | read_timeout=None, curl_config_fn=None): |
36 | 02cab3e7 | Michael Hanselmann | """Describes an HTTP request.
|
37 | 02cab3e7 | Michael Hanselmann |
|
38 | 02cab3e7 | Michael Hanselmann | @type host: string
|
39 | 02cab3e7 | Michael Hanselmann | @param host: Hostname
|
40 | 02cab3e7 | Michael Hanselmann | @type port: int
|
41 | 02cab3e7 | Michael Hanselmann | @param port: Port
|
42 | 02cab3e7 | Michael Hanselmann | @type method: string
|
43 | 02cab3e7 | Michael Hanselmann | @param method: Method name
|
44 | 02cab3e7 | Michael Hanselmann | @type path: string
|
45 | 02cab3e7 | Michael Hanselmann | @param path: Request path
|
46 | 33231500 | Michael Hanselmann | @type headers: list or None
|
47 | 33231500 | Michael Hanselmann | @param headers: Additional headers to send, list of strings
|
48 | 02cab3e7 | Michael Hanselmann | @type post_data: string or None
|
49 | 02cab3e7 | Michael Hanselmann | @param post_data: Additional data to send
|
50 | e0036155 | Iustin Pop | @type read_timeout: int
|
51 | e0036155 | Iustin Pop | @param read_timeout: if passed, it will be used as the read
|
52 | e0036155 | Iustin Pop | timeout while reading the response from the server
|
53 | 33231500 | Michael Hanselmann | @type curl_config_fn: callable
|
54 | 33231500 | Michael Hanselmann | @param curl_config_fn: Function to configure cURL object before request
|
55 | 33231500 | Michael Hanselmann | (Note: if the function configures the connection in
|
56 | 33231500 | Michael Hanselmann | a way where it wouldn't be efficient to reuse them,
|
57 | 33231500 | Michael Hanselmann | a "identity" property should be defined, see
|
58 | 33231500 | Michael Hanselmann | L{HttpClientRequest.identity})
|
59 | 02cab3e7 | Michael Hanselmann |
|
60 | 02cab3e7 | Michael Hanselmann | """
|
61 | 02cab3e7 | Michael Hanselmann | assert path.startswith("/"), "Path must start with slash (/)" |
62 | 33231500 | Michael Hanselmann | assert curl_config_fn is None or callable(curl_config_fn) |
63 | 02cab3e7 | Michael Hanselmann | |
64 | 02cab3e7 | Michael Hanselmann | # Request attributes
|
65 | 02cab3e7 | Michael Hanselmann | self.host = host
|
66 | 02cab3e7 | Michael Hanselmann | self.port = port
|
67 | 02cab3e7 | Michael Hanselmann | self.method = method
|
68 | 02cab3e7 | Michael Hanselmann | self.path = path
|
69 | e0036155 | Iustin Pop | self.read_timeout = read_timeout
|
70 | 33231500 | Michael Hanselmann | self.curl_config_fn = curl_config_fn
|
71 | 02cab3e7 | Michael Hanselmann | |
72 | 33231500 | Michael Hanselmann | if post_data is None: |
73 | 33231500 | Michael Hanselmann | self.post_data = "" |
74 | 33231500 | Michael Hanselmann | else:
|
75 | 33231500 | Michael Hanselmann | self.post_data = post_data
|
76 | 33231500 | Michael Hanselmann | |
77 | 33231500 | Michael Hanselmann | if headers is None: |
78 | 33231500 | Michael Hanselmann | self.headers = []
|
79 | 33231500 | Michael Hanselmann | elif isinstance(headers, dict): |
80 | 33231500 | Michael Hanselmann | # Support for old interface
|
81 | 33231500 | Michael Hanselmann | self.headers = ["%s: %s" % (name, value) |
82 | 33231500 | Michael Hanselmann | for name, value in headers.items()] |
83 | 33231500 | Michael Hanselmann | else:
|
84 | 33231500 | Michael Hanselmann | self.headers = headers
|
85 | 33231500 | Michael Hanselmann | |
86 | 33231500 | Michael Hanselmann | # Response status
|
87 | 02cab3e7 | Michael Hanselmann | self.success = None |
88 | 02cab3e7 | Michael Hanselmann | self.error = None |
89 | 02cab3e7 | Michael Hanselmann | |
90 | 02cab3e7 | Michael Hanselmann | # Response attributes
|
91 | 02cab3e7 | Michael Hanselmann | self.resp_status_code = None |
92 | 02cab3e7 | Michael Hanselmann | self.resp_body = None |
93 | 02cab3e7 | Michael Hanselmann | |
94 | 9fa2e150 | Michael Hanselmann | def __repr__(self): |
95 | 9fa2e150 | Michael Hanselmann | status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__), |
96 | 9fa2e150 | Michael Hanselmann | "%s:%s" % (self.host, self.port), |
97 | 9fa2e150 | Michael Hanselmann | self.method,
|
98 | 9fa2e150 | Michael Hanselmann | self.path]
|
99 | 9fa2e150 | Michael Hanselmann | |
100 | 9fa2e150 | Michael Hanselmann | return "<%s at %#x>" % (" ".join(status), id(self)) |
101 | 9fa2e150 | Michael Hanselmann | |
102 | 33231500 | Michael Hanselmann | @property
|
103 | 33231500 | Michael Hanselmann | def url(self): |
104 | 33231500 | Michael Hanselmann | """Returns the full URL for this requests.
|
105 | 02cab3e7 | Michael Hanselmann |
|
106 | 33231500 | Michael Hanselmann | """
|
107 | 33231500 | Michael Hanselmann | # TODO: Support for non-SSL requests
|
108 | 33231500 | Michael Hanselmann | return "https://%s:%s%s" % (self.host, self.port, self.path) |
109 | 02cab3e7 | Michael Hanselmann | |
110 | 33231500 | Michael Hanselmann | @property
|
111 | 33231500 | Michael Hanselmann | def identity(self): |
112 | 33231500 | Michael Hanselmann | """Returns identifier for retrieving a pooled connection for this request.
|
113 | 02cab3e7 | Michael Hanselmann |
|
114 | 33231500 | Michael Hanselmann | This allows cURL client objects to be re-used and to cache information
|
115 | 33231500 | Michael Hanselmann | (e.g. SSL session IDs or connections).
|
116 | 02cab3e7 | Michael Hanselmann |
|
117 | 02cab3e7 | Michael Hanselmann | """
|
118 | 33231500 | Michael Hanselmann | parts = [self.host, self.port] |
119 | 02cab3e7 | Michael Hanselmann | |
120 | 33231500 | Michael Hanselmann | if self.curl_config_fn: |
121 | 02cab3e7 | Michael Hanselmann | try:
|
122 | 33231500 | Michael Hanselmann | parts.append(self.curl_config_fn.identity)
|
123 | 33231500 | Michael Hanselmann | except AttributeError: |
124 | 33231500 | Michael Hanselmann | pass
|
125 | 02cab3e7 | Michael Hanselmann | |
126 | 33231500 | Michael Hanselmann | return "/".join(str(i) for i in parts) |
127 | 02cab3e7 | Michael Hanselmann | |
128 | 02cab3e7 | Michael Hanselmann | |
129 | 33231500 | Michael Hanselmann | class _HttpClient(object): |
130 | 33231500 | Michael Hanselmann | def __init__(self, curl_config_fn): |
131 | 33231500 | Michael Hanselmann | """Initializes this class.
|
132 | 02cab3e7 | Michael Hanselmann |
|
133 | 33231500 | Michael Hanselmann | @type curl_config_fn: callable
|
134 | 33231500 | Michael Hanselmann | @param curl_config_fn: Function to configure cURL object after
|
135 | 33231500 | Michael Hanselmann | initialization
|
136 | 02cab3e7 | Michael Hanselmann |
|
137 | 33231500 | Michael Hanselmann | """
|
138 | 33231500 | Michael Hanselmann | self._req = None |
139 | 02cab3e7 | Michael Hanselmann | |
140 | 33231500 | Michael Hanselmann | curl = self._CreateCurlHandle()
|
141 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.VERBOSE, False)
|
142 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.NOSIGNAL, True)
|
143 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.USERAGENT, http.HTTP_GANETI_VERSION) |
144 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.PROXY, "")
|
145 | 02cab3e7 | Michael Hanselmann | |
146 | 4ba4fe14 | Apollon Oikonomopoulos | # Disable SSL session ID caching (pycurl >= 7.16.0)
|
147 | 4ba4fe14 | Apollon Oikonomopoulos | if hasattr(pycurl, "SSL_SESSIONID_CACHE"): |
148 | 4ba4fe14 | Apollon Oikonomopoulos | curl.setopt(pycurl.SSL_SESSIONID_CACHE, False)
|
149 | 4ba4fe14 | Apollon Oikonomopoulos | |
150 | 33231500 | Michael Hanselmann | # Pass cURL object to external config function
|
151 | 33231500 | Michael Hanselmann | if curl_config_fn:
|
152 | 33231500 | Michael Hanselmann | curl_config_fn(curl) |
153 | 02cab3e7 | Michael Hanselmann | |
154 | 33231500 | Michael Hanselmann | self._curl = curl
|
155 | 02cab3e7 | Michael Hanselmann | |
156 | 33231500 | Michael Hanselmann | @staticmethod
|
157 | 33231500 | Michael Hanselmann | def _CreateCurlHandle(): |
158 | 33231500 | Michael Hanselmann | """Returns a new cURL object.
|
159 | 02cab3e7 | Michael Hanselmann |
|
160 | 02cab3e7 | Michael Hanselmann | """
|
161 | 33231500 | Michael Hanselmann | return pycurl.Curl()
|
162 | 02cab3e7 | Michael Hanselmann | |
163 | 33231500 | Michael Hanselmann | def GetCurlHandle(self): |
164 | 33231500 | Michael Hanselmann | """Returns the cURL object.
|
165 | 02cab3e7 | Michael Hanselmann |
|
166 | 33231500 | Michael Hanselmann | """
|
167 | 33231500 | Michael Hanselmann | return self._curl |
168 | 02cab3e7 | Michael Hanselmann | |
169 | 33231500 | Michael Hanselmann | def GetCurrentRequest(self): |
170 | 33231500 | Michael Hanselmann | """Returns the current request.
|
171 | 02cab3e7 | Michael Hanselmann |
|
172 | 33231500 | Michael Hanselmann | @rtype: L{HttpClientRequest} or None
|
173 | 02cab3e7 | Michael Hanselmann |
|
174 | 33231500 | Michael Hanselmann | """
|
175 | 33231500 | Michael Hanselmann | return self._req |
176 | 02cab3e7 | Michael Hanselmann | |
177 | 33231500 | Michael Hanselmann | def StartRequest(self, req): |
178 | 33231500 | Michael Hanselmann | """Starts a request on this client.
|
179 | 02cab3e7 | Michael Hanselmann |
|
180 | 33231500 | Michael Hanselmann | @type req: L{HttpClientRequest}
|
181 | 33231500 | Michael Hanselmann | @param req: HTTP request
|
182 | 02cab3e7 | Michael Hanselmann |
|
183 | 33231500 | Michael Hanselmann | """
|
184 | 33231500 | Michael Hanselmann | assert not self._req, "Another request is already started" |
185 | 33231500 | Michael Hanselmann | |
186 | 33231500 | Michael Hanselmann | self._req = req
|
187 | 33231500 | Michael Hanselmann | self._resp_buffer = StringIO()
|
188 | 33231500 | Michael Hanselmann | |
189 | 33231500 | Michael Hanselmann | url = req.url |
190 | 33231500 | Michael Hanselmann | method = req.method |
191 | 33231500 | Michael Hanselmann | post_data = req.post_data |
192 | 33231500 | Michael Hanselmann | headers = req.headers |
193 | 33231500 | Michael Hanselmann | |
194 | 33231500 | Michael Hanselmann | # PycURL requires strings to be non-unicode
|
195 | 33231500 | Michael Hanselmann | assert isinstance(method, str) |
196 | 33231500 | Michael Hanselmann | assert isinstance(url, str) |
197 | 33231500 | Michael Hanselmann | assert isinstance(post_data, str) |
198 | 33231500 | Michael Hanselmann | assert compat.all(isinstance(i, str) for i in headers) |
199 | 33231500 | Michael Hanselmann | |
200 | 33231500 | Michael Hanselmann | # Configure cURL object for request
|
201 | 33231500 | Michael Hanselmann | curl = self._curl
|
202 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.CUSTOMREQUEST, str(method))
|
203 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.URL, url) |
204 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.POSTFIELDS, post_data) |
205 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.WRITEFUNCTION, self._resp_buffer.write)
|
206 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.HTTPHEADER, headers) |
207 | 33231500 | Michael Hanselmann | |
208 | 33231500 | Michael Hanselmann | if req.read_timeout is None: |
209 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.TIMEOUT, 0)
|
210 | 33231500 | Michael Hanselmann | else:
|
211 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.TIMEOUT, int(req.read_timeout))
|
212 | 02cab3e7 | Michael Hanselmann | |
213 | 33231500 | Michael Hanselmann | # Pass cURL object to external config function
|
214 | 33231500 | Michael Hanselmann | if req.curl_config_fn:
|
215 | 33231500 | Michael Hanselmann | req.curl_config_fn(curl) |
216 | 02cab3e7 | Michael Hanselmann | |
217 | 33231500 | Michael Hanselmann | def Done(self, errmsg): |
218 | 33231500 | Michael Hanselmann | """Finishes a request.
|
219 | 02cab3e7 | Michael Hanselmann |
|
220 | 33231500 | Michael Hanselmann | @type errmsg: string or None
|
221 | 33231500 | Michael Hanselmann | @param errmsg: Error message if request failed
|
222 | 02cab3e7 | Michael Hanselmann |
|
223 | 02cab3e7 | Michael Hanselmann | """
|
224 | 33231500 | Michael Hanselmann | req = self._req
|
225 | 33231500 | Michael Hanselmann | assert req, "No request" |
226 | 02cab3e7 | Michael Hanselmann | |
227 | 33231500 | Michael Hanselmann | logging.debug("Request %s finished, errmsg=%s", req, errmsg)
|
228 | 02cab3e7 | Michael Hanselmann | |
229 | 33231500 | Michael Hanselmann | curl = self._curl
|
230 | 02cab3e7 | Michael Hanselmann | |
231 | 33231500 | Michael Hanselmann | req.success = not bool(errmsg) |
232 | 33231500 | Michael Hanselmann | req.error = errmsg |
233 | 02cab3e7 | Michael Hanselmann | |
234 | 33231500 | Michael Hanselmann | # Get HTTP response code
|
235 | 33231500 | Michael Hanselmann | req.resp_status_code = curl.getinfo(pycurl.RESPONSE_CODE) |
236 | 33231500 | Michael Hanselmann | req.resp_body = self._resp_buffer.getvalue()
|
237 | 02cab3e7 | Michael Hanselmann | |
238 | 33231500 | Michael Hanselmann | # Reset client object
|
239 | 33231500 | Michael Hanselmann | self._req = None |
240 | 33231500 | Michael Hanselmann | self._resp_buffer = None |
241 | 02cab3e7 | Michael Hanselmann | |
242 | 33231500 | Michael Hanselmann | # Ensure no potentially large variables are referenced
|
243 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.POSTFIELDS, "")
|
244 | 33231500 | Michael Hanselmann | curl.setopt(pycurl.WRITEFUNCTION, lambda _: None) |
245 | 02cab3e7 | Michael Hanselmann | |
246 | 02cab3e7 | Michael Hanselmann | |
247 | 33231500 | Michael Hanselmann | class _PooledHttpClient: |
248 | 33231500 | Michael Hanselmann | """Data structure for HTTP client pool.
|
249 | 02cab3e7 | Michael Hanselmann |
|
250 | 33231500 | Michael Hanselmann | """
|
251 | 33231500 | Michael Hanselmann | def __init__(self, identity, client): |
252 | 33231500 | Michael Hanselmann | """Initializes this class.
|
253 | f2e13d55 | Michael Hanselmann |
|
254 | 33231500 | Michael Hanselmann | @type identity: string
|
255 | 33231500 | Michael Hanselmann | @param identity: Client identifier for pool
|
256 | 33231500 | Michael Hanselmann | @type client: L{_HttpClient}
|
257 | 33231500 | Michael Hanselmann | @param client: HTTP client
|
258 | 02cab3e7 | Michael Hanselmann |
|
259 | 02cab3e7 | Michael Hanselmann | """
|
260 | 33231500 | Michael Hanselmann | self.identity = identity
|
261 | 33231500 | Michael Hanselmann | self.client = client
|
262 | 33231500 | Michael Hanselmann | self.lastused = 0 |
263 | 02cab3e7 | Michael Hanselmann | |
264 | 33231500 | Michael Hanselmann | def __repr__(self): |
265 | 33231500 | Michael Hanselmann | status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__), |
266 | 33231500 | Michael Hanselmann | "id=%s" % self.identity, |
267 | 33231500 | Michael Hanselmann | "lastuse=%s" % self.lastused, |
268 | 33231500 | Michael Hanselmann | repr(self.client)] |
269 | 02cab3e7 | Michael Hanselmann | |
270 | 33231500 | Michael Hanselmann | return "<%s at %#x>" % (" ".join(status), id(self)) |
271 | 02cab3e7 | Michael Hanselmann | |
272 | 02cab3e7 | Michael Hanselmann | |
273 | 33231500 | Michael Hanselmann | class HttpClientPool: |
274 | 33231500 | Michael Hanselmann | """A simple HTTP client pool.
|
275 | 02cab3e7 | Michael Hanselmann |
|
276 | 33231500 | Michael Hanselmann | Supports one pooled connection per identity (see
|
277 | 33231500 | Michael Hanselmann | L{HttpClientRequest.identity}).
|
278 | 02cab3e7 | Michael Hanselmann |
|
279 | 33231500 | Michael Hanselmann | """
|
280 | 33231500 | Michael Hanselmann | #: After how many generations to drop unused clients
|
281 | 33231500 | Michael Hanselmann | _MAX_GENERATIONS_DROP = 25
|
282 | 33231500 | Michael Hanselmann | |
283 | 33231500 | Michael Hanselmann | def __init__(self, curl_config_fn): |
284 | 33231500 | Michael Hanselmann | """Initializes this class.
|
285 | 33231500 | Michael Hanselmann |
|
286 | 33231500 | Michael Hanselmann | @type curl_config_fn: callable
|
287 | 33231500 | Michael Hanselmann | @param curl_config_fn: Function to configure cURL object after
|
288 | 33231500 | Michael Hanselmann | initialization
|
289 | 02cab3e7 | Michael Hanselmann |
|
290 | 02cab3e7 | Michael Hanselmann | """
|
291 | 33231500 | Michael Hanselmann | self._curl_config_fn = curl_config_fn
|
292 | 33231500 | Michael Hanselmann | self._generation = 0 |
293 | 33231500 | Michael Hanselmann | self._pool = {}
|
294 | 02cab3e7 | Michael Hanselmann | |
295 | 33231500 | Michael Hanselmann | @staticmethod
|
296 | 33231500 | Michael Hanselmann | def _GetHttpClientCreator(): |
297 | 33231500 | Michael Hanselmann | """Returns callable to create HTTP client.
|
298 | e0036155 | Iustin Pop |
|
299 | 33231500 | Michael Hanselmann | """
|
300 | 33231500 | Michael Hanselmann | return _HttpClient
|
301 | 02cab3e7 | Michael Hanselmann | |
302 | 33231500 | Michael Hanselmann | def _Get(self, identity): |
303 | 33231500 | Michael Hanselmann | """Gets an HTTP client from the pool.
|
304 | 02cab3e7 | Michael Hanselmann |
|
305 | 33231500 | Michael Hanselmann | @type identity: string
|
306 | 33231500 | Michael Hanselmann | @param identity: Client identifier
|
307 | 02cab3e7 | Michael Hanselmann |
|
308 | 33231500 | Michael Hanselmann | """
|
309 | 33231500 | Michael Hanselmann | try:
|
310 | 33231500 | Michael Hanselmann | pclient = self._pool.pop(identity)
|
311 | 33231500 | Michael Hanselmann | except KeyError: |
312 | 33231500 | Michael Hanselmann | # Need to create new client
|
313 | 33231500 | Michael Hanselmann | client = self._GetHttpClientCreator()(self._curl_config_fn) |
314 | 33231500 | Michael Hanselmann | pclient = _PooledHttpClient(identity, client) |
315 | 33231500 | Michael Hanselmann | logging.debug("Created new client %s", pclient)
|
316 | 33231500 | Michael Hanselmann | else:
|
317 | 33231500 | Michael Hanselmann | logging.debug("Reusing client %s", pclient)
|
318 | 02cab3e7 | Michael Hanselmann | |
319 | 33231500 | Michael Hanselmann | assert pclient.identity == identity
|
320 | 02cab3e7 | Michael Hanselmann | |
321 | 33231500 | Michael Hanselmann | return pclient
|
322 | 02cab3e7 | Michael Hanselmann | |
323 | 33231500 | Michael Hanselmann | def _StartRequest(self, req): |
324 | 33231500 | Michael Hanselmann | """Starts a request.
|
325 | 9fa2e150 | Michael Hanselmann |
|
326 | 33231500 | Michael Hanselmann | @type req: L{HttpClientRequest}
|
327 | 33231500 | Michael Hanselmann | @param req: HTTP request
|
328 | 9fa2e150 | Michael Hanselmann |
|
329 | 33231500 | Michael Hanselmann | """
|
330 | 33231500 | Michael Hanselmann | logging.debug("Starting request %r", req)
|
331 | 33231500 | Michael Hanselmann | pclient = self._Get(req.identity)
|
332 | 02cab3e7 | Michael Hanselmann | |
333 | 33231500 | Michael Hanselmann | assert req.identity not in self._pool |
334 | 02cab3e7 | Michael Hanselmann | |
335 | 33231500 | Michael Hanselmann | pclient.client.StartRequest(req) |
336 | 33231500 | Michael Hanselmann | pclient.lastused = self._generation
|
337 | 33231500 | Michael Hanselmann | |
338 | 33231500 | Michael Hanselmann | return pclient
|
339 | 02cab3e7 | Michael Hanselmann | |
340 | 33231500 | Michael Hanselmann | def _Return(self, pclients): |
341 | 33231500 | Michael Hanselmann | """Returns HTTP clients to the pool.
|
342 | 02cab3e7 | Michael Hanselmann |
|
343 | 33231500 | Michael Hanselmann | """
|
344 | 33231500 | Michael Hanselmann | for pc in pclients: |
345 | 33231500 | Michael Hanselmann | logging.debug("Returning client %s to pool", pc)
|
346 | 33231500 | Michael Hanselmann | assert pc.identity not in self._pool |
347 | 33231500 | Michael Hanselmann | assert pc not in self._pool.values() |
348 | 33231500 | Michael Hanselmann | self._pool[pc.identity] = pc
|
349 | 33231500 | Michael Hanselmann | |
350 | 33231500 | Michael Hanselmann | # Check for unused clients
|
351 | 33231500 | Michael Hanselmann | for pc in self._pool.values(): |
352 | 33231500 | Michael Hanselmann | if (pc.lastused + self._MAX_GENERATIONS_DROP) < self._generation: |
353 | 33231500 | Michael Hanselmann | logging.debug("Removing client %s which hasn't been used"
|
354 | 33231500 | Michael Hanselmann | " for %s generations",
|
355 | 33231500 | Michael Hanselmann | pc, self._MAX_GENERATIONS_DROP)
|
356 | 33231500 | Michael Hanselmann | self._pool.pop(pc.identity, None) |
357 | 33231500 | Michael Hanselmann | |
358 | 33231500 | Michael Hanselmann | assert compat.all(pc.lastused >= (self._generation - |
359 | 33231500 | Michael Hanselmann | self._MAX_GENERATIONS_DROP)
|
360 | 33231500 | Michael Hanselmann | for pc in self._pool.values()) |
361 | 33231500 | Michael Hanselmann | |
362 | 33231500 | Michael Hanselmann | @staticmethod
|
363 | 33231500 | Michael Hanselmann | def _CreateCurlMultiHandle(): |
364 | 33231500 | Michael Hanselmann | """Creates new cURL multi handle.
|
365 | 02cab3e7 | Michael Hanselmann |
|
366 | 33231500 | Michael Hanselmann | """
|
367 | 33231500 | Michael Hanselmann | return pycurl.CurlMulti()
|
368 | 02cab3e7 | Michael Hanselmann | |
369 | 33231500 | Michael Hanselmann | def ProcessRequests(self, requests): |
370 | 33231500 | Michael Hanselmann | """Processes any number of HTTP client requests using pooled objects.
|
371 | 02cab3e7 | Michael Hanselmann |
|
372 | 33231500 | Michael Hanselmann | @type requests: list of L{HttpClientRequest}
|
373 | 33231500 | Michael Hanselmann | @param requests: List of all requests
|
374 | 02cab3e7 | Michael Hanselmann |
|
375 | 33231500 | Michael Hanselmann | """
|
376 | 33231500 | Michael Hanselmann | multi = self._CreateCurlMultiHandle()
|
377 | 02cab3e7 | Michael Hanselmann | |
378 | 33231500 | Michael Hanselmann | # For client cleanup
|
379 | 33231500 | Michael Hanselmann | self._generation += 1 |
380 | 02cab3e7 | Michael Hanselmann | |
381 | 33231500 | Michael Hanselmann | assert compat.all((req.error is None and |
382 | 33231500 | Michael Hanselmann | req.success is None and |
383 | 33231500 | Michael Hanselmann | req.resp_status_code is None and |
384 | 33231500 | Michael Hanselmann | req.resp_body is None) |
385 | 33231500 | Michael Hanselmann | for req in requests) |
386 | 02cab3e7 | Michael Hanselmann | |
387 | 33231500 | Michael Hanselmann | curl_to_pclient = {} |
388 | 33231500 | Michael Hanselmann | for req in requests: |
389 | 33231500 | Michael Hanselmann | pclient = self._StartRequest(req)
|
390 | 33231500 | Michael Hanselmann | curl = pclient.client.GetCurlHandle() |
391 | 33231500 | Michael Hanselmann | curl_to_pclient[curl] = pclient |
392 | 33231500 | Michael Hanselmann | multi.add_handle(curl) |
393 | 33231500 | Michael Hanselmann | assert pclient.client.GetCurrentRequest() == req
|
394 | 33231500 | Michael Hanselmann | assert pclient.lastused >= 0 |
395 | 02cab3e7 | Michael Hanselmann | |
396 | 33231500 | Michael Hanselmann | assert len(curl_to_pclient) == len(requests) |
397 | 02cab3e7 | Michael Hanselmann | |
398 | 33231500 | Michael Hanselmann | done_count = 0
|
399 | 33231500 | Michael Hanselmann | while True: |
400 | 33231500 | Michael Hanselmann | (ret, _) = multi.perform() |
401 | 33231500 | Michael Hanselmann | assert ret in (pycurl.E_MULTI_OK, pycurl.E_CALL_MULTI_PERFORM) |
402 | 33231500 | Michael Hanselmann | |
403 | 33231500 | Michael Hanselmann | if ret == pycurl.E_CALL_MULTI_PERFORM:
|
404 | 33231500 | Michael Hanselmann | # cURL wants to be called again
|
405 | 33231500 | Michael Hanselmann | continue
|
406 | 33231500 | Michael Hanselmann | |
407 | 33231500 | Michael Hanselmann | while True: |
408 | 33231500 | Michael Hanselmann | (remaining_messages, successful, failed) = multi.info_read() |
409 | 33231500 | Michael Hanselmann | |
410 | 33231500 | Michael Hanselmann | for curl in successful: |
411 | 33231500 | Michael Hanselmann | multi.remove_handle(curl) |
412 | 33231500 | Michael Hanselmann | done_count += 1
|
413 | 33231500 | Michael Hanselmann | pclient = curl_to_pclient[curl] |
414 | 33231500 | Michael Hanselmann | req = pclient.client.GetCurrentRequest() |
415 | 33231500 | Michael Hanselmann | pclient.client.Done(None)
|
416 | 33231500 | Michael Hanselmann | assert req.success
|
417 | 33231500 | Michael Hanselmann | assert not pclient.client.GetCurrentRequest() |
418 | 33231500 | Michael Hanselmann | |
419 | 33231500 | Michael Hanselmann | for curl, errnum, errmsg in failed: |
420 | 33231500 | Michael Hanselmann | multi.remove_handle(curl) |
421 | 33231500 | Michael Hanselmann | done_count += 1
|
422 | 33231500 | Michael Hanselmann | pclient = curl_to_pclient[curl] |
423 | 33231500 | Michael Hanselmann | req = pclient.client.GetCurrentRequest() |
424 | 33231500 | Michael Hanselmann | pclient.client.Done("Error %s: %s" % (errnum, errmsg))
|
425 | 33231500 | Michael Hanselmann | assert req.error
|
426 | 33231500 | Michael Hanselmann | assert not pclient.client.GetCurrentRequest() |
427 | 33231500 | Michael Hanselmann | |
428 | 33231500 | Michael Hanselmann | if remaining_messages == 0: |
429 | 33231500 | Michael Hanselmann | break
|
430 | 33231500 | Michael Hanselmann | |
431 | 33231500 | Michael Hanselmann | assert done_count <= len(requests) |
432 | 33231500 | Michael Hanselmann | |
433 | 33231500 | Michael Hanselmann | if done_count == len(requests): |
434 | 33231500 | Michael Hanselmann | break
|
435 | 02cab3e7 | Michael Hanselmann | |
436 | 33231500 | Michael Hanselmann | # Wait for I/O. The I/O timeout shouldn't be too long so that HTTP
|
437 | 33231500 | Michael Hanselmann | # timeouts, which are only evaluated in multi.perform, aren't
|
438 | 33231500 | Michael Hanselmann | # unnecessarily delayed.
|
439 | 33231500 | Michael Hanselmann | multi.select(1.0)
|
440 | 02cab3e7 | Michael Hanselmann | |
441 | 33231500 | Michael Hanselmann | assert compat.all(pclient.client.GetCurrentRequest() is None |
442 | 33231500 | Michael Hanselmann | for pclient in curl_to_pclient.values()) |
443 | 02cab3e7 | Michael Hanselmann | |
444 | 33231500 | Michael Hanselmann | # Return clients to pool
|
445 | 33231500 | Michael Hanselmann | self._Return(curl_to_pclient.values())
|
446 | 02cab3e7 | Michael Hanselmann | |
447 | 33231500 | Michael Hanselmann | assert done_count == len(requests) |
448 | 33231500 | Michael Hanselmann | assert compat.all(req.error is not None or |
449 | 33231500 | Michael Hanselmann | (req.success and
|
450 | 33231500 | Michael Hanselmann | req.resp_status_code is not None and |
451 | 33231500 | Michael Hanselmann | req.resp_body is not None) |
452 | 33231500 | Michael Hanselmann | for req in requests) |