root / lib / http / client.py @ a57e502a
History | View | Annotate | Download (10.8 kB)
1 | 02cab3e7 | Michael Hanselmann | #
|
---|---|---|---|
2 | 02cab3e7 | Michael Hanselmann | #
|
3 | 02cab3e7 | Michael Hanselmann | |
4 | 33231500 | Michael Hanselmann | # Copyright (C) 2007, 2008, 2010 Google Inc.
|
5 | 02cab3e7 | Michael Hanselmann | #
|
6 | 02cab3e7 | Michael Hanselmann | # This program is free software; you can redistribute it and/or modify
|
7 | 02cab3e7 | Michael Hanselmann | # it under the terms of the GNU General Public License as published by
|
8 | 02cab3e7 | Michael Hanselmann | # the Free Software Foundation; either version 2 of the License, or
|
9 | 02cab3e7 | Michael Hanselmann | # (at your option) any later version.
|
10 | 02cab3e7 | Michael Hanselmann | #
|
11 | 02cab3e7 | Michael Hanselmann | # This program is distributed in the hope that it will be useful, but
|
12 | 02cab3e7 | Michael Hanselmann | # WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 | 02cab3e7 | Michael Hanselmann | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14 | 02cab3e7 | Michael Hanselmann | # General Public License for more details.
|
15 | 02cab3e7 | Michael Hanselmann | #
|
16 | 02cab3e7 | Michael Hanselmann | # You should have received a copy of the GNU General Public License
|
17 | 02cab3e7 | Michael Hanselmann | # along with this program; if not, write to the Free Software
|
18 | 02cab3e7 | Michael Hanselmann | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
19 | 02cab3e7 | Michael Hanselmann | # 02110-1301, USA.
|
20 | 02cab3e7 | Michael Hanselmann | |
21 | 02cab3e7 | Michael Hanselmann | """HTTP client module.
|
22 | 02cab3e7 | Michael Hanselmann |
|
23 | 02cab3e7 | Michael Hanselmann | """
|
24 | 02cab3e7 | Michael Hanselmann | |
25 | 33231500 | Michael Hanselmann | import logging |
26 | 33231500 | Michael Hanselmann | import pycurl |
27 | aea5caef | Michael Hanselmann | import threading |
28 | 33231500 | Michael Hanselmann | from cStringIO import StringIO |
29 | 6c881c52 | Iustin Pop | |
30 | 02cab3e7 | Michael Hanselmann | from ganeti import http |
31 | 33231500 | Michael Hanselmann | from ganeti import compat |
32 | 1a8337f2 | Manuel Franceschini | from ganeti import netutils |
33 | aea5caef | Michael Hanselmann | from ganeti import locking |
34 | 02cab3e7 | Michael Hanselmann | |
35 | 02cab3e7 | Michael Hanselmann | |
36 | 02cab3e7 | Michael Hanselmann | class HttpClientRequest(object): |
37 | 02cab3e7 | Michael Hanselmann | def __init__(self, host, port, method, path, headers=None, post_data=None, |
38 | bac6ea51 | Michael Hanselmann | read_timeout=None, curl_config_fn=None, nicename=None, |
39 | bac6ea51 | Michael Hanselmann | completion_cb=None):
|
40 | 02cab3e7 | Michael Hanselmann | """Describes an HTTP request.
|
41 | 02cab3e7 | Michael Hanselmann |
|
42 | 02cab3e7 | Michael Hanselmann | @type host: string
|
43 | 02cab3e7 | Michael Hanselmann | @param host: Hostname
|
44 | 02cab3e7 | Michael Hanselmann | @type port: int
|
45 | 02cab3e7 | Michael Hanselmann | @param port: Port
|
46 | 02cab3e7 | Michael Hanselmann | @type method: string
|
47 | 02cab3e7 | Michael Hanselmann | @param method: Method name
|
48 | 02cab3e7 | Michael Hanselmann | @type path: string
|
49 | 02cab3e7 | Michael Hanselmann | @param path: Request path
|
50 | 33231500 | Michael Hanselmann | @type headers: list or None
|
51 | 33231500 | Michael Hanselmann | @param headers: Additional headers to send, list of strings
|
52 | 02cab3e7 | Michael Hanselmann | @type post_data: string or None
|
53 | 02cab3e7 | Michael Hanselmann | @param post_data: Additional data to send
|
54 | e0036155 | Iustin Pop | @type read_timeout: int
|
55 | e0036155 | Iustin Pop | @param read_timeout: if passed, it will be used as the read
|
56 | e0036155 | Iustin Pop | timeout while reading the response from the server
|
57 | 33231500 | Michael Hanselmann | @type curl_config_fn: callable
|
58 | 33231500 | Michael Hanselmann | @param curl_config_fn: Function to configure cURL object before request
|
59 | 7cb2d205 | Michael Hanselmann | @type nicename: string
|
60 | 7cb2d205 | Michael Hanselmann | @param nicename: Name, presentable to a user, to describe this request (no
|
61 | 7cb2d205 | Michael Hanselmann | whitespace)
|
62 | bac6ea51 | Michael Hanselmann | @type completion_cb: callable accepting this request object as a single
|
63 | bac6ea51 | Michael Hanselmann | parameter
|
64 | bac6ea51 | Michael Hanselmann | @param completion_cb: Callback for request completion
|
65 | 02cab3e7 | Michael Hanselmann |
|
66 | 02cab3e7 | Michael Hanselmann | """
|
67 | 02cab3e7 | Michael Hanselmann | assert path.startswith("/"), "Path must start with slash (/)" |
68 | 33231500 | Michael Hanselmann | assert curl_config_fn is None or callable(curl_config_fn) |
69 | bac6ea51 | Michael Hanselmann | assert completion_cb is None or callable(completion_cb) |
70 | 02cab3e7 | Michael Hanselmann | |
71 | 02cab3e7 | Michael Hanselmann | # Request attributes
|
72 | 02cab3e7 | Michael Hanselmann | self.host = host
|
73 | 02cab3e7 | Michael Hanselmann | self.port = port
|
74 | 02cab3e7 | Michael Hanselmann | self.method = method
|
75 | 02cab3e7 | Michael Hanselmann | self.path = path
|
76 | e0036155 | Iustin Pop | self.read_timeout = read_timeout
|
77 | 33231500 | Michael Hanselmann | self.curl_config_fn = curl_config_fn
|
78 | 7cb2d205 | Michael Hanselmann | self.nicename = nicename
|
79 | bac6ea51 | Michael Hanselmann | self.completion_cb = completion_cb
|
80 | 02cab3e7 | Michael Hanselmann | |
81 | 33231500 | Michael Hanselmann | if post_data is None: |
82 | 33231500 | Michael Hanselmann | self.post_data = "" |
83 | 33231500 | Michael Hanselmann | else:
|
84 | 33231500 | Michael Hanselmann | self.post_data = post_data
|
85 | 33231500 | Michael Hanselmann | |
86 | 33231500 | Michael Hanselmann | if headers is None: |
87 | 33231500 | Michael Hanselmann | self.headers = []
|
88 | 33231500 | Michael Hanselmann | elif isinstance(headers, dict): |
89 | 33231500 | Michael Hanselmann | # Support for old interface
|
90 | 33231500 | Michael Hanselmann | self.headers = ["%s: %s" % (name, value) |
91 | 33231500 | Michael Hanselmann | for name, value in headers.items()] |
92 | 33231500 | Michael Hanselmann | else:
|
93 | 33231500 | Michael Hanselmann | self.headers = headers
|
94 | 33231500 | Michael Hanselmann | |
95 | 33231500 | Michael Hanselmann | # Response status
|
96 | 02cab3e7 | Michael Hanselmann | self.success = None |
97 | 02cab3e7 | Michael Hanselmann | self.error = None |
98 | 02cab3e7 | Michael Hanselmann | |
99 | 02cab3e7 | Michael Hanselmann | # Response attributes
|
100 | 02cab3e7 | Michael Hanselmann | self.resp_status_code = None |
101 | 02cab3e7 | Michael Hanselmann | self.resp_body = None |
102 | 02cab3e7 | Michael Hanselmann | |
103 | 9fa2e150 | Michael Hanselmann | def __repr__(self): |
104 | 9fa2e150 | Michael Hanselmann | status = ["%s.%s" % (self.__class__.__module__, self.__class__.__name__), |
105 | 9fa2e150 | Michael Hanselmann | "%s:%s" % (self.host, self.port), |
106 | 9fa2e150 | Michael Hanselmann | self.method,
|
107 | 9fa2e150 | Michael Hanselmann | self.path]
|
108 | 9fa2e150 | Michael Hanselmann | |
109 | 9fa2e150 | Michael Hanselmann | return "<%s at %#x>" % (" ".join(status), id(self)) |
110 | 9fa2e150 | Michael Hanselmann | |
111 | 33231500 | Michael Hanselmann | @property
|
112 | 33231500 | Michael Hanselmann | def url(self): |
113 | 33231500 | Michael Hanselmann | """Returns the full URL for this requests.
|
114 | 02cab3e7 | Michael Hanselmann |
|
115 | 33231500 | Michael Hanselmann | """
|
116 | 1a8337f2 | Manuel Franceschini | if netutils.IPAddress.IsValid(self.host): |
117 | 981732fb | Manuel Franceschini | address = netutils.FormatAddress((self.host, self.port)) |
118 | 1a8337f2 | Manuel Franceschini | else:
|
119 | 1a8337f2 | Manuel Franceschini | address = "%s:%s" % (self.host, self.port) |
120 | 33231500 | Michael Hanselmann | # TODO: Support for non-SSL requests
|
121 | 1a8337f2 | Manuel Franceschini | return "https://%s%s" % (address, self.path) |
122 | 02cab3e7 | Michael Hanselmann | |
123 | 02cab3e7 | Michael Hanselmann | |
124 | abbf2cd9 | Michael Hanselmann | def _StartRequest(curl, req): |
125 | abbf2cd9 | Michael Hanselmann | """Starts a request on a cURL object.
|
126 | 02cab3e7 | Michael Hanselmann |
|
127 | abbf2cd9 | Michael Hanselmann | @type curl: pycurl.Curl
|
128 | abbf2cd9 | Michael Hanselmann | @param curl: cURL object
|
129 | abbf2cd9 | Michael Hanselmann | @type req: L{HttpClientRequest}
|
130 | abbf2cd9 | Michael Hanselmann | @param req: HTTP request
|
131 | 02cab3e7 | Michael Hanselmann |
|
132 | abbf2cd9 | Michael Hanselmann | """
|
133 | abbf2cd9 | Michael Hanselmann | logging.debug("Starting request %r", req)
|
134 | 02cab3e7 | Michael Hanselmann | |
135 | abbf2cd9 | Michael Hanselmann | url = req.url |
136 | abbf2cd9 | Michael Hanselmann | method = req.method |
137 | abbf2cd9 | Michael Hanselmann | post_data = req.post_data |
138 | abbf2cd9 | Michael Hanselmann | headers = req.headers |
139 | 02cab3e7 | Michael Hanselmann | |
140 | abbf2cd9 | Michael Hanselmann | # PycURL requires strings to be non-unicode
|
141 | abbf2cd9 | Michael Hanselmann | assert isinstance(method, str) |
142 | abbf2cd9 | Michael Hanselmann | assert isinstance(url, str) |
143 | abbf2cd9 | Michael Hanselmann | assert isinstance(post_data, str) |
144 | abbf2cd9 | Michael Hanselmann | assert compat.all(isinstance(i, str) for i in headers) |
145 | 02cab3e7 | Michael Hanselmann | |
146 | abbf2cd9 | Michael Hanselmann | # Buffer for response
|
147 | abbf2cd9 | Michael Hanselmann | resp_buffer = StringIO() |
148 | 02cab3e7 | Michael Hanselmann | |
149 | abbf2cd9 | Michael Hanselmann | # Configure client for request
|
150 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.VERBOSE, False)
|
151 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.NOSIGNAL, True)
|
152 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.USERAGENT, http.HTTP_GANETI_VERSION) |
153 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.PROXY, "")
|
154 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.CUSTOMREQUEST, str(method))
|
155 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.URL, url) |
156 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.POSTFIELDS, post_data) |
157 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.HTTPHEADER, headers) |
158 | 02cab3e7 | Michael Hanselmann | |
159 | abbf2cd9 | Michael Hanselmann | if req.read_timeout is None: |
160 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.TIMEOUT, 0)
|
161 | abbf2cd9 | Michael Hanselmann | else:
|
162 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.TIMEOUT, int(req.read_timeout))
|
163 | 02cab3e7 | Michael Hanselmann | |
164 | abbf2cd9 | Michael Hanselmann | # Disable SSL session ID caching (pycurl >= 7.16.0)
|
165 | abbf2cd9 | Michael Hanselmann | if hasattr(pycurl, "SSL_SESSIONID_CACHE"): |
166 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.SSL_SESSIONID_CACHE, False)
|
167 | 02cab3e7 | Michael Hanselmann | |
168 | abbf2cd9 | Michael Hanselmann | curl.setopt(pycurl.WRITEFUNCTION, resp_buffer.write) |
169 | 7b70d7a8 | Apollon Oikonomopoulos | |
170 | abbf2cd9 | Michael Hanselmann | # Pass cURL object to external config function
|
171 | abbf2cd9 | Michael Hanselmann | if req.curl_config_fn:
|
172 | abbf2cd9 | Michael Hanselmann | req.curl_config_fn(curl) |
173 | 02cab3e7 | Michael Hanselmann | |
174 | abbf2cd9 | Michael Hanselmann | return _PendingRequest(curl, req, resp_buffer.getvalue)
|
175 | 02cab3e7 | Michael Hanselmann | |
176 | abbf2cd9 | Michael Hanselmann | |
177 | abbf2cd9 | Michael Hanselmann | class _PendingRequest: |
178 | abbf2cd9 | Michael Hanselmann | def __init__(self, curl, req, resp_buffer_read): |
179 | abbf2cd9 | Michael Hanselmann | """Initializes this class.
|
180 | abbf2cd9 | Michael Hanselmann |
|
181 | abbf2cd9 | Michael Hanselmann | @type curl: pycurl.Curl
|
182 | abbf2cd9 | Michael Hanselmann | @param curl: cURL object
|
183 | abbf2cd9 | Michael Hanselmann | @type req: L{HttpClientRequest}
|
184 | abbf2cd9 | Michael Hanselmann | @param req: HTTP request
|
185 | abbf2cd9 | Michael Hanselmann | @type resp_buffer_read: callable
|
186 | abbf2cd9 | Michael Hanselmann | @param resp_buffer_read: Function to read response body
|
187 | 02cab3e7 | Michael Hanselmann |
|
188 | 02cab3e7 | Michael Hanselmann | """
|
189 | abbf2cd9 | Michael Hanselmann | assert req.success is None |
190 | abbf2cd9 | Michael Hanselmann | |
191 | abbf2cd9 | Michael Hanselmann | self._curl = curl
|
192 | abbf2cd9 | Michael Hanselmann | self._req = req
|
193 | abbf2cd9 | Michael Hanselmann | self._resp_buffer_read = resp_buffer_read
|
194 | 02cab3e7 | Michael Hanselmann | |
195 | 33231500 | Michael Hanselmann | def GetCurlHandle(self): |
196 | 33231500 | Michael Hanselmann | """Returns the cURL object.
|
197 | 02cab3e7 | Michael Hanselmann |
|
198 | 33231500 | Michael Hanselmann | """
|
199 | 33231500 | Michael Hanselmann | return self._curl |
200 | 02cab3e7 | Michael Hanselmann | |
201 | 33231500 | Michael Hanselmann | def GetCurrentRequest(self): |
202 | 33231500 | Michael Hanselmann | """Returns the current request.
|
203 | 02cab3e7 | Michael Hanselmann |
|
204 | 33231500 | Michael Hanselmann | """
|
205 | 33231500 | Michael Hanselmann | return self._req |
206 | 02cab3e7 | Michael Hanselmann | |
207 | 33231500 | Michael Hanselmann | def Done(self, errmsg): |
208 | 33231500 | Michael Hanselmann | """Finishes a request.
|
209 | 02cab3e7 | Michael Hanselmann |
|
210 | 33231500 | Michael Hanselmann | @type errmsg: string or None
|
211 | 33231500 | Michael Hanselmann | @param errmsg: Error message if request failed
|
212 | 02cab3e7 | Michael Hanselmann |
|
213 | 02cab3e7 | Michael Hanselmann | """
|
214 | abbf2cd9 | Michael Hanselmann | curl = self._curl
|
215 | 33231500 | Michael Hanselmann | req = self._req
|
216 | 02cab3e7 | Michael Hanselmann | |
217 | abbf2cd9 | Michael Hanselmann | assert req.success is None, "Request has already been finalized" |
218 | 02cab3e7 | Michael Hanselmann | |
219 | abbf2cd9 | Michael Hanselmann | logging.debug("Request %s finished, errmsg=%s", req, errmsg)
|
220 | 02cab3e7 | Michael Hanselmann | |
221 | 33231500 | Michael Hanselmann | req.success = not bool(errmsg) |
222 | 33231500 | Michael Hanselmann | req.error = errmsg |
223 | 02cab3e7 | Michael Hanselmann | |
224 | 33231500 | Michael Hanselmann | # Get HTTP response code
|
225 | 33231500 | Michael Hanselmann | req.resp_status_code = curl.getinfo(pycurl.RESPONSE_CODE) |
226 | abbf2cd9 | Michael Hanselmann | req.resp_body = self._resp_buffer_read()
|
227 | 02cab3e7 | Michael Hanselmann | |
228 | 33231500 | Michael Hanselmann | # Ensure no potentially large variables are referenced
|
229 | bac6ea51 | Michael Hanselmann | curl.setopt(pycurl.POSTFIELDS, "")
|
230 | bac6ea51 | Michael Hanselmann | curl.setopt(pycurl.WRITEFUNCTION, lambda _: None) |
231 | bac6ea51 | Michael Hanselmann | |
232 | bac6ea51 | Michael Hanselmann | if req.completion_cb:
|
233 | bac6ea51 | Michael Hanselmann | req.completion_cb(req) |
234 | ecd61b4e | Michael Hanselmann | |
235 | ecd61b4e | Michael Hanselmann | |
236 | aea5caef | Michael Hanselmann | class _NoOpRequestMonitor: # pylint: disable=W0232 |
237 | aea5caef | Michael Hanselmann | """No-op request monitor.
|
238 | aea5caef | Michael Hanselmann |
|
239 | aea5caef | Michael Hanselmann | """
|
240 | aea5caef | Michael Hanselmann | @staticmethod
|
241 | aea5caef | Michael Hanselmann | def acquire(*args, **kwargs): |
242 | aea5caef | Michael Hanselmann | pass
|
243 | aea5caef | Michael Hanselmann | |
244 | aea5caef | Michael Hanselmann | release = acquire |
245 | aea5caef | Michael Hanselmann | Disable = acquire |
246 | aea5caef | Michael Hanselmann | |
247 | aea5caef | Michael Hanselmann | |
248 | aea5caef | Michael Hanselmann | class _PendingRequestMonitor: |
249 | aea5caef | Michael Hanselmann | _LOCK = "_lock"
|
250 | aea5caef | Michael Hanselmann | |
251 | aea5caef | Michael Hanselmann | def __init__(self, owner, pending_fn): |
252 | aea5caef | Michael Hanselmann | """Initializes this class.
|
253 | aea5caef | Michael Hanselmann |
|
254 | aea5caef | Michael Hanselmann | """
|
255 | aea5caef | Michael Hanselmann | self._owner = owner
|
256 | aea5caef | Michael Hanselmann | self._pending_fn = pending_fn
|
257 | aea5caef | Michael Hanselmann | |
258 | aea5caef | Michael Hanselmann | # The lock monitor runs in another thread, hence locking is necessary
|
259 | aea5caef | Michael Hanselmann | self._lock = locking.SharedLock("PendingHttpRequests") |
260 | aea5caef | Michael Hanselmann | self.acquire = self._lock.acquire |
261 | aea5caef | Michael Hanselmann | self.release = self._lock.release |
262 | aea5caef | Michael Hanselmann | |
263 | abbf2cd9 | Michael Hanselmann | @locking.ssynchronized(_LOCK)
|
264 | aea5caef | Michael Hanselmann | def Disable(self): |
265 | aea5caef | Michael Hanselmann | """Disable monitor.
|
266 | aea5caef | Michael Hanselmann |
|
267 | aea5caef | Michael Hanselmann | """
|
268 | aea5caef | Michael Hanselmann | self._pending_fn = None |
269 | aea5caef | Michael Hanselmann | |
270 | aea5caef | Michael Hanselmann | @locking.ssynchronized(_LOCK, shared=1) |
271 | aea5caef | Michael Hanselmann | def GetLockInfo(self, requested): # pylint: disable=W0613 |
272 | aea5caef | Michael Hanselmann | """Retrieves information about pending requests.
|
273 | aea5caef | Michael Hanselmann |
|
274 | aea5caef | Michael Hanselmann | @type requested: set
|
275 | aea5caef | Michael Hanselmann | @param requested: Requested information, see C{query.LQ_*}
|
276 | aea5caef | Michael Hanselmann |
|
277 | aea5caef | Michael Hanselmann | """
|
278 | aea5caef | Michael Hanselmann | # No need to sort here, that's being done by the lock manager and query
|
279 | aea5caef | Michael Hanselmann | # library. There are no priorities for requests, hence all show up as
|
280 | aea5caef | Michael Hanselmann | # one item under "pending".
|
281 | aea5caef | Michael Hanselmann | result = [] |
282 | aea5caef | Michael Hanselmann | |
283 | aea5caef | Michael Hanselmann | if self._pending_fn: |
284 | aea5caef | Michael Hanselmann | owner_name = self._owner.getName()
|
285 | aea5caef | Michael Hanselmann | |
286 | abbf2cd9 | Michael Hanselmann | for client in self._pending_fn(): |
287 | abbf2cd9 | Michael Hanselmann | req = client.GetCurrentRequest() |
288 | aea5caef | Michael Hanselmann | if req:
|
289 | 7cb2d205 | Michael Hanselmann | if req.nicename is None: |
290 | 7cb2d205 | Michael Hanselmann | name = "%s%s" % (req.host, req.path)
|
291 | 7cb2d205 | Michael Hanselmann | else:
|
292 | 7cb2d205 | Michael Hanselmann | name = req.nicename |
293 | 90b2eeb0 | Michael Hanselmann | result.append(("rpc/%s" % name, None, [owner_name], None)) |
294 | aea5caef | Michael Hanselmann | |
295 | aea5caef | Michael Hanselmann | return result
|
296 | aea5caef | Michael Hanselmann | |
297 | aea5caef | Michael Hanselmann | |
298 | ecd61b4e | Michael Hanselmann | def _ProcessCurlRequests(multi, requests): |
299 | ecd61b4e | Michael Hanselmann | """cURL request processor.
|
300 | ecd61b4e | Michael Hanselmann |
|
301 | ecd61b4e | Michael Hanselmann | This generator yields a tuple once for every completed request, successful or
|
302 | ecd61b4e | Michael Hanselmann | not. The first value in the tuple is the handle, the second an error message
|
303 | ecd61b4e | Michael Hanselmann | or C{None} for successful requests.
|
304 | ecd61b4e | Michael Hanselmann |
|
305 | ecd61b4e | Michael Hanselmann | @type multi: C{pycurl.CurlMulti}
|
306 | ecd61b4e | Michael Hanselmann | @param multi: cURL multi object
|
307 | ecd61b4e | Michael Hanselmann | @type requests: sequence
|
308 | ecd61b4e | Michael Hanselmann | @param requests: cURL request handles
|
309 | ecd61b4e | Michael Hanselmann |
|
310 | ecd61b4e | Michael Hanselmann | """
|
311 | ecd61b4e | Michael Hanselmann | for curl in requests: |
312 | ecd61b4e | Michael Hanselmann | multi.add_handle(curl) |
313 | ecd61b4e | Michael Hanselmann | |
314 | ecd61b4e | Michael Hanselmann | while True: |
315 | ecd61b4e | Michael Hanselmann | (ret, active) = multi.perform() |
316 | ecd61b4e | Michael Hanselmann | assert ret in (pycurl.E_MULTI_OK, pycurl.E_CALL_MULTI_PERFORM) |
317 | ecd61b4e | Michael Hanselmann | |
318 | ecd61b4e | Michael Hanselmann | if ret == pycurl.E_CALL_MULTI_PERFORM:
|
319 | ecd61b4e | Michael Hanselmann | # cURL wants to be called again
|
320 | ecd61b4e | Michael Hanselmann | continue
|
321 | ecd61b4e | Michael Hanselmann | |
322 | ecd61b4e | Michael Hanselmann | while True: |
323 | ecd61b4e | Michael Hanselmann | (remaining_messages, successful, failed) = multi.info_read() |
324 | ecd61b4e | Michael Hanselmann | |
325 | ecd61b4e | Michael Hanselmann | for curl in successful: |
326 | ecd61b4e | Michael Hanselmann | multi.remove_handle(curl) |
327 | ecd61b4e | Michael Hanselmann | yield (curl, None) |
328 | ecd61b4e | Michael Hanselmann | |
329 | ecd61b4e | Michael Hanselmann | for curl, errnum, errmsg in failed: |
330 | ecd61b4e | Michael Hanselmann | multi.remove_handle(curl) |
331 | ecd61b4e | Michael Hanselmann | yield (curl, "Error %s: %s" % (errnum, errmsg)) |
332 | ecd61b4e | Michael Hanselmann | |
333 | ecd61b4e | Michael Hanselmann | if remaining_messages == 0: |
334 | ecd61b4e | Michael Hanselmann | break
|
335 | ecd61b4e | Michael Hanselmann | |
336 | ecd61b4e | Michael Hanselmann | if active == 0: |
337 | ecd61b4e | Michael Hanselmann | # No active handles anymore
|
338 | ecd61b4e | Michael Hanselmann | break
|
339 | ecd61b4e | Michael Hanselmann | |
340 | ecd61b4e | Michael Hanselmann | # Wait for I/O. The I/O timeout shouldn't be too long so that HTTP
|
341 | ecd61b4e | Michael Hanselmann | # timeouts, which are only evaluated in multi.perform, aren't
|
342 | ecd61b4e | Michael Hanselmann | # unnecessarily delayed.
|
343 | ecd61b4e | Michael Hanselmann | multi.select(1.0)
|
344 | abbf2cd9 | Michael Hanselmann | |
345 | abbf2cd9 | Michael Hanselmann | |
346 | abbf2cd9 | Michael Hanselmann | def ProcessRequests(requests, lock_monitor_cb=None, _curl=pycurl.Curl, |
347 | abbf2cd9 | Michael Hanselmann | _curl_multi=pycurl.CurlMulti, |
348 | abbf2cd9 | Michael Hanselmann | _curl_process=_ProcessCurlRequests): |
349 | abbf2cd9 | Michael Hanselmann | """Processes any number of HTTP client requests.
|
350 | abbf2cd9 | Michael Hanselmann |
|
351 | abbf2cd9 | Michael Hanselmann | @type requests: list of L{HttpClientRequest}
|
352 | abbf2cd9 | Michael Hanselmann | @param requests: List of all requests
|
353 | abbf2cd9 | Michael Hanselmann | @param lock_monitor_cb: Callable for registering with lock monitor
|
354 | abbf2cd9 | Michael Hanselmann |
|
355 | abbf2cd9 | Michael Hanselmann | """
|
356 | abbf2cd9 | Michael Hanselmann | assert compat.all((req.error is None and |
357 | abbf2cd9 | Michael Hanselmann | req.success is None and |
358 | abbf2cd9 | Michael Hanselmann | req.resp_status_code is None and |
359 | abbf2cd9 | Michael Hanselmann | req.resp_body is None) |
360 | abbf2cd9 | Michael Hanselmann | for req in requests) |
361 | abbf2cd9 | Michael Hanselmann | |
362 | abbf2cd9 | Michael Hanselmann | # Prepare all requests
|
363 | abbf2cd9 | Michael Hanselmann | curl_to_client = \ |
364 | abbf2cd9 | Michael Hanselmann | dict((client.GetCurlHandle(), client)
|
365 | abbf2cd9 | Michael Hanselmann | for client in map(lambda req: _StartRequest(_curl(), req), requests)) |
366 | abbf2cd9 | Michael Hanselmann | |
367 | abbf2cd9 | Michael Hanselmann | assert len(curl_to_client) == len(requests) |
368 | abbf2cd9 | Michael Hanselmann | |
369 | abbf2cd9 | Michael Hanselmann | if lock_monitor_cb:
|
370 | abbf2cd9 | Michael Hanselmann | monitor = _PendingRequestMonitor(threading.currentThread(), |
371 | abbf2cd9 | Michael Hanselmann | curl_to_client.values) |
372 | abbf2cd9 | Michael Hanselmann | lock_monitor_cb(monitor) |
373 | abbf2cd9 | Michael Hanselmann | else:
|
374 | abbf2cd9 | Michael Hanselmann | monitor = _NoOpRequestMonitor |
375 | abbf2cd9 | Michael Hanselmann | |
376 | abbf2cd9 | Michael Hanselmann | # Process all requests and act based on the returned values
|
377 | abbf2cd9 | Michael Hanselmann | for (curl, msg) in _curl_process(_curl_multi(), curl_to_client.keys()): |
378 | abbf2cd9 | Michael Hanselmann | monitor.acquire(shared=0)
|
379 | abbf2cd9 | Michael Hanselmann | try:
|
380 | abbf2cd9 | Michael Hanselmann | curl_to_client.pop(curl).Done(msg) |
381 | abbf2cd9 | Michael Hanselmann | finally:
|
382 | abbf2cd9 | Michael Hanselmann | monitor.release() |
383 | abbf2cd9 | Michael Hanselmann | |
384 | abbf2cd9 | Michael Hanselmann | assert not curl_to_client, "Not all requests were processed" |
385 | abbf2cd9 | Michael Hanselmann | |
386 | abbf2cd9 | Michael Hanselmann | # Don't try to read information anymore as all requests have been processed
|
387 | abbf2cd9 | Michael Hanselmann | monitor.Disable() |
388 | abbf2cd9 | Michael Hanselmann | |
389 | abbf2cd9 | Michael Hanselmann | assert compat.all(req.error is not None or |
390 | abbf2cd9 | Michael Hanselmann | (req.success and
|
391 | abbf2cd9 | Michael Hanselmann | req.resp_status_code is not None and |
392 | abbf2cd9 | Michael Hanselmann | req.resp_body is not None) |
393 | abbf2cd9 | Michael Hanselmann | for req in requests) |