Revision ecd61b4e
b/lib/http/client.py | ||
---|---|---|
385 | 385 |
@param requests: List of all requests |
386 | 386 |
|
387 | 387 |
""" |
388 |
multi = self._CreateCurlMultiHandle() |
|
389 |
|
|
390 | 388 |
# For client cleanup |
391 | 389 |
self._generation += 1 |
392 | 390 |
|
... | ... | |
399 | 397 |
curl_to_pclient = {} |
400 | 398 |
for req in requests: |
401 | 399 |
pclient = self._StartRequest(req) |
402 |
curl = pclient.client.GetCurlHandle() |
|
403 |
curl_to_pclient[curl] = pclient |
|
404 |
multi.add_handle(curl) |
|
400 |
curl_to_pclient[pclient.client.GetCurlHandle()] = pclient |
|
405 | 401 |
assert pclient.client.GetCurrentRequest() == req |
406 | 402 |
assert pclient.lastused >= 0 |
407 | 403 |
|
408 | 404 |
assert len(curl_to_pclient) == len(requests) |
409 | 405 |
|
410 |
done_count = 0 |
|
411 |
while True: |
|
412 |
(ret, _) = multi.perform() |
|
413 |
assert ret in (pycurl.E_MULTI_OK, pycurl.E_CALL_MULTI_PERFORM) |
|
414 |
|
|
415 |
if ret == pycurl.E_CALL_MULTI_PERFORM: |
|
416 |
# cURL wants to be called again |
|
417 |
continue |
|
418 |
|
|
419 |
while True: |
|
420 |
(remaining_messages, successful, failed) = multi.info_read() |
|
421 |
|
|
422 |
for curl in successful: |
|
423 |
multi.remove_handle(curl) |
|
424 |
done_count += 1 |
|
425 |
pclient = curl_to_pclient[curl] |
|
426 |
req = pclient.client.GetCurrentRequest() |
|
427 |
pclient.client.Done(None) |
|
428 |
assert req.success |
|
429 |
assert not pclient.client.GetCurrentRequest() |
|
430 |
|
|
431 |
for curl, errnum, errmsg in failed: |
|
432 |
multi.remove_handle(curl) |
|
433 |
done_count += 1 |
|
434 |
pclient = curl_to_pclient[curl] |
|
435 |
req = pclient.client.GetCurrentRequest() |
|
436 |
pclient.client.Done("Error %s: %s" % (errnum, errmsg)) |
|
437 |
assert req.error |
|
438 |
assert not pclient.client.GetCurrentRequest() |
|
439 |
|
|
440 |
if remaining_messages == 0: |
|
441 |
break |
|
442 |
|
|
443 |
assert done_count <= len(requests) |
|
444 |
|
|
445 |
if done_count == len(requests): |
|
446 |
break |
|
406 |
# Process all requests and act based on the returned values |
|
407 |
for (curl, msg) in _ProcessCurlRequests(self._CreateCurlMultiHandle(), |
|
408 |
curl_to_pclient.keys()): |
|
409 |
pclient = curl_to_pclient[curl] |
|
410 |
req = pclient.client.GetCurrentRequest() |
|
411 |
pclient.client.Done(msg) |
|
447 | 412 |
|
448 |
# Wait for I/O. The I/O timeout shouldn't be too long so that HTTP |
|
449 |
# timeouts, which are only evaluated in multi.perform, aren't |
|
450 |
# unnecessarily delayed. |
|
451 |
multi.select(1.0) |
|
413 |
assert ((msg is None and req.success and req.error is None) ^ |
|
414 |
(msg is not None and not req.success and req.error == msg)) |
|
452 | 415 |
|
453 | 416 |
assert compat.all(pclient.client.GetCurrentRequest() is None |
454 | 417 |
for pclient in curl_to_pclient.values()) |
... | ... | |
456 | 419 |
# Return clients to pool |
457 | 420 |
self._Return(curl_to_pclient.values()) |
458 | 421 |
|
459 |
assert done_count == len(requests) |
|
460 | 422 |
assert compat.all(req.error is not None or |
461 | 423 |
(req.success and |
462 | 424 |
req.resp_status_code is not None and |
463 | 425 |
req.resp_body is not None) |
464 | 426 |
for req in requests) |
427 |
|
|
428 |
|
|
429 |
def _ProcessCurlRequests(multi, requests): |
|
430 |
"""cURL request processor. |
|
431 |
|
|
432 |
This generator yields a tuple once for every completed request, successful or |
|
433 |
not. The first value in the tuple is the handle, the second an error message |
|
434 |
or C{None} for successful requests. |
|
435 |
|
|
436 |
@type multi: C{pycurl.CurlMulti} |
|
437 |
@param multi: cURL multi object |
|
438 |
@type requests: sequence |
|
439 |
@param requests: cURL request handles |
|
440 |
|
|
441 |
""" |
|
442 |
for curl in requests: |
|
443 |
multi.add_handle(curl) |
|
444 |
|
|
445 |
while True: |
|
446 |
(ret, active) = multi.perform() |
|
447 |
assert ret in (pycurl.E_MULTI_OK, pycurl.E_CALL_MULTI_PERFORM) |
|
448 |
|
|
449 |
if ret == pycurl.E_CALL_MULTI_PERFORM: |
|
450 |
# cURL wants to be called again |
|
451 |
continue |
|
452 |
|
|
453 |
while True: |
|
454 |
(remaining_messages, successful, failed) = multi.info_read() |
|
455 |
|
|
456 |
for curl in successful: |
|
457 |
multi.remove_handle(curl) |
|
458 |
yield (curl, None) |
|
459 |
|
|
460 |
for curl, errnum, errmsg in failed: |
|
461 |
multi.remove_handle(curl) |
|
462 |
yield (curl, "Error %s: %s" % (errnum, errmsg)) |
|
463 |
|
|
464 |
if remaining_messages == 0: |
|
465 |
break |
|
466 |
|
|
467 |
if active == 0: |
|
468 |
# No active handles anymore |
|
469 |
break |
|
470 |
|
|
471 |
# Wait for I/O. The I/O timeout shouldn't be too long so that HTTP |
|
472 |
# timeouts, which are only evaluated in multi.perform, aren't |
|
473 |
# unnecessarily delayed. |
|
474 |
multi.select(1.0) |
Also available in: Unified diff