Revision 73a59d9e lib/http.py
| b/lib/http.py | ||
|---|---|---|
| 90 | 90 |
|
| 91 | 91 |
_SSL_UNEXPECTED_EOF = "Unexpected EOF" |
| 92 | 92 |
|
| 93 |
# Socket operations |
|
| 94 |
(SOCKOP_SEND, |
|
| 95 |
SOCKOP_RECV, |
|
| 96 |
SOCKOP_SHUTDOWN) = range(3) |
|
| 97 |
|
|
| 93 | 98 |
|
| 94 | 99 |
class SocketClosed(socket.error): |
| 95 | 100 |
pass |
| ... | ... | |
| 101 | 106 |
This should only be used for internal error reporting. |
| 102 | 107 |
|
| 103 | 108 |
""" |
| 104 |
pass |
|
| 109 |
|
|
| 110 |
|
|
| 111 |
class _HttpSocketTimeout(Exception): |
|
| 112 |
"""Internal exception for socket timeouts. |
|
| 113 |
|
|
| 114 |
This should only be used for internal error reporting. |
|
| 115 |
|
|
| 116 |
""" |
|
| 105 | 117 |
|
| 106 | 118 |
|
| 107 | 119 |
class HTTPException(Exception): |
| ... | ... | |
| 160 | 172 |
return serializer.LoadJson(data) |
| 161 | 173 |
|
| 162 | 174 |
|
| 163 |
def WaitForSocketCondition(sock, poller, event, timeout):
|
|
| 175 |
def WaitForSocketCondition(poller, sock, event, timeout):
|
|
| 164 | 176 |
"""Waits for a condition to occur on the socket. |
| 165 | 177 |
|
| 166 |
@type sock: socket |
|
| 167 |
@param socket: Wait for events on this socket |
|
| 168 | 178 |
@type poller: select.Poller |
| 169 | 179 |
@param poller: Poller object as created by select.poll() |
| 180 |
@type sock: socket |
|
| 181 |
@param socket: Wait for events on this socket |
|
| 170 | 182 |
@type event: int |
| 171 | 183 |
@param event: ORed condition (see select module) |
| 172 | 184 |
@type timeout: float or None |
| ... | ... | |
| 199 | 211 |
poller.unregister(sock) |
| 200 | 212 |
|
| 201 | 213 |
|
| 214 |
def SocketOperation(poller, sock, op, arg1, timeout): |
|
| 215 |
"""Wrapper around socket functions. |
|
| 216 |
|
|
| 217 |
This function abstracts error handling for socket operations, especially |
|
| 218 |
for the complicated interaction with OpenSSL. |
|
| 219 |
|
|
| 220 |
@type poller: select.Poller |
|
| 221 |
@param poller: Poller object as created by select.poll() |
|
| 222 |
@type sock: socket |
|
| 223 |
@param socket: Socket for the operation |
|
| 224 |
@type op: int |
|
| 225 |
@param op: Operation to execute (SOCKOP_* constants) |
|
| 226 |
@type arg1: any |
|
| 227 |
@param arg1: Parameter for function (if needed) |
|
| 228 |
@type timeout: None or float |
|
| 229 |
@param timeout: Timeout in seconds or None |
|
| 230 |
|
|
| 231 |
""" |
|
| 232 |
# TODO: event_poll/event_check/override |
|
| 233 |
if op == SOCKOP_SEND: |
|
| 234 |
event_poll = select.POLLOUT |
|
| 235 |
event_check = select.POLLOUT |
|
| 236 |
|
|
| 237 |
elif op == SOCKOP_RECV: |
|
| 238 |
event_poll = select.POLLIN |
|
| 239 |
event_check = select.POLLIN | select.POLLPRI |
|
| 240 |
|
|
| 241 |
elif op == SOCKOP_SHUTDOWN: |
|
| 242 |
event_poll = None |
|
| 243 |
event_check = None |
|
| 244 |
|
|
| 245 |
# The timeout is only used when OpenSSL requests polling for a condition. |
|
| 246 |
# It is not advisable to have no timeout for shutdown. |
|
| 247 |
assert timeout |
|
| 248 |
|
|
| 249 |
else: |
|
| 250 |
raise AssertionError("Invalid socket operation")
|
|
| 251 |
|
|
| 252 |
# No override by default |
|
| 253 |
event_override = 0 |
|
| 254 |
|
|
| 255 |
while True: |
|
| 256 |
# Poll only for certain operations and when asked for by an override |
|
| 257 |
if event_override or op in (SOCKOP_SEND, SOCKOP_RECV): |
|
| 258 |
if event_override: |
|
| 259 |
wait_for_event = event_override |
|
| 260 |
else: |
|
| 261 |
wait_for_event = event_poll |
|
| 262 |
|
|
| 263 |
event = WaitForSocketCondition(poller, sock, wait_for_event, timeout) |
|
| 264 |
if event is None: |
|
| 265 |
raise _HttpSocketTimeout() |
|
| 266 |
|
|
| 267 |
if (op == SOCKOP_RECV and |
|
| 268 |
event & (select.POLLNVAL | select.POLLHUP | select.POLLERR)): |
|
| 269 |
return "" |
|
| 270 |
|
|
| 271 |
if not event & wait_for_event: |
|
| 272 |
continue |
|
| 273 |
|
|
| 274 |
# Reset override |
|
| 275 |
event_override = 0 |
|
| 276 |
|
|
| 277 |
try: |
|
| 278 |
try: |
|
| 279 |
if op == SOCKOP_SEND: |
|
| 280 |
return sock.send(arg1) |
|
| 281 |
|
|
| 282 |
elif op == SOCKOP_RECV: |
|
| 283 |
return sock.recv(arg1) |
|
| 284 |
|
|
| 285 |
elif op == SOCKOP_SHUTDOWN: |
|
| 286 |
if isinstance(sock, OpenSSL.SSL.ConnectionType): |
|
| 287 |
# PyOpenSSL's shutdown() doesn't take arguments |
|
| 288 |
return sock.shutdown() |
|
| 289 |
else: |
|
| 290 |
return sock.shutdown(arg1) |
|
| 291 |
|
|
| 292 |
except OpenSSL.SSL.WantWriteError: |
|
| 293 |
# OpenSSL wants to write, poll for POLLOUT |
|
| 294 |
event_override = select.POLLOUT |
|
| 295 |
continue |
|
| 296 |
|
|
| 297 |
except OpenSSL.SSL.WantReadError: |
|
| 298 |
# OpenSSL wants to read, poll for POLLIN |
|
| 299 |
event_override = select.POLLIN | select.POLLPRI |
|
| 300 |
continue |
|
| 301 |
|
|
| 302 |
except OpenSSL.SSL.WantX509LookupError: |
|
| 303 |
continue |
|
| 304 |
|
|
| 305 |
except OpenSSL.SSL.SysCallError, err: |
|
| 306 |
if op == SOCKOP_SEND: |
|
| 307 |
# arg1 is the data when writing |
|
| 308 |
if err.args and err.args[0] == -1 and arg1 == "": |
|
| 309 |
# errors when writing empty strings are expected |
|
| 310 |
# and can be ignored |
|
| 311 |
return 0 |
|
| 312 |
|
|
| 313 |
elif op == SOCKOP_RECV: |
|
| 314 |
if err.args == (-1, _SSL_UNEXPECTED_EOF): |
|
| 315 |
return "" |
|
| 316 |
|
|
| 317 |
raise socket.error(err.args) |
|
| 318 |
|
|
| 319 |
except OpenSSL.SSL.Error, err: |
|
| 320 |
raise socket.error(err.args) |
|
| 321 |
|
|
| 322 |
except socket.error, err: |
|
| 323 |
if err.args and err.args[0] == errno.EAGAIN: |
|
| 324 |
# Ignore EAGAIN |
|
| 325 |
continue |
|
| 326 |
|
|
| 327 |
raise |
|
| 328 |
|
|
| 329 |
|
|
| 202 | 330 |
class HttpSslParams(object): |
| 203 | 331 |
"""Data class for SSL key and certificate. |
| 204 | 332 |
|
| ... | ... | |
| 773 | 901 |
PS_BODY = "body" |
| 774 | 902 |
PS_COMPLETE = "complete" |
| 775 | 903 |
|
| 776 |
# Socket operations |
|
| 777 |
(OP_SEND, |
|
| 778 |
OP_RECV, |
|
| 779 |
OP_CLOSE_CHECK, |
|
| 780 |
OP_SHUTDOWN) = range(4) |
|
| 781 |
|
|
| 782 | 904 |
def __init__(self, req): |
| 783 | 905 |
"""Initializes the HttpClientRequestExecutor class. |
| 784 | 906 |
|
| ... | ... | |
| 1045 | 1167 |
|
| 1046 | 1168 |
return buf |
| 1047 | 1169 |
|
| 1048 |
def _SocketOperation(self, op, arg1, error_msg, timeout_msg): |
|
| 1049 |
"""Wrapper around socket functions. |
|
| 1050 |
|
|
| 1051 |
This function abstracts error handling for socket operations, especially |
|
| 1052 |
for the complicated interaction with OpenSSL. |
|
| 1053 |
|
|
| 1054 |
""" |
|
| 1055 |
if op == self.OP_SEND: |
|
| 1056 |
event_poll = select.POLLOUT |
|
| 1057 |
event_check = select.POLLOUT |
|
| 1058 |
timeout = self.WRITE_TIMEOUT |
|
| 1059 |
|
|
| 1060 |
elif op in (self.OP_RECV, self.OP_CLOSE_CHECK): |
|
| 1061 |
event_poll = select.POLLIN |
|
| 1062 |
event_check = select.POLLIN | select.POLLPRI |
|
| 1063 |
if op == self.OP_CLOSE_CHECK: |
|
| 1064 |
timeout = self.CLOSE_TIMEOUT |
|
| 1065 |
else: |
|
| 1066 |
timeout = self.READ_TIMEOUT |
|
| 1067 |
|
|
| 1068 |
elif op == self.OP_SHUTDOWN: |
|
| 1069 |
event_poll = None |
|
| 1070 |
event_check = None |
|
| 1071 |
|
|
| 1072 |
# The timeout is only used when OpenSSL requests polling for a condition. |
|
| 1073 |
# It is not advisable to have no timeout for shutdown. |
|
| 1074 |
timeout = self.WRITE_TIMEOUT |
|
| 1075 |
|
|
| 1076 |
else: |
|
| 1077 |
raise AssertionError("Invalid socket operation")
|
|
| 1078 |
|
|
| 1079 |
# No override by default |
|
| 1080 |
event_override = 0 |
|
| 1081 |
|
|
| 1082 |
while True: |
|
| 1083 |
# Poll only for certain operations and when asked for by an override |
|
| 1084 |
if (event_override or |
|
| 1085 |
op in (self.OP_SEND, self.OP_RECV, self.OP_CLOSE_CHECK)): |
|
| 1086 |
if event_override: |
|
| 1087 |
wait_for_event = event_override |
|
| 1088 |
else: |
|
| 1089 |
wait_for_event = event_poll |
|
| 1090 |
|
|
| 1091 |
event = WaitForSocketCondition(self.sock, self.poller, wait_for_event, |
|
| 1092 |
timeout) |
|
| 1093 |
if event is None: |
|
| 1094 |
raise _HttpClientTimeout(timeout_msg) |
|
| 1095 |
|
|
| 1096 |
if (op == self.OP_RECV and |
|
| 1097 |
event & (select.POLLNVAL | select.POLLHUP | select.POLLERR)): |
|
| 1098 |
return "" |
|
| 1099 |
|
|
| 1100 |
if not event & wait_for_event: |
|
| 1101 |
continue |
|
| 1102 |
|
|
| 1103 |
# Reset override |
|
| 1104 |
event_override = 0 |
|
| 1105 |
|
|
| 1106 |
try: |
|
| 1107 |
try: |
|
| 1108 |
if op == self.OP_SEND: |
|
| 1109 |
return self.sock.send(arg1) |
|
| 1110 |
|
|
| 1111 |
elif op in (self.OP_RECV, self.OP_CLOSE_CHECK): |
|
| 1112 |
return self.sock.recv(arg1) |
|
| 1113 |
|
|
| 1114 |
elif op == self.OP_SHUTDOWN: |
|
| 1115 |
if self._using_ssl: |
|
| 1116 |
# PyOpenSSL's shutdown() doesn't take arguments |
|
| 1117 |
return self.sock.shutdown() |
|
| 1118 |
else: |
|
| 1119 |
return self.sock.shutdown(arg1) |
|
| 1120 |
|
|
| 1121 |
except OpenSSL.SSL.WantWriteError: |
|
| 1122 |
# OpenSSL wants to write, poll for POLLOUT |
|
| 1123 |
event_override = select.POLLOUT |
|
| 1124 |
continue |
|
| 1125 |
|
|
| 1126 |
except OpenSSL.SSL.WantReadError: |
|
| 1127 |
# OpenSSL wants to read, poll for POLLIN |
|
| 1128 |
event_override = select.POLLIN | select.POLLPRI |
|
| 1129 |
continue |
|
| 1130 |
|
|
| 1131 |
except OpenSSL.SSL.WantX509LookupError: |
|
| 1132 |
continue |
|
| 1133 |
|
|
| 1134 |
except OpenSSL.SSL.SysCallError, err: |
|
| 1135 |
if op == self.OP_SEND: |
|
| 1136 |
# arg1 is the data when writing |
|
| 1137 |
if err.args and err.args[0] == -1 and arg1 == "": |
|
| 1138 |
# errors when writing empty strings are expected |
|
| 1139 |
# and can be ignored |
|
| 1140 |
return 0 |
|
| 1141 |
|
|
| 1142 |
elif op == self.OP_RECV: |
|
| 1143 |
if err.args == (-1, _SSL_UNEXPECTED_EOF): |
|
| 1144 |
return "" |
|
| 1145 |
|
|
| 1146 |
raise socket.error(err.args) |
|
| 1147 |
|
|
| 1148 |
except OpenSSL.SSL.Error, err: |
|
| 1149 |
raise socket.error(err.args) |
|
| 1150 |
|
|
| 1151 |
except socket.error, err: |
|
| 1152 |
if err.args and err.args[0] == errno.EAGAIN: |
|
| 1153 |
# Ignore EAGAIN |
|
| 1154 |
continue |
|
| 1155 |
|
|
| 1156 |
raise _HttpClientError("%s: %s" % (error_msg, str(err)))
|
|
| 1157 |
|
|
| 1158 | 1170 |
def _Connect(self): |
| 1159 | 1171 |
"""Non-blocking connect to host with timeout. |
| 1160 | 1172 |
|
| ... | ... | |
| 1185 | 1197 |
|
| 1186 | 1198 |
if not connected: |
| 1187 | 1199 |
# Wait for connection |
| 1188 |
event = WaitForSocketCondition(self.sock, self.poller,
|
|
| 1200 |
event = WaitForSocketCondition(self.poller, self.sock,
|
|
| 1189 | 1201 |
select.POLLOUT, self.CONNECT_TIMEOUT) |
| 1190 | 1202 |
if event is None: |
| 1191 | 1203 |
raise _HttpClientError("Timeout while connecting to server")
|
| ... | ... | |
| 1213 | 1225 |
# Send only 4 KB at a time |
| 1214 | 1226 |
data = buf[:4096] |
| 1215 | 1227 |
|
| 1216 |
sent = self._SocketOperation(self.OP_SEND, data, |
|
| 1217 |
"Error while sending request", |
|
| 1218 |
"Timeout while sending request") |
|
| 1228 |
try: |
|
| 1229 |
sent = SocketOperation(self.poller, self.sock, SOCKOP_SEND, data, |
|
| 1230 |
self.WRITE_TIMEOUT) |
|
| 1231 |
except _HttpSocketTimeout: |
|
| 1232 |
raise _HttpClientError("Timeout while sending request")
|
|
| 1233 |
except socket.error, err: |
|
| 1234 |
raise _HttpClientError("Error sending request: %s" % err)
|
|
| 1219 | 1235 |
|
| 1220 | 1236 |
# Remove sent bytes |
| 1221 | 1237 |
buf = buf[sent:] |
| ... | ... | |
| 1231 | 1247 |
buf = "" |
| 1232 | 1248 |
eof = False |
| 1233 | 1249 |
while self.parser_status != self.PS_COMPLETE: |
| 1234 |
data = self._SocketOperation(self.OP_RECV, 4096, |
|
| 1235 |
"Error while reading response", |
|
| 1236 |
"Timeout while reading response") |
|
| 1250 |
try: |
|
| 1251 |
data = SocketOperation(self.poller, self.sock, SOCKOP_RECV, 4096, |
|
| 1252 |
self.READ_TIMEOUT) |
|
| 1253 |
except _HttpSocketTimeout: |
|
| 1254 |
raise _HttpClientError("Timeout while reading response")
|
|
| 1255 |
except socket.error, err: |
|
| 1256 |
raise _HttpClientError("Error while reading response: %s" % err)
|
|
| 1237 | 1257 |
|
| 1238 | 1258 |
if data: |
| 1239 | 1259 |
buf += data |
| ... | ... | |
| 1263 | 1283 |
# Wait for server to close |
| 1264 | 1284 |
try: |
| 1265 | 1285 |
# Check whether it's actually closed |
| 1266 |
if not self._SocketOperation(self.OP_CLOSE_CHECK, 1,
|
|
| 1267 |
"Error", "Timeout"):
|
|
| 1286 |
if not SocketOperation(self.poller, self.sock, SOCKOP_RECV, 1,
|
|
| 1287 |
self.CLOSE_TIMEOUT):
|
|
| 1268 | 1288 |
return |
| 1269 |
except (socket.error, _HttpClientError): |
|
| 1289 |
except (socket.error, _HttpClientError, _HttpSocketTimeout):
|
|
| 1270 | 1290 |
# Ignore errors at this stage |
| 1271 | 1291 |
pass |
| 1272 | 1292 |
|
| 1273 | 1293 |
# Close the connection from our side |
| 1274 |
self._SocketOperation(self.OP_SHUTDOWN, socket.SHUT_RDWR, |
|
| 1275 |
"Error while shutting down connection", |
|
| 1276 |
"Timeout while shutting down connection") |
|
| 1294 |
try: |
|
| 1295 |
SocketOperation(self.poller, self.sock, SOCKOP_SHUTDOWN, |
|
| 1296 |
socket.SHUT_RDWR, self.WRITE_TIMEOUT) |
|
| 1297 |
except _HttpSocketTimeout: |
|
| 1298 |
raise _HttpClientError("Timeout while shutting down connection")
|
|
| 1299 |
except socket.error, err: |
|
| 1300 |
raise _HttpClientError("Error while shutting down connection: %s" % err)
|
|
| 1277 | 1301 |
|
| 1278 | 1302 |
|
| 1279 | 1303 |
class _HttpClientPendingRequest(object): |
Also available in: Unified diff