4 # Copyright (C) 2007, 2008 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
32 from cStringIO import StringIO
34 from ganeti import constants
35 from ganeti import serializer
36 from ganeti import utils
39 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION
43 HTTP_NOT_MODIFIED = 304
53 HTTP_DELETE = "DELETE"
57 HTTP_SERVER = "Server"
59 HTTP_USER_AGENT = "User-Agent"
60 HTTP_CONTENT_TYPE = "Content-Type"
61 HTTP_CONTENT_LENGTH = "Content-Length"
62 HTTP_CONNECTION = "Connection"
63 HTTP_KEEP_ALIVE = "Keep-Alive"
64 HTTP_WWW_AUTHENTICATE = "WWW-Authenticate"
65 HTTP_AUTHORIZATION = "Authorization"
66 HTTP_AUTHENTICATION_INFO = "Authentication-Info"
69 _SSL_UNEXPECTED_EOF = "Unexpected EOF"
75 SOCKOP_HANDSHAKE) = range(4)
77 # send/receive quantum
81 class HttpError(Exception):
82 """Internal exception for HTTP errors.
84 This should only be used for internal error reporting.
89 class HttpConnectionClosed(Exception):
90 """Internal exception for a closed connection.
92 This should only be used for internal error reporting. Only use
93 it if there's no other way to report this condition.
98 class HttpSessionHandshakeUnexpectedEOF(HttpError):
99 """Internal exception for errors during SSL handshake.
101 This should only be used for internal error reporting.
106 class HttpSocketTimeout(Exception):
107 """Internal exception for socket timeouts.
109 This should only be used for internal error reporting.
114 class HttpException(Exception):
118 def __init__(self, message=None, headers=None):
119 Exception.__init__(self)
120 self.message = message
121 self.headers = headers
124 class HttpBadRequest(HttpException):
127 RFC2616, 10.4.1: The request could not be understood by the server
128 due to malformed syntax. The client SHOULD NOT repeat the request
129 without modifications.
135 class HttpUnauthorized(HttpException):
138 RFC2616, section 10.4.2: The request requires user
139 authentication. The response MUST include a WWW-Authenticate header
140 field (section 14.47) containing a challenge applicable to the
147 class HttpForbidden(HttpException):
150 RFC2616, 10.4.4: The server understood the request, but is refusing
151 to fulfill it. Authorization will not help and the request SHOULD
158 class HttpNotFound(HttpException):
161 RFC2616, 10.4.5: The server has not found anything matching the
162 Request-URI. No indication is given of whether the condition is
163 temporary or permanent.
169 class HttpMethodNotAllowed(HttpException):
170 """405 Method Not Allowed
172 RFC2616, 10.4.6: The method specified in the Request-Line is not
173 allowed for the resource identified by the Request-URI. The response
174 MUST include an Allow header containing a list of valid methods for
175 the requested resource.
181 class HttpRequestTimeout(HttpException):
182 """408 Request Timeout
184 RFC2616, 10.4.9: The client did not produce a request within the
185 time that the server was prepared to wait. The client MAY repeat the
186 request without modifications at any later time.
192 class HttpConflict(HttpException):
195 RFC2616, 10.4.10: The request could not be completed due to a
196 conflict with the current state of the resource. This code is only
197 allowed in situations where it is expected that the user might be
198 able to resolve the conflict and resubmit the request.
204 class HttpGone(HttpException):
207 RFC2616, 10.4.11: The requested resource is no longer available at
208 the server and no forwarding address is known. This condition is
209 expected to be considered permanent.
215 class HttpLengthRequired(HttpException):
216 """411 Length Required
218 RFC2616, 10.4.12: The server refuses to accept the request without a
219 defined Content-Length. The client MAY repeat the request if it adds
220 a valid Content-Length header field containing the length of the
221 message-body in the request message.
227 class HttpPreconditionFailed(HttpException):
228 """412 Precondition Failed
230 RFC2616, 10.4.13: The precondition given in one or more of the
231 request-header fields evaluated to false when it was tested on the
238 class HttpInternalServerError(HttpException):
239 """500 Internal Server Error
241 RFC2616, 10.5.1: The server encountered an unexpected condition
242 which prevented it from fulfilling the request.
248 class HttpNotImplemented(HttpException):
249 """501 Not Implemented
251 RFC2616, 10.5.2: The server does not support the functionality
252 required to fulfill the request.
258 class HttpBadGateway(HttpException):
261 RFC2616, 10.5.3: The server, while acting as a gateway or proxy,
262 received an invalid response from the upstream server it accessed in
263 attempting to fulfill the request.
269 class HttpServiceUnavailable(HttpException):
270 """503 Service Unavailable
272 RFC2616, 10.5.4: The server is currently unable to handle the
273 request due to a temporary overloading or maintenance of the server.
279 class HttpGatewayTimeout(HttpException):
280 """504 Gateway Timeout
282 RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did
283 not receive a timely response from the upstream server specified by
284 the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server
285 (e.g. DNS) it needed to access in attempting to complete the
292 class HttpVersionNotSupported(HttpException):
293 """505 HTTP Version Not Supported
295 RFC2616, 10.5.6: The server does not support, or refuses to support,
296 the HTTP protocol version that was used in the request message.
302 class HttpJsonConverter:
303 CONTENT_TYPE = "application/json"
305 def Encode(self, data):
306 return serializer.DumpJson(data)
308 def Decode(self, data):
309 return serializer.LoadJson(data)
312 def WaitForSocketCondition(sock, event, timeout):
313 """Waits for a condition to occur on the socket.
316 @param sock: Wait for events on this socket
318 @param event: ORed condition (see select module)
319 @type timeout: float or None
320 @param timeout: Timeout in seconds
322 @return: None for timeout, otherwise occured conditions
325 check = (event | select.POLLPRI |
326 select.POLLNVAL | select.POLLHUP | select.POLLERR)
328 if timeout is not None:
329 # Poller object expects milliseconds
332 poller = select.poll()
333 poller.register(sock, event)
336 # TODO: If the main thread receives a signal and we have no timeout, we
337 # could wait forever. This should check a global "quit" flag or
338 # something every so often.
339 io_events = poller.poll(timeout)
343 for (evfd, evcond) in io_events:
347 poller.unregister(sock)
350 def SocketOperation(sock, op, arg1, timeout):
351 """Wrapper around socket functions.
353 This function abstracts error handling for socket operations, especially
354 for the complicated interaction with OpenSSL.
357 @param sock: Socket for the operation
359 @param op: Operation to execute (SOCKOP_* constants)
361 @param arg1: Parameter for function (if needed)
362 @type timeout: None or float
363 @param timeout: Timeout in seconds or None
364 @return: Return value of socket function
367 # TODO: event_poll/event_check/override
368 if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE):
369 event_poll = select.POLLOUT
370 event_check = select.POLLOUT
372 elif op == SOCKOP_RECV:
373 event_poll = select.POLLIN
374 event_check = select.POLLIN | select.POLLPRI
376 elif op == SOCKOP_SHUTDOWN:
380 # The timeout is only used when OpenSSL requests polling for a condition.
381 # It is not advisable to have no timeout for shutdown.
385 raise AssertionError("Invalid socket operation")
387 # Handshake is only supported by SSL sockets
388 if (op == SOCKOP_HANDSHAKE and
389 not isinstance(sock, OpenSSL.SSL.ConnectionType)):
392 # No override by default
396 # Poll only for certain operations and when asked for by an override
397 if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE):
399 wait_for_event = event_override
401 wait_for_event = event_poll
403 event = WaitForSocketCondition(sock, wait_for_event, timeout)
405 raise HttpSocketTimeout()
407 if (op == SOCKOP_RECV and
408 event & (select.POLLNVAL | select.POLLHUP | select.POLLERR)):
411 if not event & wait_for_event:
419 if op == SOCKOP_SEND:
420 return sock.send(arg1)
422 elif op == SOCKOP_RECV:
423 return sock.recv(arg1)
425 elif op == SOCKOP_SHUTDOWN:
426 if isinstance(sock, OpenSSL.SSL.ConnectionType):
427 # PyOpenSSL's shutdown() doesn't take arguments
428 return sock.shutdown()
430 return sock.shutdown(arg1)
432 elif op == SOCKOP_HANDSHAKE:
433 return sock.do_handshake()
435 except OpenSSL.SSL.WantWriteError:
436 # OpenSSL wants to write, poll for POLLOUT
437 event_override = select.POLLOUT
440 except OpenSSL.SSL.WantReadError:
441 # OpenSSL wants to read, poll for POLLIN
442 event_override = select.POLLIN | select.POLLPRI
445 except OpenSSL.SSL.WantX509LookupError:
448 except OpenSSL.SSL.ZeroReturnError, err:
449 # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only
450 # occurs if a closure alert has occurred in the protocol, i.e. the
451 # connection has been closed cleanly. Note that this does not
452 # necessarily mean that the transport layer (e.g. a socket) has been
454 if op == SOCKOP_SEND:
455 # Can happen during a renegotiation
456 raise HttpConnectionClosed(err.args)
457 elif op == SOCKOP_RECV:
460 # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN
461 raise socket.error(err.args)
463 except OpenSSL.SSL.SysCallError, err:
464 if op == SOCKOP_SEND:
465 # arg1 is the data when writing
466 if err.args and err.args[0] == -1 and arg1 == "":
467 # errors when writing empty strings are expected
471 if err.args == (-1, _SSL_UNEXPECTED_EOF):
472 if op == SOCKOP_RECV:
474 elif op == SOCKOP_HANDSHAKE:
475 # Can happen if peer disconnects directly after the connection is
477 raise HttpSessionHandshakeUnexpectedEOF(err.args)
479 raise socket.error(err.args)
481 except OpenSSL.SSL.Error, err:
482 raise socket.error(err.args)
484 except socket.error, err:
485 if err.args and err.args[0] == errno.EAGAIN:
492 def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):
493 """Closes the connection.
496 @param sock: Socket to be shut down
497 @type close_timeout: float
498 @param close_timeout: How long to wait for the peer to close
500 @type write_timeout: float
501 @param write_timeout: Write timeout for shutdown
502 @type msgreader: http.HttpMessageReader
503 @param msgreader: Request message reader, used to determine whether
504 peer should close connection
506 @param force: Whether to forcibly close the connection without
510 #print msgreader.peer_will_close, force
511 if msgreader and msgreader.peer_will_close and not force:
512 # Wait for peer to close
514 # Check whether it's actually closed
515 if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout):
517 except (socket.error, HttpError, HttpSocketTimeout):
518 # Ignore errors at this stage
521 # Close the connection from our side
523 # We don't care about the return value, see NOTES in SSL_shutdown(3).
524 SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR,
526 except HttpSocketTimeout:
527 raise HttpError("Timeout while shutting down connection")
528 except socket.error, err:
530 if not (err.args and err.args[0] == errno.ENOTCONN):
531 raise HttpError("Error while shutting down connection: %s" % err)
534 def Handshake(sock, write_timeout):
535 """Shakes peer's hands.
538 @param sock: Socket to be shut down
539 @type write_timeout: float
540 @param write_timeout: Write timeout for handshake
544 return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout)
545 except HttpSocketTimeout:
546 raise HttpError("Timeout during SSL handshake")
547 except socket.error, err:
548 raise HttpError("Error in SSL handshake: %s" % err)
551 class HttpSslParams(object):
552 """Data class for SSL key and certificate.
555 def __init__(self, ssl_key_path, ssl_cert_path):
556 """Initializes this class.
558 @type ssl_key_path: string
559 @param ssl_key_path: Path to file containing SSL key in PEM format
560 @type ssl_cert_path: string
561 @param ssl_cert_path: Path to file containing SSL certificate
565 self.ssl_key_pem = utils.ReadFile(ssl_key_path)
566 self.ssl_cert_pem = utils.ReadFile(ssl_cert_path)
569 return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM,
572 def GetCertificate(self):
573 return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
577 class HttpBase(object):
578 """Base class for HTTP server and client.
582 self.using_ssl = None
583 self._ssl_params = None
585 self._ssl_cert = None
587 def _CreateSocket(self, ssl_params, ssl_verify_peer):
588 """Creates a TCP socket and initializes SSL if needed.
590 @type ssl_params: HttpSslParams
591 @param ssl_params: SSL key and certificate
592 @type ssl_verify_peer: bool
593 @param ssl_verify_peer: Whether to require client certificate
594 and compare it with our certificate
597 self._ssl_params = ssl_params
599 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
601 # Should we enable SSL?
602 self.using_ssl = ssl_params is not None
604 if not self.using_ssl:
607 self._ssl_key = ssl_params.GetKey()
608 self._ssl_cert = ssl_params.GetCertificate()
610 ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
611 ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2)
613 ctx.use_privatekey(self._ssl_key)
614 ctx.use_certificate(self._ssl_cert)
615 ctx.check_privatekey()
618 ctx.set_verify(OpenSSL.SSL.VERIFY_PEER |
619 OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
620 self._SSLVerifyCallback)
622 return OpenSSL.SSL.Connection(ctx, sock)
624 def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):
625 """Verify the certificate provided by the peer
627 We only compare fingerprints. The client must use the same certificate as
631 assert self._ssl_params, "SSL not initialized"
633 return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and
634 self._ssl_cert.digest("md5") == cert.digest("md5"))
637 class HttpMessage(object):
638 """Data structure for HTTP message.
642 self.start_line = None
645 self.decoded_body = None
648 class HttpClientToServerStartLine(object):
649 """Data structure for HTTP request start line.
652 def __init__(self, method, path, version):
655 self.version = version
658 return "%s %s %s" % (self.method, self.path, self.version)
661 class HttpServerToClientStartLine(object):
662 """Data structure for HTTP response start line.
665 def __init__(self, version, code, reason):
666 self.version = version
671 return "%s %s %s" % (self.version, self.code, self.reason)
674 class HttpMessageWriter(object):
675 """Writes an HTTP message to a socket.
678 def __init__(self, sock, msg, write_timeout):
679 """Initializes this class and writes an HTTP message to a socket.
682 @param sock: Socket to be written to
683 @type msg: http.HttpMessage
684 @param msg: HTTP message to be written
685 @type write_timeout: float
686 @param write_timeout: Write timeout for socket
691 self._PrepareMessage()
693 buf = self._FormatMessage()
698 # Send only SOCK_BUF_SIZE bytes at a time
699 data = buf[pos:(pos + SOCK_BUF_SIZE)]
701 sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout)
706 assert pos == end, "Message wasn't sent completely"
708 def _PrepareMessage(self):
709 """Prepares the HTTP message by setting mandatory headers.
712 # RFC2616, section 4.3: "The presence of a message-body in a request is
713 # signaled by the inclusion of a Content-Length or Transfer-Encoding header
714 # field in the request's message-headers."
716 self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body)
718 def _FormatMessage(self):
719 """Serializes the HTTP message into a string.
725 buf.write(str(self._msg.start_line))
729 if self._msg.start_line.version != HTTP_0_9:
730 for name, value in self._msg.headers.iteritems():
731 buf.write("%s: %s\r\n" % (name, value))
735 # Add message body if needed
736 if self.HasMessageBody():
737 buf.write(self._msg.body)
740 logging.warning("Ignoring message body")
742 return buf.getvalue()
744 def HasMessageBody(self):
745 """Checks whether the HTTP message contains a body.
747 Can be overriden by subclasses.
750 return bool(self._msg.body)
753 class HttpMessageReader(object):
754 """Reads HTTP message from socket.
758 START_LINE_LENGTH_MAX = None
759 HEADER_LENGTH_MAX = None
761 # Parser state machine
762 PS_START_LINE = "start-line"
763 PS_HEADERS = "headers"
764 PS_BODY = "entity-body"
765 PS_COMPLETE = "complete"
767 def __init__(self, sock, msg, read_timeout):
768 """Reads an HTTP message from a socket.
771 @param sock: Socket to be read from
772 @type msg: http.HttpMessage
773 @param msg: Object for the read message
774 @type read_timeout: float
775 @param read_timeout: Read timeout for socket
781 self.start_line_buffer = None
782 self.header_buffer = StringIO()
783 self.body_buffer = StringIO()
784 self.parser_status = self.PS_START_LINE
785 self.content_length = None
786 self.peer_will_close = None
790 while self.parser_status != self.PS_COMPLETE:
791 # TODO: Don't read more than necessary (Content-Length), otherwise
792 # data might be lost and/or an error could occur
793 data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout)
800 # Do some parsing and error checking while more data arrives
801 buf = self._ContinueParsing(buf, eof)
803 # Must be done only after the buffer has been evaluated
804 # TODO: Connection-length < len(data read) and connection closed
806 self.parser_status in (self.PS_START_LINE,
808 raise HttpError("Connection closed prematurely")
811 buf = self._ContinueParsing(buf, True)
813 assert self.parser_status == self.PS_COMPLETE
814 assert not buf, "Parser didn't read full response"
816 msg.body = self.body_buffer.getvalue()
818 # TODO: Content-type, error handling
820 msg.decoded_body = HttpJsonConverter().Decode(msg.body)
822 msg.decoded_body = None
825 logging.debug("Message body: %s", msg.decoded_body)
827 def _ContinueParsing(self, buf, eof):
828 """Main function for HTTP message state machine.
831 @param buf: Receive buffer
833 @param eof: Whether we've reached EOF on the socket
835 @return: Updated receive buffer
838 # TODO: Use offset instead of slicing when possible
839 if self.parser_status == self.PS_START_LINE:
842 idx = buf.find("\r\n")
844 # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD
845 # ignore any empty line(s) received where a Request-Line is expected.
846 # In other words, if the server is reading the protocol stream at the
847 # beginning of a message and receives a CRLF first, it should ignore
850 # TODO: Limit number of CRLFs/empty lines for safety?
855 self.start_line_buffer = buf[:idx]
857 self._CheckStartLineLength(len(self.start_line_buffer))
859 # Remove status line, including CRLF
862 self.msg.start_line = self.ParseStartLine(self.start_line_buffer)
864 self.parser_status = self.PS_HEADERS
866 # Check whether incoming data is getting too large, otherwise we just
867 # fill our read buffer.
868 self._CheckStartLineLength(len(buf))
872 # TODO: Handle messages without headers
873 if self.parser_status == self.PS_HEADERS:
874 # Wait for header end
875 idx = buf.find("\r\n\r\n")
877 self.header_buffer.write(buf[:idx + 2])
879 self._CheckHeaderLength(self.header_buffer.tell())
881 # Remove headers, including CRLF
886 self.parser_status = self.PS_BODY
888 # Check whether incoming data is getting too large, otherwise we just
889 # fill our read buffer.
890 self._CheckHeaderLength(len(buf))
892 if self.parser_status == self.PS_BODY:
893 # TODO: Implement max size for body_buffer
894 self.body_buffer.write(buf)
897 # Check whether we've read everything
899 # RFC2616, section 4.4: "When a message-body is included with a message,
900 # the transfer-length of that body is determined by one of the following
901 # [...] 5. By the server closing the connection. (Closing the connection
902 # cannot be used to indicate the end of a request body, since that would
903 # leave no possibility for the server to send back a response.)"
905 # TODO: Error when buffer length > Content-Length header
907 self.content_length is None or
908 (self.content_length is not None and
909 self.body_buffer.tell() >= self.content_length)):
910 self.parser_status = self.PS_COMPLETE
914 def _CheckStartLineLength(self, length):
915 """Limits the start line buffer size.
918 @param length: Buffer size
921 if (self.START_LINE_LENGTH_MAX is not None and
922 length > self.START_LINE_LENGTH_MAX):
923 raise HttpError("Start line longer than %d chars" %
924 self.START_LINE_LENGTH_MAX)
926 def _CheckHeaderLength(self, length):
927 """Limits the header buffer size.
930 @param length: Buffer size
933 if (self.HEADER_LENGTH_MAX is not None and
934 length > self.HEADER_LENGTH_MAX):
935 raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX)
937 def ParseStartLine(self, start_line):
938 """Parses the start line of a message.
940 Must be overriden by subclass.
942 @type start_line: string
943 @param start_line: Start line string
946 raise NotImplementedError()
948 def _WillPeerCloseConnection(self):
949 """Evaluate whether peer will close the connection.
952 @return: Whether peer will close the connection
955 # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option
956 # for the sender to signal that the connection will be closed after
957 # completion of the response. For example,
961 # in either the request or the response header fields indicates that the
962 # connection SHOULD NOT be considered `persistent' (section 8.1) after the
963 # current request/response is complete."
965 hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None)
967 hdr_connection = hdr_connection.lower()
969 # An HTTP/1.1 server is assumed to stay open unless explicitly closed.
970 if self.msg.start_line.version == HTTP_1_1:
971 return (hdr_connection and "close" in hdr_connection)
973 # Some HTTP/1.0 implementations have support for persistent connections,
974 # using rules different than HTTP/1.1.
976 # For older HTTP, Keep-Alive indicates persistent connection.
977 if self.msg.headers.get(HTTP_KEEP_ALIVE):
980 # At least Akamai returns a "Connection: Keep-Alive" header, which was
981 # supposed to be sent by the client.
982 if hdr_connection and "keep-alive" in hdr_connection:
987 def _ParseHeaders(self):
988 """Parses the headers.
990 This function also adjusts internal variables based on header values.
992 RFC2616, section 4.3: The presence of a message-body in a request is
993 signaled by the inclusion of a Content-Length or Transfer-Encoding header
994 field in the request's message-headers.
998 self.header_buffer.seek(0, 0)
999 self.msg.headers = mimetools.Message(self.header_buffer, 0)
1001 self.peer_will_close = self._WillPeerCloseConnection()
1003 # Do we have a Content-Length header?
1004 hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None)
1005 if hdr_content_length:
1007 self.content_length = int(hdr_content_length)
1009 self.content_length = None
1010 if self.content_length is not None and self.content_length < 0:
1011 self.content_length = None
1013 # if the connection remains open and a content-length was not provided,
1014 # then assume that the connection WILL close.
1015 if self.content_length is None:
1016 self.peer_will_close = True