4 # Copyright (C) 2007, 2008 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
32 from cStringIO import StringIO
34 from ganeti import constants
35 from ganeti import utils
38 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION
42 HTTP_NOT_MODIFIED = 304
52 HTTP_DELETE = "DELETE"
56 HTTP_SERVER = "Server"
58 HTTP_USER_AGENT = "User-Agent"
59 HTTP_CONTENT_TYPE = "Content-Type"
60 HTTP_CONTENT_LENGTH = "Content-Length"
61 HTTP_CONNECTION = "Connection"
62 HTTP_KEEP_ALIVE = "Keep-Alive"
63 HTTP_WWW_AUTHENTICATE = "WWW-Authenticate"
64 HTTP_AUTHORIZATION = "Authorization"
65 HTTP_AUTHENTICATION_INFO = "Authentication-Info"
68 HTTP_APP_OCTET_STREAM = "application/octet-stream"
69 HTTP_APP_JSON = "application/json"
71 _SSL_UNEXPECTED_EOF = "Unexpected EOF"
77 SOCKOP_HANDSHAKE) = range(4)
79 # send/receive quantum
83 class HttpError(Exception):
84 """Internal exception for HTTP errors.
86 This should only be used for internal error reporting.
91 class HttpConnectionClosed(Exception):
92 """Internal exception for a closed connection.
94 This should only be used for internal error reporting. Only use
95 it if there's no other way to report this condition.
100 class HttpSessionHandshakeUnexpectedEOF(HttpError):
101 """Internal exception for errors during SSL handshake.
103 This should only be used for internal error reporting.
108 class HttpSocketTimeout(Exception):
109 """Internal exception for socket timeouts.
111 This should only be used for internal error reporting.
116 class HttpException(Exception):
120 def __init__(self, message=None, headers=None):
121 Exception.__init__(self)
122 self.message = message
123 self.headers = headers
126 class HttpBadRequest(HttpException):
129 RFC2616, 10.4.1: The request could not be understood by the server
130 due to malformed syntax. The client SHOULD NOT repeat the request
131 without modifications.
137 class HttpUnauthorized(HttpException):
140 RFC2616, section 10.4.2: The request requires user
141 authentication. The response MUST include a WWW-Authenticate header
142 field (section 14.47) containing a challenge applicable to the
149 class HttpForbidden(HttpException):
152 RFC2616, 10.4.4: The server understood the request, but is refusing
153 to fulfill it. Authorization will not help and the request SHOULD
160 class HttpNotFound(HttpException):
163 RFC2616, 10.4.5: The server has not found anything matching the
164 Request-URI. No indication is given of whether the condition is
165 temporary or permanent.
171 class HttpMethodNotAllowed(HttpException):
172 """405 Method Not Allowed
174 RFC2616, 10.4.6: The method specified in the Request-Line is not
175 allowed for the resource identified by the Request-URI. The response
176 MUST include an Allow header containing a list of valid methods for
177 the requested resource.
183 class HttpNotAcceptable(HttpException):
184 """406 Not Acceptable
186 RFC2616, 10.4.7: The resource identified by the request is only capable of
187 generating response entities which have content characteristics not
188 acceptable according to the accept headers sent in the request.
194 class HttpRequestTimeout(HttpException):
195 """408 Request Timeout
197 RFC2616, 10.4.9: The client did not produce a request within the
198 time that the server was prepared to wait. The client MAY repeat the
199 request without modifications at any later time.
205 class HttpConflict(HttpException):
208 RFC2616, 10.4.10: The request could not be completed due to a
209 conflict with the current state of the resource. This code is only
210 allowed in situations where it is expected that the user might be
211 able to resolve the conflict and resubmit the request.
217 class HttpGone(HttpException):
220 RFC2616, 10.4.11: The requested resource is no longer available at
221 the server and no forwarding address is known. This condition is
222 expected to be considered permanent.
228 class HttpLengthRequired(HttpException):
229 """411 Length Required
231 RFC2616, 10.4.12: The server refuses to accept the request without a
232 defined Content-Length. The client MAY repeat the request if it adds
233 a valid Content-Length header field containing the length of the
234 message-body in the request message.
240 class HttpPreconditionFailed(HttpException):
241 """412 Precondition Failed
243 RFC2616, 10.4.13: The precondition given in one or more of the
244 request-header fields evaluated to false when it was tested on the
251 class HttpUnsupportedMediaType(HttpException):
252 """415 Unsupported Media Type
254 RFC2616, 10.4.16: The server is refusing to service the request because the
255 entity of the request is in a format not supported by the requested resource
256 for the requested method.
262 class HttpInternalServerError(HttpException):
263 """500 Internal Server Error
265 RFC2616, 10.5.1: The server encountered an unexpected condition
266 which prevented it from fulfilling the request.
272 class HttpNotImplemented(HttpException):
273 """501 Not Implemented
275 RFC2616, 10.5.2: The server does not support the functionality
276 required to fulfill the request.
282 class HttpBadGateway(HttpException):
285 RFC2616, 10.5.3: The server, while acting as a gateway or proxy,
286 received an invalid response from the upstream server it accessed in
287 attempting to fulfill the request.
293 class HttpServiceUnavailable(HttpException):
294 """503 Service Unavailable
296 RFC2616, 10.5.4: The server is currently unable to handle the
297 request due to a temporary overloading or maintenance of the server.
303 class HttpGatewayTimeout(HttpException):
304 """504 Gateway Timeout
306 RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did
307 not receive a timely response from the upstream server specified by
308 the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server
309 (e.g. DNS) it needed to access in attempting to complete the
316 class HttpVersionNotSupported(HttpException):
317 """505 HTTP Version Not Supported
319 RFC2616, 10.5.6: The server does not support, or refuses to support,
320 the HTTP protocol version that was used in the request message.
326 def SocketOperation(sock, op, arg1, timeout):
327 """Wrapper around socket functions.
329 This function abstracts error handling for socket operations, especially
330 for the complicated interaction with OpenSSL.
333 @param sock: Socket for the operation
335 @param op: Operation to execute (SOCKOP_* constants)
337 @param arg1: Parameter for function (if needed)
338 @type timeout: None or float
339 @param timeout: Timeout in seconds or None
340 @return: Return value of socket function
343 # TODO: event_poll/event_check/override
344 if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE):
345 event_poll = select.POLLOUT
347 elif op == SOCKOP_RECV:
348 event_poll = select.POLLIN
350 elif op == SOCKOP_SHUTDOWN:
353 # The timeout is only used when OpenSSL requests polling for a condition.
354 # It is not advisable to have no timeout for shutdown.
358 raise AssertionError("Invalid socket operation")
360 # Handshake is only supported by SSL sockets
361 if (op == SOCKOP_HANDSHAKE and
362 not isinstance(sock, OpenSSL.SSL.ConnectionType)):
365 # No override by default
369 # Poll only for certain operations and when asked for by an override
370 if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE):
372 wait_for_event = event_override
374 wait_for_event = event_poll
376 event = utils.WaitForFdCondition(sock, wait_for_event, timeout)
378 raise HttpSocketTimeout()
380 if event & (select.POLLNVAL | select.POLLHUP | select.POLLERR):
381 # Let the socket functions handle these
384 if not event & wait_for_event:
392 if op == SOCKOP_SEND:
393 return sock.send(arg1)
395 elif op == SOCKOP_RECV:
396 return sock.recv(arg1)
398 elif op == SOCKOP_SHUTDOWN:
399 if isinstance(sock, OpenSSL.SSL.ConnectionType):
400 # PyOpenSSL's shutdown() doesn't take arguments
401 return sock.shutdown()
403 return sock.shutdown(arg1)
405 elif op == SOCKOP_HANDSHAKE:
406 return sock.do_handshake()
408 except OpenSSL.SSL.WantWriteError:
409 # OpenSSL wants to write, poll for POLLOUT
410 event_override = select.POLLOUT
413 except OpenSSL.SSL.WantReadError:
414 # OpenSSL wants to read, poll for POLLIN
415 event_override = select.POLLIN | select.POLLPRI
418 except OpenSSL.SSL.WantX509LookupError:
421 except OpenSSL.SSL.ZeroReturnError, err:
422 # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only
423 # occurs if a closure alert has occurred in the protocol, i.e. the
424 # connection has been closed cleanly. Note that this does not
425 # necessarily mean that the transport layer (e.g. a socket) has been
427 if op == SOCKOP_SEND:
428 # Can happen during a renegotiation
429 raise HttpConnectionClosed(err.args)
430 elif op == SOCKOP_RECV:
433 # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN
434 raise socket.error(err.args)
436 except OpenSSL.SSL.SysCallError, err:
437 if op == SOCKOP_SEND:
438 # arg1 is the data when writing
439 if err.args and err.args[0] == -1 and arg1 == "":
440 # errors when writing empty strings are expected
444 if err.args == (-1, _SSL_UNEXPECTED_EOF):
445 if op == SOCKOP_RECV:
447 elif op == SOCKOP_HANDSHAKE:
448 # Can happen if peer disconnects directly after the connection is
450 raise HttpSessionHandshakeUnexpectedEOF(err.args)
452 raise socket.error(err.args)
454 except OpenSSL.SSL.Error, err:
455 raise socket.error(err.args)
457 except socket.error, err:
458 if err.args and err.args[0] == errno.EAGAIN:
465 def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):
466 """Closes the connection.
469 @param sock: Socket to be shut down
470 @type close_timeout: float
471 @param close_timeout: How long to wait for the peer to close
473 @type write_timeout: float
474 @param write_timeout: Write timeout for shutdown
475 @type msgreader: http.HttpMessageReader
476 @param msgreader: Request message reader, used to determine whether
477 peer should close connection
479 @param force: Whether to forcibly close the connection without
483 #print msgreader.peer_will_close, force
484 if msgreader and msgreader.peer_will_close and not force:
485 # Wait for peer to close
487 # Check whether it's actually closed
488 if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout):
490 except (socket.error, HttpError, HttpSocketTimeout):
491 # Ignore errors at this stage
494 # Close the connection from our side
496 # We don't care about the return value, see NOTES in SSL_shutdown(3).
497 SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR,
499 except HttpSocketTimeout:
500 raise HttpError("Timeout while shutting down connection")
501 except socket.error, err:
503 if not (err.args and err.args[0] == errno.ENOTCONN):
504 raise HttpError("Error while shutting down connection: %s" % err)
507 def Handshake(sock, write_timeout):
508 """Shakes peer's hands.
511 @param sock: Socket to be shut down
512 @type write_timeout: float
513 @param write_timeout: Write timeout for handshake
517 return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout)
518 except HttpSocketTimeout:
519 raise HttpError("Timeout during SSL handshake")
520 except socket.error, err:
521 raise HttpError("Error in SSL handshake: %s" % err)
525 """Initializes the SSL infrastructure.
527 This function is idempotent.
530 if not OpenSSL.rand.status():
531 raise EnvironmentError("OpenSSL could not collect enough entropy"
534 # TODO: Maybe add some additional seeding for OpenSSL's PRNG
537 class HttpSslParams(object):
538 """Data class for SSL key and certificate.
541 def __init__(self, ssl_key_path, ssl_cert_path):
542 """Initializes this class.
544 @type ssl_key_path: string
545 @param ssl_key_path: Path to file containing SSL key in PEM format
546 @type ssl_cert_path: string
547 @param ssl_cert_path: Path to file containing SSL certificate
551 self.ssl_key_pem = utils.ReadFile(ssl_key_path)
552 self.ssl_cert_pem = utils.ReadFile(ssl_cert_path)
555 return OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM,
558 def GetCertificate(self):
559 return OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
563 class HttpBase(object):
564 """Base class for HTTP server and client.
568 self.using_ssl = None
569 self._ssl_params = None
571 self._ssl_cert = None
573 def _CreateSocket(self, ssl_params, ssl_verify_peer):
574 """Creates a TCP socket and initializes SSL if needed.
576 @type ssl_params: HttpSslParams
577 @param ssl_params: SSL key and certificate
578 @type ssl_verify_peer: bool
579 @param ssl_verify_peer: Whether to require client certificate
580 and compare it with our certificate
583 self._ssl_params = ssl_params
585 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
587 # Should we enable SSL?
588 self.using_ssl = ssl_params is not None
590 if not self.using_ssl:
593 self._ssl_key = ssl_params.GetKey()
594 self._ssl_cert = ssl_params.GetCertificate()
596 ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
597 ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2)
599 ciphers = self.GetSslCiphers()
600 logging.debug("Setting SSL cipher string %s", ciphers)
601 ctx.set_cipher_list(ciphers)
603 ctx.use_privatekey(self._ssl_key)
604 ctx.use_certificate(self._ssl_cert)
605 ctx.check_privatekey()
608 ctx.set_verify(OpenSSL.SSL.VERIFY_PEER |
609 OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
610 self._SSLVerifyCallback)
612 return OpenSSL.SSL.Connection(ctx, sock)
614 def GetSslCiphers(self): # pylint: disable-msg=R0201
615 """Returns the ciphers string for SSL.
618 return constants.OPENSSL_CIPHERS
620 def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):
621 """Verify the certificate provided by the peer
623 We only compare fingerprints. The client must use the same certificate as
627 # some parameters are unused, but this is the API
628 # pylint: disable-msg=W0613
629 assert self._ssl_params, "SSL not initialized"
631 return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and
632 self._ssl_cert.digest("md5") == cert.digest("md5"))
635 class HttpMessage(object):
636 """Data structure for HTTP message.
640 self.start_line = None
645 class HttpClientToServerStartLine(object):
646 """Data structure for HTTP request start line.
649 def __init__(self, method, path, version):
652 self.version = version
655 return "%s %s %s" % (self.method, self.path, self.version)
658 class HttpServerToClientStartLine(object):
659 """Data structure for HTTP response start line.
662 def __init__(self, version, code, reason):
663 self.version = version
668 return "%s %s %s" % (self.version, self.code, self.reason)
671 class HttpMessageWriter(object):
672 """Writes an HTTP message to a socket.
675 def __init__(self, sock, msg, write_timeout):
676 """Initializes this class and writes an HTTP message to a socket.
679 @param sock: Socket to be written to
680 @type msg: http.HttpMessage
681 @param msg: HTTP message to be written
682 @type write_timeout: float
683 @param write_timeout: Write timeout for socket
688 self._PrepareMessage()
690 buf = self._FormatMessage()
695 # Send only SOCK_BUF_SIZE bytes at a time
696 data = buf[pos:(pos + SOCK_BUF_SIZE)]
698 sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout)
703 assert pos == end, "Message wasn't sent completely"
705 def _PrepareMessage(self):
706 """Prepares the HTTP message by setting mandatory headers.
709 # RFC2616, section 4.3: "The presence of a message-body in a request is
710 # signaled by the inclusion of a Content-Length or Transfer-Encoding header
711 # field in the request's message-headers."
713 self._msg.headers[HTTP_CONTENT_LENGTH] = len(self._msg.body)
715 def _FormatMessage(self):
716 """Serializes the HTTP message into a string.
722 buf.write(str(self._msg.start_line))
726 if self._msg.start_line.version != HTTP_0_9:
727 for name, value in self._msg.headers.iteritems():
728 buf.write("%s: %s\r\n" % (name, value))
732 # Add message body if needed
733 if self.HasMessageBody():
734 buf.write(self._msg.body)
737 logging.warning("Ignoring message body")
739 return buf.getvalue()
741 def HasMessageBody(self):
742 """Checks whether the HTTP message contains a body.
744 Can be overridden by subclasses.
747 return bool(self._msg.body)
750 class HttpMessageReader(object):
751 """Reads HTTP message from socket.
755 START_LINE_LENGTH_MAX = None
756 HEADER_LENGTH_MAX = None
758 # Parser state machine
759 PS_START_LINE = "start-line"
760 PS_HEADERS = "headers"
761 PS_BODY = "entity-body"
762 PS_COMPLETE = "complete"
764 def __init__(self, sock, msg, read_timeout):
765 """Reads an HTTP message from a socket.
768 @param sock: Socket to be read from
769 @type msg: http.HttpMessage
770 @param msg: Object for the read message
771 @type read_timeout: float
772 @param read_timeout: Read timeout for socket
778 self.start_line_buffer = None
779 self.header_buffer = StringIO()
780 self.body_buffer = StringIO()
781 self.parser_status = self.PS_START_LINE
782 self.content_length = None
783 self.peer_will_close = None
787 while self.parser_status != self.PS_COMPLETE:
788 # TODO: Don't read more than necessary (Content-Length), otherwise
789 # data might be lost and/or an error could occur
790 data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout)
797 # Do some parsing and error checking while more data arrives
798 buf = self._ContinueParsing(buf, eof)
800 # Must be done only after the buffer has been evaluated
801 # TODO: Content-Length < len(data read) and connection closed
803 self.parser_status in (self.PS_START_LINE,
805 raise HttpError("Connection closed prematurely")
808 buf = self._ContinueParsing(buf, True)
810 assert self.parser_status == self.PS_COMPLETE
811 assert not buf, "Parser didn't read full response"
814 msg.body = self.body_buffer.getvalue()
816 def _ContinueParsing(self, buf, eof):
817 """Main function for HTTP message state machine.
820 @param buf: Receive buffer
822 @param eof: Whether we've reached EOF on the socket
824 @return: Updated receive buffer
827 # TODO: Use offset instead of slicing when possible
828 if self.parser_status == self.PS_START_LINE:
831 idx = buf.find("\r\n")
833 # RFC2616, section 4.1: "In the interest of robustness, servers SHOULD
834 # ignore any empty line(s) received where a Request-Line is expected.
835 # In other words, if the server is reading the protocol stream at the
836 # beginning of a message and receives a CRLF first, it should ignore
839 # TODO: Limit number of CRLFs/empty lines for safety?
844 self.start_line_buffer = buf[:idx]
846 self._CheckStartLineLength(len(self.start_line_buffer))
848 # Remove status line, including CRLF
851 self.msg.start_line = self.ParseStartLine(self.start_line_buffer)
853 self.parser_status = self.PS_HEADERS
855 # Check whether incoming data is getting too large, otherwise we just
856 # fill our read buffer.
857 self._CheckStartLineLength(len(buf))
861 # TODO: Handle messages without headers
862 if self.parser_status == self.PS_HEADERS:
863 # Wait for header end
864 idx = buf.find("\r\n\r\n")
866 self.header_buffer.write(buf[:idx + 2])
868 self._CheckHeaderLength(self.header_buffer.tell())
870 # Remove headers, including CRLF
875 self.parser_status = self.PS_BODY
877 # Check whether incoming data is getting too large, otherwise we just
878 # fill our read buffer.
879 self._CheckHeaderLength(len(buf))
881 if self.parser_status == self.PS_BODY:
882 # TODO: Implement max size for body_buffer
883 self.body_buffer.write(buf)
886 # Check whether we've read everything
888 # RFC2616, section 4.4: "When a message-body is included with a message,
889 # the transfer-length of that body is determined by one of the following
890 # [...] 5. By the server closing the connection. (Closing the connection
891 # cannot be used to indicate the end of a request body, since that would
892 # leave no possibility for the server to send back a response.)"
894 # TODO: Error when buffer length > Content-Length header
896 self.content_length is None or
897 (self.content_length is not None and
898 self.body_buffer.tell() >= self.content_length)):
899 self.parser_status = self.PS_COMPLETE
903 def _CheckStartLineLength(self, length):
904 """Limits the start line buffer size.
907 @param length: Buffer size
910 if (self.START_LINE_LENGTH_MAX is not None and
911 length > self.START_LINE_LENGTH_MAX):
912 raise HttpError("Start line longer than %d chars" %
913 self.START_LINE_LENGTH_MAX)
915 def _CheckHeaderLength(self, length):
916 """Limits the header buffer size.
919 @param length: Buffer size
922 if (self.HEADER_LENGTH_MAX is not None and
923 length > self.HEADER_LENGTH_MAX):
924 raise HttpError("Headers longer than %d chars" % self.HEADER_LENGTH_MAX)
926 def ParseStartLine(self, start_line):
927 """Parses the start line of a message.
929 Must be overridden by subclass.
931 @type start_line: string
932 @param start_line: Start line string
935 raise NotImplementedError()
937 def _WillPeerCloseConnection(self):
938 """Evaluate whether peer will close the connection.
941 @return: Whether peer will close the connection
944 # RFC2616, section 14.10: "HTTP/1.1 defines the "close" connection option
945 # for the sender to signal that the connection will be closed after
946 # completion of the response. For example,
950 # in either the request or the response header fields indicates that the
951 # connection SHOULD NOT be considered `persistent' (section 8.1) after the
952 # current request/response is complete."
954 hdr_connection = self.msg.headers.get(HTTP_CONNECTION, None)
956 hdr_connection = hdr_connection.lower()
958 # An HTTP/1.1 server is assumed to stay open unless explicitly closed.
959 if self.msg.start_line.version == HTTP_1_1:
960 return (hdr_connection and "close" in hdr_connection)
962 # Some HTTP/1.0 implementations have support for persistent connections,
963 # using rules different than HTTP/1.1.
965 # For older HTTP, Keep-Alive indicates persistent connection.
966 if self.msg.headers.get(HTTP_KEEP_ALIVE):
969 # At least Akamai returns a "Connection: Keep-Alive" header, which was
970 # supposed to be sent by the client.
971 if hdr_connection and "keep-alive" in hdr_connection:
976 def _ParseHeaders(self):
977 """Parses the headers.
979 This function also adjusts internal variables based on header values.
981 RFC2616, section 4.3: The presence of a message-body in a request is
982 signaled by the inclusion of a Content-Length or Transfer-Encoding header
983 field in the request's message-headers.
987 self.header_buffer.seek(0, 0)
988 self.msg.headers = mimetools.Message(self.header_buffer, 0)
990 self.peer_will_close = self._WillPeerCloseConnection()
992 # Do we have a Content-Length header?
993 hdr_content_length = self.msg.headers.get(HTTP_CONTENT_LENGTH, None)
994 if hdr_content_length:
996 self.content_length = int(hdr_content_length)
997 except (TypeError, ValueError):
998 self.content_length = None
999 if self.content_length is not None and self.content_length < 0:
1000 self.content_length = None
1002 # if the connection remains open and a content-length was not provided,
1003 # then assume that the connection WILL close.
1004 if self.content_length is None:
1005 self.peer_will_close = True