rapi.client, http.client: Format url correctly when using IPv6

[ganeti-local] / lib / http / __init__.py

diff --git a/lib/http/__init__.py b/lib/http/__init__.py
index 26345fd..203e0d5 100644 (file)
--- a/lib/http/__init__.py
+++ b/lib/http/__init__.py
@@ -1,7 +1,7 @@
 #
 #
 
 #
 #
 

-# Copyright (C) 2007, 2008 Google Inc.
+# Copyright (C) 2007, 2008, 2010 Google Inc.

 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

@@ -32,12 +32,9 @@ import errno
 from cStringIO import StringIO
 
 from ganeti import constants
 from cStringIO import StringIO
 
 from ganeti import constants

-from ganeti import serializer

 from ganeti import utils
 
 
 from ganeti import utils
 
 

-HTTP_CLIENT_THREADS = 10
-

 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION
 
 HTTP_OK = 200
 HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION
 
 HTTP_OK = 200

@@ -52,6 +49,7 @@ HTTP_GET = "GET"
 HTTP_HEAD = "HEAD"
 HTTP_POST = "POST"
 HTTP_PUT = "PUT"
 HTTP_HEAD = "HEAD"
 HTTP_POST = "POST"
 HTTP_PUT = "PUT"

+HTTP_DELETE = "DELETE"

 
 HTTP_ETAG = "ETag"
 HTTP_HOST = "Host"
 
 HTTP_ETAG = "ETag"
 HTTP_HOST = "Host"

@@ -62,13 +60,24 @@ HTTP_CONTENT_TYPE = "Content-Type"
 HTTP_CONTENT_LENGTH = "Content-Length"
 HTTP_CONNECTION = "Connection"
 HTTP_KEEP_ALIVE = "Keep-Alive"
 HTTP_CONTENT_LENGTH = "Content-Length"
 HTTP_CONNECTION = "Connection"
 HTTP_KEEP_ALIVE = "Keep-Alive"

+HTTP_WWW_AUTHENTICATE = "WWW-Authenticate"
+HTTP_AUTHORIZATION = "Authorization"
+HTTP_AUTHENTICATION_INFO = "Authentication-Info"
+HTTP_ALLOW = "Allow"
+
+HTTP_APP_OCTET_STREAM = "application/octet-stream"
+HTTP_APP_JSON = "application/json"

 
 _SSL_UNEXPECTED_EOF = "Unexpected EOF"
 
 # Socket operations
 (SOCKOP_SEND,
  SOCKOP_RECV,
 
 _SSL_UNEXPECTED_EOF = "Unexpected EOF"
 
 # Socket operations
 (SOCKOP_SEND,
  SOCKOP_RECV,

- SOCKOP_SHUTDOWN) = range(3)
+ SOCKOP_SHUTDOWN,
+ SOCKOP_HANDSHAKE) = range(4)
+
+# send/receive quantum
+SOCK_BUF_SIZE = 32768

 
 
 class HttpError(Exception):
 
 
 class HttpError(Exception):

@@ -79,8 +88,17 @@ class HttpError(Exception):
   """
 
 
   """
 
 

-class _HttpClientError(Exception):
-  """Internal exception for HTTP client errors.
+class HttpConnectionClosed(Exception):
+  """Internal exception for a closed connection.
+
+  This should only be used for internal error reporting. Only use
+  it if there's no other way to report this condition.
+
+  """
+
+
+class HttpSessionHandshakeUnexpectedEOF(HttpError):
+  """Internal exception for errors during SSL handshake.

 
   This should only be used for internal error reporting.
 
 
   This should only be used for internal error reporting.
 

@@ -99,127 +117,238 @@ class HttpException(Exception):
   code = None
   message = None
 
   code = None
   message = None
 

-  def __init__(self, message=None):
+  def __init__(self, message=None, headers=None):

     Exception.__init__(self)
     Exception.__init__(self)

-    if message is not None:
-      self.message = message
+    self.message = message
+    self.headers = headers

 
 
 class HttpBadRequest(HttpException):
 
 
 class HttpBadRequest(HttpException):

+  """400 Bad Request
+
+  RFC2616, 10.4.1: The request could not be understood by the server
+  due to malformed syntax. The client SHOULD NOT repeat the request
+  without modifications.
+
+  """

   code = 400
 
 
   code = 400
 
 

+class HttpUnauthorized(HttpException):
+  """401 Unauthorized
+
+  RFC2616, section 10.4.2: The request requires user
+  authentication. The response MUST include a WWW-Authenticate header
+  field (section 14.47) containing a challenge applicable to the
+  requested resource.
+
+  """
+  code = 401
+
+

 class HttpForbidden(HttpException):
 class HttpForbidden(HttpException):

+  """403 Forbidden
+
+  RFC2616, 10.4.4: The server understood the request, but is refusing
+  to fulfill it.  Authorization will not help and the request SHOULD
+  NOT be repeated.
+
+  """

   code = 403
 
 
 class HttpNotFound(HttpException):
   code = 403
 
 
 class HttpNotFound(HttpException):

+  """404 Not Found
+
+  RFC2616, 10.4.5: The server has not found anything matching the
+  Request-URI.  No indication is given of whether the condition is
+  temporary or permanent.
+
+  """

   code = 404
 
 
   code = 404
 
 

+class HttpMethodNotAllowed(HttpException):
+  """405 Method Not Allowed
+
+  RFC2616, 10.4.6: The method specified in the Request-Line is not
+  allowed for the resource identified by the Request-URI. The response
+  MUST include an Allow header containing a list of valid methods for
+  the requested resource.
+
+  """
+  code = 405
+
+
+class HttpNotAcceptable(HttpException):
+  """406 Not Acceptable
+
+  RFC2616, 10.4.7: The resource identified by the request is only capable of
+  generating response entities which have content characteristics not
+  acceptable according to the accept headers sent in the request.
+
+  """
+  code = 406
+
+
+class HttpRequestTimeout(HttpException):
+  """408 Request Timeout
+
+  RFC2616, 10.4.9: The client did not produce a request within the
+  time that the server was prepared to wait. The client MAY repeat the
+  request without modifications at any later time.
+
+  """
+  code = 408
+
+
+class HttpConflict(HttpException):
+  """409 Conflict
+
+  RFC2616, 10.4.10: The request could not be completed due to a
+  conflict with the current state of the resource. This code is only
+  allowed in situations where it is expected that the user might be
+  able to resolve the conflict and resubmit the request.
+
+  """
+  code = 409
+
+

 class HttpGone(HttpException):
 class HttpGone(HttpException):

+  """410 Gone
+
+  RFC2616, 10.4.11: The requested resource is no longer available at
+  the server and no forwarding address is known. This condition is
+  expected to be considered permanent.
+
+  """

   code = 410
 
 
 class HttpLengthRequired(HttpException):
   code = 410
 
 
 class HttpLengthRequired(HttpException):

+  """411 Length Required
+
+  RFC2616, 10.4.12: The server refuses to accept the request without a
+  defined Content-Length. The client MAY repeat the request if it adds
+  a valid Content-Length header field containing the length of the
+  message-body in the request message.
+
+  """

   code = 411
 
 
   code = 411
 
 

-class HttpInternalError(HttpException):
+class HttpPreconditionFailed(HttpException):
+  """412 Precondition Failed
+
+  RFC2616, 10.4.13: The precondition given in one or more of the
+  request-header fields evaluated to false when it was tested on the
+  server.
+
+  """
+  code = 412
+
+
+class HttpUnsupportedMediaType(HttpException):
+  """415 Unsupported Media Type
+
+  RFC2616, 10.4.16: The server is refusing to service the request because the
+  entity of the request is in a format not supported by the requested resource
+  for the requested method.
+
+  """
+  code = 415
+
+
+class HttpInternalServerError(HttpException):
+  """500 Internal Server Error
+
+  RFC2616, 10.5.1: The server encountered an unexpected condition
+  which prevented it from fulfilling the request.
+
+  """

   code = 500
 
 
 class HttpNotImplemented(HttpException):
   code = 500
 
 
 class HttpNotImplemented(HttpException):

+  """501 Not Implemented
+
+  RFC2616, 10.5.2: The server does not support the functionality
+  required to fulfill the request.
+
+  """

   code = 501
 
 
   code = 501
 
 

+class HttpBadGateway(HttpException):
+  """502 Bad Gateway
+
+  RFC2616, 10.5.3: The server, while acting as a gateway or proxy,
+  received an invalid response from the upstream server it accessed in
+  attempting to fulfill the request.
+
+  """
+  code = 502
+
+

 class HttpServiceUnavailable(HttpException):
 class HttpServiceUnavailable(HttpException):

-  code = 503
+  """503 Service Unavailable

 
 

+  RFC2616, 10.5.4: The server is currently unable to handle the
+  request due to a temporary overloading or maintenance of the server.

 
 

-class HttpVersionNotSupported(HttpException):
-  code = 505
+  """
+  code = 503

 
 
 
 

-class HttpJsonConverter:
-  CONTENT_TYPE = "application/json"
+class HttpGatewayTimeout(HttpException):
+  """504 Gateway Timeout

 
 

-  def Encode(self, data):
-    return serializer.DumpJson(data)
+  RFC2616, 10.5.5: The server, while acting as a gateway or proxy, did
+  not receive a timely response from the upstream server specified by
+  the URI (e.g.  HTTP, FTP, LDAP) or some other auxiliary server
+  (e.g. DNS) it needed to access in attempting to complete the
+  request.

 
 

-  def Decode(self, data):
-    return serializer.LoadJson(data)
+  """
+  code = 504

 
 
 
 

-def WaitForSocketCondition(poller, sock, event, timeout):
-  """Waits for a condition to occur on the socket.
+class HttpVersionNotSupported(HttpException):
+  """505 HTTP Version Not Supported

 
 

-  @type poller: select.Poller
-  @param poller: Poller object as created by select.poll()
-  @type sock: socket
-  @param socket: Wait for events on this socket
-  @type event: int
-  @param event: ORed condition (see select module)
-  @type timeout: float or None
-  @param timeout: Timeout in seconds
-  @rtype: int or None
-  @return: None for timeout, otherwise occured conditions
+  RFC2616, 10.5.6: The server does not support, or refuses to support,
+  the HTTP protocol version that was used in the request message.

 
   """
 
   """

-  check = (event | select.POLLPRI |
-           select.POLLNVAL | select.POLLHUP | select.POLLERR)
+  code = 505

 
 

-  if timeout is not None:
-    # Poller object expects milliseconds
-    timeout *= 1000

 
 

-  poller.register(sock, event)
-  try:
-    while True:
-      # TODO: If the main thread receives a signal and we have no timeout, we
-      # could wait forever. This should check a global "quit" flag or
-      # something every so often.
-      io_events = poller.poll(timeout)
-      if not io_events:
-        # Timeout
-        return None
-      for (evfd, evcond) in io_events:
-        if evcond & check:
-          return evcond
-  finally:
-    poller.unregister(sock)
-
-
-def SocketOperation(poller, sock, op, arg1, timeout):
+def SocketOperation(sock, op, arg1, timeout):

   """Wrapper around socket functions.
 
   This function abstracts error handling for socket operations, especially
   for the complicated interaction with OpenSSL.
 
   """Wrapper around socket functions.
 
   This function abstracts error handling for socket operations, especially
   for the complicated interaction with OpenSSL.
 

-  @type poller: select.Poller
-  @param poller: Poller object as created by select.poll()

   @type sock: socket
   @type sock: socket

-  @param socket: Socket for the operation
+  @param sock: Socket for the operation

   @type op: int
   @param op: Operation to execute (SOCKOP_* constants)
   @type arg1: any
   @param arg1: Parameter for function (if needed)
   @type timeout: None or float
   @param timeout: Timeout in seconds or None
   @type op: int
   @param op: Operation to execute (SOCKOP_* constants)
   @type arg1: any
   @param arg1: Parameter for function (if needed)
   @type timeout: None or float
   @param timeout: Timeout in seconds or None

+  @return: Return value of socket function

 
   """
   # TODO: event_poll/event_check/override
 
   """
   # TODO: event_poll/event_check/override

-  if op == SOCKOP_SEND:
+  if op in (SOCKOP_SEND, SOCKOP_HANDSHAKE):

     event_poll = select.POLLOUT
     event_poll = select.POLLOUT

-    event_check = select.POLLOUT

 
   elif op == SOCKOP_RECV:
     event_poll = select.POLLIN
 
   elif op == SOCKOP_RECV:
     event_poll = select.POLLIN

-    event_check = select.POLLIN | select.POLLPRI

 
   elif op == SOCKOP_SHUTDOWN:
     event_poll = None
 
   elif op == SOCKOP_SHUTDOWN:
     event_poll = None

-    event_check = None

 
     # The timeout is only used when OpenSSL requests polling for a condition.
     # It is not advisable to have no timeout for shutdown.
 
     # The timeout is only used when OpenSSL requests polling for a condition.
     # It is not advisable to have no timeout for shutdown.

@@ -228,24 +357,29 @@ def SocketOperation(poller, sock, op, arg1, timeout):
   else:
     raise AssertionError("Invalid socket operation")
 
   else:
     raise AssertionError("Invalid socket operation")
 

+  # Handshake is only supported by SSL sockets
+  if (op == SOCKOP_HANDSHAKE and
+      not isinstance(sock, OpenSSL.SSL.ConnectionType)):
+    return
+

   # No override by default
   event_override = 0
 
   while True:
     # Poll only for certain operations and when asked for by an override
   # No override by default
   event_override = 0
 
   while True:
     # Poll only for certain operations and when asked for by an override

-    if event_override or op in (SOCKOP_SEND, SOCKOP_RECV):
+    if event_override or op in (SOCKOP_SEND, SOCKOP_RECV, SOCKOP_HANDSHAKE):

       if event_override:
         wait_for_event = event_override
       else:
         wait_for_event = event_poll
 
       if event_override:
         wait_for_event = event_override
       else:
         wait_for_event = event_poll
 

-      event = WaitForSocketCondition(poller, sock, wait_for_event, timeout)
+      event = utils.WaitForFdCondition(sock, wait_for_event, timeout)

       if event is None:
         raise HttpSocketTimeout()
 
       if event is None:
         raise HttpSocketTimeout()
 

-      if (op == SOCKOP_RECV and
-          event & (select.POLLNVAL | select.POLLHUP | select.POLLERR)):
-        return ""
+      if event & (select.POLLNVAL | select.POLLHUP | select.POLLERR):
+        # Let the socket functions handle these
+        break

 
       if not event & wait_for_event:
         continue
 
       if not event & wait_for_event:
         continue

@@ -268,6 +402,9 @@ def SocketOperation(poller, sock, op, arg1, timeout):
           else:
             return sock.shutdown(arg1)
 
           else:
             return sock.shutdown(arg1)
 

+        elif op == SOCKOP_HANDSHAKE:
+          return sock.do_handshake()
+

       except OpenSSL.SSL.WantWriteError:
         # OpenSSL wants to write, poll for POLLOUT
         event_override = select.POLLOUT
       except OpenSSL.SSL.WantWriteError:
         # OpenSSL wants to write, poll for POLLOUT
         event_override = select.POLLOUT

@@ -281,6 +418,21 @@ def SocketOperation(poller, sock, op, arg1, timeout):
       except OpenSSL.SSL.WantX509LookupError:
         continue
 
       except OpenSSL.SSL.WantX509LookupError:
         continue
 

+      except OpenSSL.SSL.ZeroReturnError, err:
+        # SSL Connection has been closed. In SSL 3.0 and TLS 1.0, this only
+        # occurs if a closure alert has occurred in the protocol, i.e. the
+        # connection has been closed cleanly. Note that this does not
+        # necessarily mean that the transport layer (e.g. a socket) has been
+        # closed.
+        if op == SOCKOP_SEND:
+          # Can happen during a renegotiation
+          raise HttpConnectionClosed(err.args)
+        elif op == SOCKOP_RECV:
+          return ""
+
+        # SSL_shutdown shouldn't return SSL_ERROR_ZERO_RETURN
+        raise socket.error(err.args)
+

       except OpenSSL.SSL.SysCallError, err:
         if op == SOCKOP_SEND:
           # arg1 is the data when writing
       except OpenSSL.SSL.SysCallError, err:
         if op == SOCKOP_SEND:
           # arg1 is the data when writing

@@ -289,9 +441,13 @@ def SocketOperation(poller, sock, op, arg1, timeout):
             # and can be ignored
             return 0
 
             # and can be ignored
             return 0
 

-        elif op == SOCKOP_RECV:
-          if err.args == (-1, _SSL_UNEXPECTED_EOF):
+        if err.args == (-1, _SSL_UNEXPECTED_EOF):
+          if op == SOCKOP_RECV:

             return ""
             return ""

+          elif op == SOCKOP_HANDSHAKE:
+            # Can happen if peer disconnects directly after the connection is
+            # opened.
+            raise HttpSessionHandshakeUnexpectedEOF(err.args)

 
         raise socket.error(err.args)
 
 
         raise socket.error(err.args)
 

@@ -306,19 +462,30 @@ def SocketOperation(poller, sock, op, arg1, timeout):
       raise
 
 
       raise
 
 

-def ShutdownConnection(poller, sock, close_timeout, write_timeout, msgreader,
-                       force):
+def ShutdownConnection(sock, close_timeout, write_timeout, msgreader, force):

   """Closes the connection.
 
   """Closes the connection.
 

-  """
-  poller = select.poll()
+  @type sock: socket
+  @param sock: Socket to be shut down
+  @type close_timeout: float
+  @param close_timeout: How long to wait for the peer to close
+      the connection
+  @type write_timeout: float
+  @param write_timeout: Write timeout for shutdown
+  @type msgreader: http.HttpMessageReader
+  @param msgreader: Request message reader, used to determine whether
+      peer should close connection
+  @type force: bool
+  @param force: Whether to forcibly close the connection without
+      waiting for peer

 
 

+  """

   #print msgreader.peer_will_close, force
   if msgreader and msgreader.peer_will_close and not force:
     # Wait for peer to close
     try:
       # Check whether it's actually closed
   #print msgreader.peer_will_close, force
   if msgreader and msgreader.peer_will_close and not force:
     # Wait for peer to close
     try:
       # Check whether it's actually closed

-      if not SocketOperation(poller, sock, SOCKOP_RECV, 1, close_timeout):
+      if not SocketOperation(sock, SOCKOP_RECV, 1, close_timeout):

         return
     except (socket.error, HttpError, HttpSocketTimeout):
       # Ignore errors at this stage
         return
     except (socket.error, HttpError, HttpSocketTimeout):
       # Ignore errors at this stage

@@ -326,12 +493,45 @@ def ShutdownConnection(poller, sock, close_timeout, write_timeout, msgreader,
 
   # Close the connection from our side
   try:
 
   # Close the connection from our side
   try:

-    SocketOperation(poller, sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR,
+    # We don't care about the return value, see NOTES in SSL_shutdown(3).
+    SocketOperation(sock, SOCKOP_SHUTDOWN, socket.SHUT_RDWR,

                     write_timeout)
   except HttpSocketTimeout:
     raise HttpError("Timeout while shutting down connection")
   except socket.error, err:
                     write_timeout)
   except HttpSocketTimeout:
     raise HttpError("Timeout while shutting down connection")
   except socket.error, err:

-    raise HttpError("Error while shutting down connection: %s" % err)
+    # Ignore ENOTCONN
+    if not (err.args and err.args[0] == errno.ENOTCONN):
+      raise HttpError("Error while shutting down connection: %s" % err)
+
+
+def Handshake(sock, write_timeout):
+  """Shakes peer's hands.
+
+  @type sock: socket
+  @param sock: Socket to be shut down
+  @type write_timeout: float
+  @param write_timeout: Write timeout for handshake
+
+  """
+  try:
+    return SocketOperation(sock, SOCKOP_HANDSHAKE, None, write_timeout)
+  except HttpSocketTimeout:
+    raise HttpError("Timeout during SSL handshake")
+  except socket.error, err:
+    raise HttpError("Error in SSL handshake: %s" % err)
+
+
+def InitSsl():
+  """Initializes the SSL infrastructure.
+
+  This function is idempotent.
+
+  """
+  if not OpenSSL.rand.status():
+    raise EnvironmentError("OpenSSL could not collect enough entropy"
+                           " for the PRNG")
+
+  # TODO: Maybe add some additional seeding for OpenSSL's PRNG

 
 
 class HttpSslParams(object):
 
 
 class HttpSslParams(object):

@@ -344,7 +544,8 @@ class HttpSslParams(object):
     @type ssl_key_path: string
     @param ssl_key_path: Path to file containing SSL key in PEM format
     @type ssl_cert_path: string
     @type ssl_key_path: string
     @param ssl_key_path: Path to file containing SSL key in PEM format
     @type ssl_cert_path: string

-    @param ssl_cert_path: Path to file containing SSL certificate in PEM format
+    @param ssl_cert_path: Path to file containing SSL certificate
+        in PEM format

 
     """
     self.ssl_key_pem = utils.ReadFile(ssl_key_path)
 
     """
     self.ssl_key_pem = utils.ReadFile(ssl_key_path)

@@ -359,34 +560,37 @@ class HttpSslParams(object):
                                            self.ssl_cert_pem)
 
 
                                            self.ssl_cert_pem)
 
 

-class HttpSocketBase(object):
+class HttpBase(object):

   """Base class for HTTP server and client.
 
   """
   def __init__(self):
   """Base class for HTTP server and client.
 
   """
   def __init__(self):

-    self._using_ssl = None
+    self.using_ssl = None

     self._ssl_params = None
     self._ssl_key = None
     self._ssl_cert = None
 
     self._ssl_params = None
     self._ssl_key = None
     self._ssl_cert = None
 

-  def _CreateSocket(self, ssl_params, ssl_verify_peer):
+  def _CreateSocket(self, ssl_params, ssl_verify_peer, family):

     """Creates a TCP socket and initializes SSL if needed.
 
     @type ssl_params: HttpSslParams
     @param ssl_params: SSL key and certificate
     @type ssl_verify_peer: bool
     """Creates a TCP socket and initializes SSL if needed.
 
     @type ssl_params: HttpSslParams
     @param ssl_params: SSL key and certificate
     @type ssl_verify_peer: bool

-    @param ssl_verify_peer: Whether to require client certificate and compare
-                            it with our certificate
+    @param ssl_verify_peer: Whether to require client certificate
+        and compare it with our certificate
+    @type family: int
+    @param family: socket.AF_INET | socket.AF_INET6

 
     """
 
     """

-    self._ssl_params = ssl_params
+    assert family in (socket.AF_INET, socket.AF_INET6)

 
 

-    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    self._ssl_params = ssl_params
+    sock = socket.socket(family, socket.SOCK_STREAM)

 
     # Should we enable SSL?
 
     # Should we enable SSL?

-    self._using_ssl = ssl_params is not None
+    self.using_ssl = ssl_params is not None

 
 

-    if not self._using_ssl:
+    if not self.using_ssl:

       return sock
 
     self._ssl_key = ssl_params.GetKey()
       return sock
 
     self._ssl_key = ssl_params.GetKey()

@@ -394,6 +598,7 @@ class HttpSocketBase(object):
 
     ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
     ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2)
 
     ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
     ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2)

+    ctx.set_cipher_list(constants.OPENSSL_CIPHERS)

 
     ctx.use_privatekey(self._ssl_key)
     ctx.use_certificate(self._ssl_cert)
 
     ctx.use_privatekey(self._ssl_key)
     ctx.use_certificate(self._ssl_cert)

@@ -413,6 +618,8 @@ class HttpSocketBase(object):
     we do on our side.
 
     """
     we do on our side.
 
     """

+    # some parameters are unused, but this is the API
+    # pylint: disable-msg=W0613

     assert self._ssl_params, "SSL not initialized"
 
     return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and
     assert self._ssl_params, "SSL not initialized"
 
     return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and

@@ -427,7 +634,6 @@ class HttpMessage(object):
     self.start_line = None
     self.headers = None
     self.body = None
     self.start_line = None
     self.headers = None
     self.body = None

-    self.decoded_body = None

 
 
 class HttpClientToServerStartLine(object):
 
 
 class HttpClientToServerStartLine(object):

@@ -461,24 +667,34 @@ class HttpMessageWriter(object):
 
   """
   def __init__(self, sock, msg, write_timeout):
 
   """
   def __init__(self, sock, msg, write_timeout):

+    """Initializes this class and writes an HTTP message to a socket.
+
+    @type sock: socket
+    @param sock: Socket to be written to
+    @type msg: http.HttpMessage
+    @param msg: HTTP message to be written
+    @type write_timeout: float
+    @param write_timeout: Write timeout for socket
+
+    """

     self._msg = msg
 
     self._PrepareMessage()
 
     buf = self._FormatMessage()
 
     self._msg = msg
 
     self._PrepareMessage()
 
     buf = self._FormatMessage()
 

-    poller = select.poll()
-    while buf:
-      # Send only 4 KB at a time
-      data = buf[:4096]
+    pos = 0
+    end = len(buf)
+    while pos < end:
+      # Send only SOCK_BUF_SIZE bytes at a time
+      data = buf[pos:(pos + SOCK_BUF_SIZE)]

 
 

-      sent = SocketOperation(poller, sock, SOCKOP_SEND, data,
-                             write_timeout)
+      sent = SocketOperation(sock, SOCKOP_SEND, data, write_timeout)

 
       # Remove sent bytes
 
       # Remove sent bytes

-      buf = buf[sent:]
+      pos += sent

 
 

-    assert not buf, "Message wasn't sent completely"
+    assert pos == end, "Message wasn't sent completely"

 
   def _PrepareMessage(self):
     """Prepares the HTTP message by setting mandatory headers.
 
   def _PrepareMessage(self):
     """Prepares the HTTP message by setting mandatory headers.

@@ -519,7 +735,7 @@ class HttpMessageWriter(object):
   def HasMessageBody(self):
     """Checks whether the HTTP message contains a body.
 
   def HasMessageBody(self):
     """Checks whether the HTTP message contains a body.
 

-    Can be overriden by subclasses.
+    Can be overridden by subclasses.

 
     """
     return bool(self._msg.body)
 
     """
     return bool(self._msg.body)

@@ -540,10 +756,19 @@ class HttpMessageReader(object):
   PS_COMPLETE = "complete"
 
   def __init__(self, sock, msg, read_timeout):
   PS_COMPLETE = "complete"
 
   def __init__(self, sock, msg, read_timeout):

+    """Reads an HTTP message from a socket.
+
+    @type sock: socket
+    @param sock: Socket to be read from
+    @type msg: http.HttpMessage
+    @param msg: Object for the read message
+    @type read_timeout: float
+    @param read_timeout: Read timeout for socket
+
+    """

     self.sock = sock
     self.msg = msg
 
     self.sock = sock
     self.msg = msg
 

-    self.poller = select.poll()

     self.start_line_buffer = None
     self.header_buffer = StringIO()
     self.body_buffer = StringIO()
     self.start_line_buffer = None
     self.header_buffer = StringIO()
     self.body_buffer = StringIO()

@@ -554,8 +779,9 @@ class HttpMessageReader(object):
     buf = ""
     eof = False
     while self.parser_status != self.PS_COMPLETE:
     buf = ""
     eof = False
     while self.parser_status != self.PS_COMPLETE:

-      data = SocketOperation(self.poller, sock, SOCKOP_RECV, 4096,
-                             read_timeout)
+      # TODO: Don't read more than necessary (Content-Length), otherwise
+      # data might be lost and/or an error could occur
+      data = SocketOperation(sock, SOCKOP_RECV, SOCK_BUF_SIZE, read_timeout)

 
       if data:
         buf += data
 
       if data:
         buf += data

@@ -566,7 +792,7 @@ class HttpMessageReader(object):
       buf = self._ContinueParsing(buf, eof)
 
       # Must be done only after the buffer has been evaluated
       buf = self._ContinueParsing(buf, eof)
 
       # Must be done only after the buffer has been evaluated

-      # TODO: Connection-length < len(data read) and connection closed
+      # TODO: Content-Length < len(data read) and connection closed

       if (eof and
           self.parser_status in (self.PS_START_LINE,
                                  self.PS_HEADERS)):
       if (eof and
           self.parser_status in (self.PS_START_LINE,
                                  self.PS_HEADERS)):

@@ -578,17 +804,9 @@ class HttpMessageReader(object):
     assert self.parser_status == self.PS_COMPLETE
     assert not buf, "Parser didn't read full response"
 
     assert self.parser_status == self.PS_COMPLETE
     assert not buf, "Parser didn't read full response"
 

+    # Body is complete

     msg.body = self.body_buffer.getvalue()
 
     msg.body = self.body_buffer.getvalue()
 

-    # TODO: Content-type, error handling
-    if msg.body:
-      msg.decoded_body = HttpJsonConverter().Decode(msg.body)
-    else:
-      msg.decoded_body = None
-
-    if msg.decoded_body:
-      logging.debug("Message body: %s", msg.decoded_body)
-

   def _ContinueParsing(self, buf, eof):
     """Main function for HTTP message state machine.
 
   def _ContinueParsing(self, buf, eof):
     """Main function for HTTP message state machine.
 

@@ -600,6 +818,7 @@ class HttpMessageReader(object):
     @return: Updated receive buffer
 
     """
     @return: Updated receive buffer
 
     """

+    # TODO: Use offset instead of slicing when possible

     if self.parser_status == self.PS_START_LINE:
       # Expect start line
       while True:
     if self.parser_status == self.PS_START_LINE:
       # Expect start line
       while True:

@@ -611,8 +830,8 @@ class HttpMessageReader(object):
         # beginning of a message and receives a CRLF first, it should ignore
         # the CRLF."
         if idx == 0:
         # beginning of a message and receives a CRLF first, it should ignore
         # the CRLF."
         if idx == 0:

-          # TODO: Limit number of CRLFs for safety?
-          buf = buf[:2]
+          # TODO: Limit number of CRLFs/empty lines for safety?
+          buf = buf[2:]

           continue
 
         if idx > 0:
           continue
 
         if idx > 0:

@@ -665,6 +884,8 @@ class HttpMessageReader(object):
       # [...] 5. By the server closing the connection. (Closing the connection
       # cannot be used to indicate the end of a request body, since that would
       # leave no possibility for the server to send back a response.)"
       # [...] 5. By the server closing the connection. (Closing the connection
       # cannot be used to indicate the end of a request body, since that would
       # leave no possibility for the server to send back a response.)"

+      #
+      # TODO: Error when buffer length > Content-Length header

       if (eof or
           self.content_length is None or
           (self.content_length is not None and
       if (eof or
           self.content_length is None or
           (self.content_length is not None and

@@ -699,7 +920,7 @@ class HttpMessageReader(object):
   def ParseStartLine(self, start_line):
     """Parses the start line of a message.
 
   def ParseStartLine(self, start_line):
     """Parses the start line of a message.
 

-    Must be overriden by subclass.
+    Must be overridden by subclass.

 
     @type start_line: string
     @param start_line: Start line string
 
     @type start_line: string
     @param start_line: Start line string

@@ -751,9 +972,9 @@ class HttpMessageReader(object):
 
     This function also adjusts internal variables based on header values.
 
 
     This function also adjusts internal variables based on header values.
 

-    RFC2616, section 4.3: "The presence of a message-body in a request is
+    RFC2616, section 4.3: The presence of a message-body in a request is

     signaled by the inclusion of a Content-Length or Transfer-Encoding header
     signaled by the inclusion of a Content-Length or Transfer-Encoding header

-    field in the request's message-headers."
+    field in the request's message-headers.

 
     """
     # Parse headers
 
     """
     # Parse headers

@@ -767,7 +988,7 @@ class HttpMessageReader(object):
     if hdr_content_length:
       try:
         self.content_length = int(hdr_content_length)
     if hdr_content_length:
       try:
         self.content_length = int(hdr_content_length)

-      except ValueError:
+      except (TypeError, ValueError):

         self.content_length = None
       if self.content_length is not None and self.content_length < 0:
         self.content_length = None
         self.content_length = None
       if self.content_length is not None and self.content_length < 0:
         self.content_length = None