/lib/http.py - Annotate - snf-ganeti - Greek Research and Technology Network's projects

| Branch: | Tag: | Revision:

root / lib / http.py @ cb4e8387

History | View | Annotate | Download (36.2 kB)

1	a43f68dc	Michael Hanselmann	#
2	a43f68dc	Michael Hanselmann	#
3	a43f68dc	Michael Hanselmann	# This program is free software; you can redistribute it and/or modify
4	a43f68dc	Michael Hanselmann	# it under the terms of the GNU General Public License as published by
5	a43f68dc	Michael Hanselmann	# the Free Software Foundation; either version 2 of the License, or
6	a43f68dc	Michael Hanselmann	# (at your option) any later version.
7	a43f68dc	Michael Hanselmann	#
8	a43f68dc	Michael Hanselmann	# This program is distributed in the hope that it will be useful, but
9	a43f68dc	Michael Hanselmann	# WITHOUT ANY WARRANTY; without even the implied warranty of
10	a43f68dc	Michael Hanselmann	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11	a43f68dc	Michael Hanselmann	# General Public License for more details.
12	a43f68dc	Michael Hanselmann	#
13	a43f68dc	Michael Hanselmann	# You should have received a copy of the GNU General Public License
14	a43f68dc	Michael Hanselmann	# along with this program; if not, write to the Free Software
15	a43f68dc	Michael Hanselmann	# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16	a43f68dc	Michael Hanselmann	# 02110-1301, USA.
17	a43f68dc	Michael Hanselmann
18	a43f68dc	Michael Hanselmann	"""HTTP server module.
19	a43f68dc	Michael Hanselmann
20	a43f68dc	Michael Hanselmann	"""
21	a43f68dc	Michael Hanselmann
22	a43f68dc	Michael Hanselmann	import BaseHTTPServer
23	42242313	Michael Hanselmann	import cgi
24	42242313	Michael Hanselmann	import logging
25	42242313	Michael Hanselmann	import mimetools
26	a43f68dc	Michael Hanselmann	import OpenSSL
27	42242313	Michael Hanselmann	import os
28	42242313	Michael Hanselmann	import select
29	42242313	Michael Hanselmann	import socket
30	42242313	Michael Hanselmann	import sys
31	a43f68dc	Michael Hanselmann	import time
32	42242313	Michael Hanselmann	import signal
33	263ab7cf	Iustin Pop	import logging
34	8a0b06d2	Michael Hanselmann	import errno
35	8a0b06d2	Michael Hanselmann
36	8a0b06d2	Michael Hanselmann	from cStringIO import StringIO
37	a43f68dc	Michael Hanselmann
38	42242313	Michael Hanselmann	from ganeti import constants
39	a43f68dc	Michael Hanselmann	from ganeti import serializer
40	8a0b06d2	Michael Hanselmann	from ganeti import workerpool
41	f2a6fc9e	Michael Hanselmann	from ganeti import utils
42	a43f68dc	Michael Hanselmann
43	a43f68dc	Michael Hanselmann
44	8a0b06d2	Michael Hanselmann	HTTP_CLIENT_THREADS = 10
45	8a0b06d2	Michael Hanselmann
46	8a9f9060	Michael Hanselmann	HTTP_GANETI_VERSION = "Ganeti %s" % constants.RELEASE_VERSION
47	8a9f9060	Michael Hanselmann
48	42242313	Michael Hanselmann	WEEKDAYNAME = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
49	42242313	Michael Hanselmann	MONTHNAME = [None,
50	42242313	Michael Hanselmann	'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
51	42242313	Michael Hanselmann	'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
52	42242313	Michael Hanselmann
53	42242313	Michael Hanselmann	# Default error message
54	42242313	Michael Hanselmann	DEFAULT_ERROR_CONTENT_TYPE = "text/html"
55	42242313	Michael Hanselmann	DEFAULT_ERROR_MESSAGE = """\
56	42242313	Michael Hanselmann	<head>
57	42242313	Michael Hanselmann	<title>Error response</title>
58	42242313	Michael Hanselmann	</head>
59	42242313	Michael Hanselmann	<body>
60	42242313	Michael Hanselmann	<h1>Error response</h1>
61	42242313	Michael Hanselmann	<p>Error code %(code)d.
62	42242313	Michael Hanselmann	<p>Message: %(message)s.
63	42242313	Michael Hanselmann	<p>Error code explanation: %(code)s = %(explain)s.
64	42242313	Michael Hanselmann	</body>
65	42242313	Michael Hanselmann	"""
66	42242313	Michael Hanselmann
67	42242313	Michael Hanselmann	HTTP_OK = 200
68	42242313	Michael Hanselmann	HTTP_NO_CONTENT = 204
69	42242313	Michael Hanselmann	HTTP_NOT_MODIFIED = 304
70	42242313	Michael Hanselmann
71	42242313	Michael Hanselmann	HTTP_0_9 = "HTTP/0.9"
72	42242313	Michael Hanselmann	HTTP_1_0 = "HTTP/1.0"
73	42242313	Michael Hanselmann	HTTP_1_1 = "HTTP/1.1"
74	42242313	Michael Hanselmann
75	42242313	Michael Hanselmann	HTTP_GET = "GET"
76	42242313	Michael Hanselmann	HTTP_HEAD = "HEAD"
77	8a9f9060	Michael Hanselmann	HTTP_POST = "POST"
78	8a9f9060	Michael Hanselmann	HTTP_PUT = "PUT"
79	8a9f9060	Michael Hanselmann
80	713faea6	Oleksiy Mishchenko	HTTP_ETAG = "ETag"
81	8a9f9060	Michael Hanselmann	HTTP_HOST = "Host"
82	8a9f9060	Michael Hanselmann	HTTP_SERVER = "Server"
83	8a9f9060	Michael Hanselmann	HTTP_DATE = "Date"
84	8a9f9060	Michael Hanselmann	HTTP_USER_AGENT = "User-Agent"
85	8a9f9060	Michael Hanselmann	HTTP_CONTENT_TYPE = "Content-Type"
86	8a9f9060	Michael Hanselmann	HTTP_CONTENT_LENGTH = "Content-Length"
87	8a9f9060	Michael Hanselmann	HTTP_CONNECTION = "Connection"
88	8a9f9060	Michael Hanselmann	HTTP_KEEP_ALIVE = "Keep-Alive"
89	42242313	Michael Hanselmann
90	d7bace1b	Michael Hanselmann	_SSL_UNEXPECTED_EOF = "Unexpected EOF"
91	d7bace1b	Michael Hanselmann
92	42242313	Michael Hanselmann
93	42242313	Michael Hanselmann	class SocketClosed(socket.error):
94	42242313	Michael Hanselmann	pass
95	42242313	Michael Hanselmann
96	42242313	Michael Hanselmann
97	8a0b06d2	Michael Hanselmann	class ResponseError(Exception):
98	8a0b06d2	Michael Hanselmann	pass
99	8a0b06d2	Michael Hanselmann
100	8a0b06d2	Michael Hanselmann
101	a43f68dc	Michael Hanselmann	class HTTPException(Exception):
102	a43f68dc	Michael Hanselmann	code = None
103	a43f68dc	Michael Hanselmann	message = None
104	a43f68dc	Michael Hanselmann
105	a43f68dc	Michael Hanselmann	def __init__(self, message=None):
106	42242313	Michael Hanselmann	Exception.__init__(self)
107	a43f68dc	Michael Hanselmann	if message is not None:
108	a43f68dc	Michael Hanselmann	self.message = message
109	a43f68dc	Michael Hanselmann
110	a43f68dc	Michael Hanselmann
111	a43f68dc	Michael Hanselmann	class HTTPBadRequest(HTTPException):
112	a43f68dc	Michael Hanselmann	code = 400
113	a43f68dc	Michael Hanselmann
114	a43f68dc	Michael Hanselmann
115	a43f68dc	Michael Hanselmann	class HTTPForbidden(HTTPException):
116	a43f68dc	Michael Hanselmann	code = 403
117	a43f68dc	Michael Hanselmann
118	a43f68dc	Michael Hanselmann
119	a43f68dc	Michael Hanselmann	class HTTPNotFound(HTTPException):
120	a43f68dc	Michael Hanselmann	code = 404
121	a43f68dc	Michael Hanselmann
122	a43f68dc	Michael Hanselmann
123	a43f68dc	Michael Hanselmann	class HTTPGone(HTTPException):
124	a43f68dc	Michael Hanselmann	code = 410
125	a43f68dc	Michael Hanselmann
126	a43f68dc	Michael Hanselmann
127	a43f68dc	Michael Hanselmann	class HTTPLengthRequired(HTTPException):
128	a43f68dc	Michael Hanselmann	code = 411
129	a43f68dc	Michael Hanselmann
130	a43f68dc	Michael Hanselmann
131	a43f68dc	Michael Hanselmann	class HTTPInternalError(HTTPException):
132	a43f68dc	Michael Hanselmann	code = 500
133	a43f68dc	Michael Hanselmann
134	a43f68dc	Michael Hanselmann
135	a43f68dc	Michael Hanselmann	class HTTPNotImplemented(HTTPException):
136	a43f68dc	Michael Hanselmann	code = 501
137	a43f68dc	Michael Hanselmann
138	a43f68dc	Michael Hanselmann
139	a43f68dc	Michael Hanselmann	class HTTPServiceUnavailable(HTTPException):
140	a43f68dc	Michael Hanselmann	code = 503
141	a43f68dc	Michael Hanselmann
142	a43f68dc	Michael Hanselmann
143	42242313	Michael Hanselmann	class HTTPVersionNotSupported(HTTPException):
144	42242313	Michael Hanselmann	code = 505
145	42242313	Michael Hanselmann
146	42242313	Michael Hanselmann
147	a43f68dc	Michael Hanselmann	class ApacheLogfile:
148	a43f68dc	Michael Hanselmann	"""Utility class to write HTTP server log files.
149	a43f68dc	Michael Hanselmann
150	a43f68dc	Michael Hanselmann	The written format is the "Common Log Format" as defined by Apache:
151	a43f68dc	Michael Hanselmann	http://httpd.apache.org/docs/2.2/mod/mod_log_config.html#examples
152	a43f68dc	Michael Hanselmann
153	a43f68dc	Michael Hanselmann	"""
154	a43f68dc	Michael Hanselmann	def __init__(self, fd):
155	a43f68dc	Michael Hanselmann	"""Constructor for ApacheLogfile class.
156	a43f68dc	Michael Hanselmann
157	a43f68dc	Michael Hanselmann	Args:
158	a43f68dc	Michael Hanselmann	- fd: Open file object
159	a43f68dc	Michael Hanselmann
160	a43f68dc	Michael Hanselmann	"""
161	a43f68dc	Michael Hanselmann	self._fd = fd
162	a43f68dc	Michael Hanselmann
163	a43f68dc	Michael Hanselmann	def LogRequest(self, request, format, *args):
164	a43f68dc	Michael Hanselmann	self._fd.write("%s %s %s [%s] %s\n" % (
165	a43f68dc	Michael Hanselmann	# Remote host address
166	a43f68dc	Michael Hanselmann	request.address_string(),
167	a43f68dc	Michael Hanselmann
168	a43f68dc	Michael Hanselmann	# RFC1413 identity (identd)
169	a43f68dc	Michael Hanselmann	"-",
170	a43f68dc	Michael Hanselmann
171	a43f68dc	Michael Hanselmann	# Remote user
172	a43f68dc	Michael Hanselmann	"-",
173	a43f68dc	Michael Hanselmann
174	a43f68dc	Michael Hanselmann	# Request time
175	a43f68dc	Michael Hanselmann	self._FormatCurrentTime(),
176	a43f68dc	Michael Hanselmann
177	a43f68dc	Michael Hanselmann	# Message
178	a43f68dc	Michael Hanselmann	format % args,
179	a43f68dc	Michael Hanselmann	))
180	a0638838	Oleksiy Mishchenko	self._fd.flush()
181	a43f68dc	Michael Hanselmann
182	a43f68dc	Michael Hanselmann	def _FormatCurrentTime(self):
183	a43f68dc	Michael Hanselmann	"""Formats current time in Common Log Format.
184	a43f68dc	Michael Hanselmann
185	a43f68dc	Michael Hanselmann	"""
186	a43f68dc	Michael Hanselmann	return self._FormatLogTime(time.time())
187	a43f68dc	Michael Hanselmann
188	a43f68dc	Michael Hanselmann	def _FormatLogTime(self, seconds):
189	a43f68dc	Michael Hanselmann	"""Formats time for Common Log Format.
190	a43f68dc	Michael Hanselmann
191	a43f68dc	Michael Hanselmann	All timestamps are logged in the UTC timezone.
192	a43f68dc	Michael Hanselmann
193	a43f68dc	Michael Hanselmann	Args:
194	a43f68dc	Michael Hanselmann	- seconds: Time in seconds since the epoch
195	a43f68dc	Michael Hanselmann
196	a43f68dc	Michael Hanselmann	"""
197	a43f68dc	Michael Hanselmann	(_, month, _, _, _, _, _, _, _) = tm = time.gmtime(seconds)
198	42242313	Michael Hanselmann	format = "%d/" + MONTHNAME[month] + "/%Y:%H:%M:%S +0000"
199	a43f68dc	Michael Hanselmann	return time.strftime(format, tm)
200	a43f68dc	Michael Hanselmann
201	a43f68dc	Michael Hanselmann
202	a43f68dc	Michael Hanselmann	class HTTPJsonConverter:
203	a43f68dc	Michael Hanselmann	CONTENT_TYPE = "application/json"
204	a43f68dc	Michael Hanselmann
205	a43f68dc	Michael Hanselmann	def Encode(self, data):
206	a43f68dc	Michael Hanselmann	return serializer.DumpJson(data)
207	a43f68dc	Michael Hanselmann
208	a43f68dc	Michael Hanselmann	def Decode(self, data):
209	a43f68dc	Michael Hanselmann	return serializer.LoadJson(data)
210	a43f68dc	Michael Hanselmann
211	a43f68dc	Michael Hanselmann
212	b14f759e	Michael Hanselmann	class _HttpSocketBase(object):
213	b14f759e	Michael Hanselmann	"""Base class for HTTP server and client.
214	b14f759e	Michael Hanselmann
215	b14f759e	Michael Hanselmann	"""
216	b14f759e	Michael Hanselmann	def __init__(self):
217	b14f759e	Michael Hanselmann	self._using_ssl = None
218	b14f759e	Michael Hanselmann	self._ssl_cert = None
219	b14f759e	Michael Hanselmann	self._ssl_key = None
220	b14f759e	Michael Hanselmann
221	b14f759e	Michael Hanselmann	def _CreateSocket(self, ssl_key_path, ssl_cert_path, ssl_verify_peer):
222	b14f759e	Michael Hanselmann	"""Creates a TCP socket and initializes SSL if needed.
223	b14f759e	Michael Hanselmann
224	b14f759e	Michael Hanselmann	@type ssl_key_path: string
225	b14f759e	Michael Hanselmann	@param ssl_key_path: Path to file containing SSL key in PEM format
226	b14f759e	Michael Hanselmann	@type ssl_cert_path: string
227	b14f759e	Michael Hanselmann	@param ssl_cert_path: Path to file containing SSL certificate in PEM format
228	b14f759e	Michael Hanselmann	@type ssl_verify_peer: bool
229	b14f759e	Michael Hanselmann	@param ssl_verify_peer: Whether to require client certificate and compare
230	b14f759e	Michael Hanselmann	it with our certificate
231	b14f759e	Michael Hanselmann
232	b14f759e	Michael Hanselmann	"""
233	b14f759e	Michael Hanselmann	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
234	b14f759e	Michael Hanselmann
235	b14f759e	Michael Hanselmann	# Should we enable SSL?
236	b14f759e	Michael Hanselmann	self._using_ssl = (ssl_cert_path and ssl_key_path)
237	b14f759e	Michael Hanselmann
238	b14f759e	Michael Hanselmann	if not self._using_ssl:
239	b14f759e	Michael Hanselmann	return sock
240	b14f759e	Michael Hanselmann
241	b14f759e	Michael Hanselmann	ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD)
242	b14f759e	Michael Hanselmann	ctx.set_options(OpenSSL.SSL.OP_NO_SSLv2)
243	b14f759e	Michael Hanselmann
244	b14f759e	Michael Hanselmann	ssl_key_pem = utils.ReadFile(ssl_key_path)
245	b14f759e	Michael Hanselmann	ssl_cert_pem = utils.ReadFile(ssl_cert_path)
246	b14f759e	Michael Hanselmann
247	b14f759e	Michael Hanselmann	cr = OpenSSL.crypto
248	b14f759e	Michael Hanselmann	self._ssl_cert = cr.load_certificate(cr.FILETYPE_PEM, ssl_cert_pem)
249	b14f759e	Michael Hanselmann	self._ssl_key = cr.load_privatekey(cr.FILETYPE_PEM, ssl_key_pem)
250	b14f759e	Michael Hanselmann	del cr
251	b14f759e	Michael Hanselmann
252	b14f759e	Michael Hanselmann	ctx.use_privatekey(self._ssl_key)
253	b14f759e	Michael Hanselmann	ctx.use_certificate(self._ssl_cert)
254	b14f759e	Michael Hanselmann	ctx.check_privatekey()
255	b14f759e	Michael Hanselmann
256	b14f759e	Michael Hanselmann	if ssl_verify_peer:
257	b14f759e	Michael Hanselmann	ctx.set_verify(OpenSSL.SSL.VERIFY_PEER \|
258	b14f759e	Michael Hanselmann	OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
259	b14f759e	Michael Hanselmann	self._SSLVerifyCallback)
260	b14f759e	Michael Hanselmann
261	b14f759e	Michael Hanselmann	return OpenSSL.SSL.Connection(ctx, sock)
262	b14f759e	Michael Hanselmann
263	b14f759e	Michael Hanselmann	def _SSLVerifyCallback(self, conn, cert, errnum, errdepth, ok):
264	b14f759e	Michael Hanselmann	"""Verify the certificate provided by the peer
265	b14f759e	Michael Hanselmann
266	b14f759e	Michael Hanselmann	We only compare fingerprints. The client must use the same certificate as
267	b14f759e	Michael Hanselmann	we do on our side.
268	b14f759e	Michael Hanselmann
269	b14f759e	Michael Hanselmann	"""
270	b14f759e	Michael Hanselmann	assert self._ssl_cert and self._ssl_key, "SSL not initialized"
271	b14f759e	Michael Hanselmann
272	b14f759e	Michael Hanselmann	return (self._ssl_cert.digest("sha1") == cert.digest("sha1") and
273	b14f759e	Michael Hanselmann	self._ssl_cert.digest("md5") == cert.digest("md5"))
274	b14f759e	Michael Hanselmann
275	b14f759e	Michael Hanselmann
276	42242313	Michael Hanselmann	class _HttpConnectionHandler(object):
277	42242313	Michael Hanselmann	"""Implements server side of HTTP
278	42242313	Michael Hanselmann
279	42242313	Michael Hanselmann	This class implements the server side of HTTP. It's based on code of Python's
280	42242313	Michael Hanselmann	BaseHTTPServer, from both version 2.4 and 3k. It does not support non-ASCII
281	42242313	Michael Hanselmann	character encodings. Keep-alive connections are not supported.
282	42242313	Michael Hanselmann
283	42242313	Michael Hanselmann	"""
284	42242313	Michael Hanselmann	# The default request version. This only affects responses up until
285	42242313	Michael Hanselmann	# the point where the request line is parsed, so it mainly decides what
286	42242313	Michael Hanselmann	# the client gets back when sending a malformed request line.
287	42242313	Michael Hanselmann	# Most web servers default to HTTP 0.9, i.e. don't send a status line.
288	42242313	Michael Hanselmann	default_request_version = HTTP_0_9
289	42242313	Michael Hanselmann
290	42242313	Michael Hanselmann	# Error message settings
291	42242313	Michael Hanselmann	error_message_format = DEFAULT_ERROR_MESSAGE
292	42242313	Michael Hanselmann	error_content_type = DEFAULT_ERROR_CONTENT_TYPE
293	42242313	Michael Hanselmann
294	42242313	Michael Hanselmann	responses = BaseHTTPServer.BaseHTTPRequestHandler.responses
295	42242313	Michael Hanselmann
296	42242313	Michael Hanselmann	def __init__(self, server, conn, client_addr, fileio_class):
297	42242313	Michael Hanselmann	"""Initializes this class.
298	42242313	Michael Hanselmann
299	42242313	Michael Hanselmann	Part of the initialization is reading the request and eventual POST/PUT
300	42242313	Michael Hanselmann	data sent by the client.
301	42242313	Michael Hanselmann
302	42242313	Michael Hanselmann	"""
303	42242313	Michael Hanselmann	self._server = server
304	42242313	Michael Hanselmann
305	42242313	Michael Hanselmann	# We default rfile to buffered because otherwise it could be
306	42242313	Michael Hanselmann	# really slow for large data (a getc() call per byte); we make
307	42242313	Michael Hanselmann	# wfile unbuffered because (a) often after a write() we want to
308	42242313	Michael Hanselmann	# read and we need to flush the line; (b) big writes to unbuffered
309	42242313	Michael Hanselmann	# files are typically optimized by stdio even when big reads
310	42242313	Michael Hanselmann	# aren't.
311	42242313	Michael Hanselmann	self.rfile = fileio_class(conn, mode="rb", bufsize=-1)
312	42242313	Michael Hanselmann	self.wfile = fileio_class(conn, mode="wb", bufsize=0)
313	42242313	Michael Hanselmann
314	42242313	Michael Hanselmann	self.client_addr = client_addr
315	42242313	Michael Hanselmann
316	42242313	Michael Hanselmann	self.request_headers = None
317	42242313	Michael Hanselmann	self.request_method = None
318	42242313	Michael Hanselmann	self.request_path = None
319	42242313	Michael Hanselmann	self.request_requestline = None
320	42242313	Michael Hanselmann	self.request_version = self.default_request_version
321	42242313	Michael Hanselmann
322	42242313	Michael Hanselmann	self.response_body = None
323	42242313	Michael Hanselmann	self.response_code = HTTP_OK
324	42242313	Michael Hanselmann	self.response_content_type = None
325	713faea6	Oleksiy Mishchenko	self.response_headers = {}
326	42242313	Michael Hanselmann
327	42242313	Michael Hanselmann	self.should_fork = False
328	42242313	Michael Hanselmann
329	42242313	Michael Hanselmann	try:
330	42242313	Michael Hanselmann	self._ReadRequest()
331	42242313	Michael Hanselmann	self._ReadPostData()
332	42242313	Michael Hanselmann	except HTTPException, err:
333	42242313	Michael Hanselmann	self._SetErrorStatus(err)
334	42242313	Michael Hanselmann
335	42242313	Michael Hanselmann	def Close(self):
336	42242313	Michael Hanselmann	if not self.wfile.closed:
337	42242313	Michael Hanselmann	self.wfile.flush()
338	42242313	Michael Hanselmann	self.wfile.close()
339	42242313	Michael Hanselmann	self.rfile.close()
340	42242313	Michael Hanselmann
341	42242313	Michael Hanselmann	def _DateTimeHeader(self):
342	42242313	Michael Hanselmann	"""Return the current date and time formatted for a message header.
343	42242313	Michael Hanselmann
344	42242313	Michael Hanselmann	"""
345	42242313	Michael Hanselmann	(year, month, day, hh, mm, ss, wd, _, _) = time.gmtime()
346	42242313	Michael Hanselmann	return ("%s, %02d %3s %4d %02d:%02d:%02d GMT" %
347	42242313	Michael Hanselmann	(WEEKDAYNAME[wd], day, MONTHNAME[month], year, hh, mm, ss))
348	42242313	Michael Hanselmann
349	42242313	Michael Hanselmann	def _SetErrorStatus(self, err):
350	42242313	Michael Hanselmann	"""Sets the response code and body from a HTTPException.
351	42242313	Michael Hanselmann
352	42242313	Michael Hanselmann	@type err: HTTPException
353	42242313	Michael Hanselmann	@param err: Exception instance
354	42242313	Michael Hanselmann
355	42242313	Michael Hanselmann	"""
356	42242313	Michael Hanselmann	try:
357	42242313	Michael Hanselmann	(shortmsg, longmsg) = self.responses[err.code]
358	42242313	Michael Hanselmann	except KeyError:
359	42242313	Michael Hanselmann	shortmsg = longmsg = "Unknown"
360	42242313	Michael Hanselmann
361	42242313	Michael Hanselmann	if err.message:
362	42242313	Michael Hanselmann	message = err.message
363	42242313	Michael Hanselmann	else:
364	42242313	Michael Hanselmann	message = shortmsg
365	42242313	Michael Hanselmann
366	42242313	Michael Hanselmann	values = {
367	42242313	Michael Hanselmann	"code": err.code,
368	42242313	Michael Hanselmann	"message": cgi.escape(message),
369	42242313	Michael Hanselmann	"explain": longmsg,
370	42242313	Michael Hanselmann	}
371	42242313	Michael Hanselmann
372	42242313	Michael Hanselmann	self.response_code = err.code
373	42242313	Michael Hanselmann	self.response_content_type = self.error_content_type
374	42242313	Michael Hanselmann	self.response_body = self.error_message_format % values
375	42242313	Michael Hanselmann
376	42242313	Michael Hanselmann	def HandleRequest(self):
377	42242313	Michael Hanselmann	"""Handle the actual request.
378	42242313	Michael Hanselmann
379	42242313	Michael Hanselmann	Calls the actual handler function and converts exceptions into HTTP errors.
380	42242313	Michael Hanselmann
381	42242313	Michael Hanselmann	"""
382	42242313	Michael Hanselmann	# Don't do anything if there's already been a problem
383	42242313	Michael Hanselmann	if self.response_code != HTTP_OK:
384	42242313	Michael Hanselmann	return
385	42242313	Michael Hanselmann
386	42242313	Michael Hanselmann	assert self.request_method, "Status code %s requires a method" % HTTP_OK
387	42242313	Michael Hanselmann
388	42242313	Michael Hanselmann	# Check whether client is still there
389	42242313	Michael Hanselmann	self.rfile.read(0)
390	42242313	Michael Hanselmann
391	42242313	Michael Hanselmann	try:
392	42242313	Michael Hanselmann	try:
393	42242313	Michael Hanselmann	result = self._server.HandleRequest(self)
394	42242313	Michael Hanselmann
395	42242313	Michael Hanselmann	# TODO: Content-type
396	42242313	Michael Hanselmann	encoder = HTTPJsonConverter()
397	42242313	Michael Hanselmann	body = encoder.Encode(result)
398	42242313	Michael Hanselmann
399	42242313	Michael Hanselmann	self.response_content_type = encoder.CONTENT_TYPE
400	42242313	Michael Hanselmann	self.response_body = body
401	42242313	Michael Hanselmann	except (HTTPException, KeyboardInterrupt, SystemExit):
402	42242313	Michael Hanselmann	raise
403	42242313	Michael Hanselmann	except Exception, err:
404	42242313	Michael Hanselmann	logging.exception("Caught exception")
405	42242313	Michael Hanselmann	raise HTTPInternalError(message=str(err))
406	42242313	Michael Hanselmann	except:
407	42242313	Michael Hanselmann	logging.exception("Unknown exception")
408	42242313	Michael Hanselmann	raise HTTPInternalError(message="Unknown error")
409	42242313	Michael Hanselmann
410	42242313	Michael Hanselmann	except HTTPException, err:
411	42242313	Michael Hanselmann	self._SetErrorStatus(err)
412	42242313	Michael Hanselmann
413	42242313	Michael Hanselmann	def SendResponse(self):
414	42242313	Michael Hanselmann	"""Sends response to the client.
415	42242313	Michael Hanselmann
416	42242313	Michael Hanselmann	"""
417	42242313	Michael Hanselmann	# Check whether client is still there
418	42242313	Michael Hanselmann	self.rfile.read(0)
419	42242313	Michael Hanselmann
420	42242313	Michael Hanselmann	logging.info("%s:%s %s %s", self.client_addr[0], self.client_addr[1],
421	42242313	Michael Hanselmann	self.request_requestline, self.response_code)
422	42242313	Michael Hanselmann
423	42242313	Michael Hanselmann	if self.response_code in self.responses:
424	42242313	Michael Hanselmann	response_message = self.responses[self.response_code][0]
425	42242313	Michael Hanselmann	else:
426	42242313	Michael Hanselmann	response_message = ""
427	42242313	Michael Hanselmann
428	42242313	Michael Hanselmann	if self.request_version != HTTP_0_9:
429	42242313	Michael Hanselmann	self.wfile.write("%s %d %s\r\n" %
430	42242313	Michael Hanselmann	(self.request_version, self.response_code,
431	42242313	Michael Hanselmann	response_message))
432	8a9f9060	Michael Hanselmann	self._SendHeader(HTTP_SERVER, HTTP_GANETI_VERSION)
433	8a9f9060	Michael Hanselmann	self._SendHeader(HTTP_DATE, self._DateTimeHeader())
434	8a9f9060	Michael Hanselmann	self._SendHeader(HTTP_CONTENT_TYPE, self.response_content_type)
435	8a9f9060	Michael Hanselmann	self._SendHeader(HTTP_CONTENT_LENGTH, str(len(self.response_body)))
436	713faea6	Oleksiy Mishchenko	for key, val in self.response_headers.iteritems():
437	713faea6	Oleksiy Mishchenko	self._SendHeader(key, val)
438	713faea6	Oleksiy Mishchenko
439	42242313	Michael Hanselmann	# We don't support keep-alive at this time
440	8a9f9060	Michael Hanselmann	self._SendHeader(HTTP_CONNECTION, "close")
441	42242313	Michael Hanselmann	self.wfile.write("\r\n")
442	42242313	Michael Hanselmann
443	42242313	Michael Hanselmann	if (self.request_method != HTTP_HEAD and
444	42242313	Michael Hanselmann	self.response_code >= HTTP_OK and
445	42242313	Michael Hanselmann	self.response_code not in (HTTP_NO_CONTENT, HTTP_NOT_MODIFIED)):
446	42242313	Michael Hanselmann	self.wfile.write(self.response_body)
447	42242313	Michael Hanselmann
448	42242313	Michael Hanselmann	def _SendHeader(self, name, value):
449	42242313	Michael Hanselmann	if self.request_version != HTTP_0_9:
450	42242313	Michael Hanselmann	self.wfile.write("%s: %s\r\n" % (name, value))
451	42242313	Michael Hanselmann
452	42242313	Michael Hanselmann	def _ReadRequest(self):
453	42242313	Michael Hanselmann	"""Reads and parses request line
454	42242313	Michael Hanselmann
455	42242313	Michael Hanselmann	"""
456	42242313	Michael Hanselmann	raw_requestline = self.rfile.readline()
457	42242313	Michael Hanselmann
458	42242313	Michael Hanselmann	requestline = raw_requestline
459	42242313	Michael Hanselmann	if requestline[-2:] == '\r\n':
460	42242313	Michael Hanselmann	requestline = requestline[:-2]
461	42242313	Michael Hanselmann	elif requestline[-1:] == '\n':
462	42242313	Michael Hanselmann	requestline = requestline[:-1]
463	42242313	Michael Hanselmann
464	42242313	Michael Hanselmann	if not requestline:
465	42242313	Michael Hanselmann	raise HTTPBadRequest("Empty request line")
466	42242313	Michael Hanselmann
467	42242313	Michael Hanselmann	self.request_requestline = requestline
468	42242313	Michael Hanselmann
469	42242313	Michael Hanselmann	logging.debug("HTTP request: %s", raw_requestline.rstrip("\r\n"))
470	42242313	Michael Hanselmann
471	42242313	Michael Hanselmann	words = requestline.split()
472	42242313	Michael Hanselmann
473	42242313	Michael Hanselmann	if len(words) == 3:
474	42242313	Michael Hanselmann	[method, path, version] = words
475	42242313	Michael Hanselmann	if version[:5] != 'HTTP/':
476	42242313	Michael Hanselmann	raise HTTPBadRequest("Bad request version (%r)" % version)
477	42242313	Michael Hanselmann
478	42242313	Michael Hanselmann	try:
479	42242313	Michael Hanselmann	base_version_number = version.split('/', 1)[1]
480	42242313	Michael Hanselmann	version_number = base_version_number.split(".")
481	42242313	Michael Hanselmann
482	42242313	Michael Hanselmann	# RFC 2145 section 3.1 says there can be only one "." and
483	42242313	Michael Hanselmann	# - major and minor numbers MUST be treated as
484	42242313	Michael Hanselmann	# separate integers;
485	42242313	Michael Hanselmann	# - HTTP/2.4 is a lower version than HTTP/2.13, which in
486	42242313	Michael Hanselmann	# turn is lower than HTTP/12.3;
487	42242313	Michael Hanselmann	# - Leading zeros MUST be ignored by recipients.
488	42242313	Michael Hanselmann	if len(version_number) != 2:
489	42242313	Michael Hanselmann	raise HTTPBadRequest("Bad request version (%r)" % version)
490	42242313	Michael Hanselmann
491	42242313	Michael Hanselmann	version_number = int(version_number[0]), int(version_number[1])
492	42242313	Michael Hanselmann	except (ValueError, IndexError):
493	42242313	Michael Hanselmann	raise HTTPBadRequest("Bad request version (%r)" % version)
494	42242313	Michael Hanselmann
495	42242313	Michael Hanselmann	if version_number >= (2, 0):
496	42242313	Michael Hanselmann	raise HTTPVersionNotSupported("Invalid HTTP Version (%s)" %
497	42242313	Michael Hanselmann	base_version_number)
498	42242313	Michael Hanselmann
499	42242313	Michael Hanselmann	elif len(words) == 2:
500	42242313	Michael Hanselmann	version = HTTP_0_9
501	42242313	Michael Hanselmann	[method, path] = words
502	42242313	Michael Hanselmann	if method != HTTP_GET:
503	42242313	Michael Hanselmann	raise HTTPBadRequest("Bad HTTP/0.9 request type (%r)" % method)
504	42242313	Michael Hanselmann
505	42242313	Michael Hanselmann	else:
506	42242313	Michael Hanselmann	raise HTTPBadRequest("Bad request syntax (%r)" % requestline)
507	42242313	Michael Hanselmann
508	42242313	Michael Hanselmann	# Examine the headers and look for a Connection directive
509	42242313	Michael Hanselmann	headers = mimetools.Message(self.rfile, 0)
510	42242313	Michael Hanselmann
511	42242313	Michael Hanselmann	self.request_method = method
512	42242313	Michael Hanselmann	self.request_path = path
513	42242313	Michael Hanselmann	self.request_version = version
514	42242313	Michael Hanselmann	self.request_headers = headers
515	42242313	Michael Hanselmann
516	42242313	Michael Hanselmann	def _ReadPostData(self):
517	42242313	Michael Hanselmann	"""Reads POST/PUT data
518	42242313	Michael Hanselmann
519	64357ed8	Michael Hanselmann	Quoting RFC1945, section 7.2 (HTTP/1.0): "The presence of an entity body in
520	64357ed8	Michael Hanselmann	a request is signaled by the inclusion of a Content-Length header field in
521	64357ed8	Michael Hanselmann	the request message headers. HTTP/1.0 requests containing an entity body
522	64357ed8	Michael Hanselmann	must include a valid Content-Length header field."
523	64357ed8	Michael Hanselmann
524	42242313	Michael Hanselmann	"""
525	64357ed8	Michael Hanselmann	# While not according to specification, we only support an entity body for
526	64357ed8	Michael Hanselmann	# POST and PUT.
527	8a9f9060	Michael Hanselmann	if (not self.request_method or
528	8a9f9060	Michael Hanselmann	self.request_method.upper() not in (HTTP_POST, HTTP_PUT)):
529	42242313	Michael Hanselmann	self.request_post_data = None
530	42242313	Michael Hanselmann	return
531	42242313	Michael Hanselmann
532	64357ed8	Michael Hanselmann	content_length = None
533	42242313	Michael Hanselmann	try:
534	64357ed8	Michael Hanselmann	if HTTP_CONTENT_LENGTH in self.request_headers:
535	64357ed8	Michael Hanselmann	content_length = int(self.request_headers[HTTP_CONTENT_LENGTH])
536	64357ed8	Michael Hanselmann	except TypeError:
537	64357ed8	Michael Hanselmann	pass
538	42242313	Michael Hanselmann	except ValueError:
539	64357ed8	Michael Hanselmann	pass
540	64357ed8	Michael Hanselmann
541	64357ed8	Michael Hanselmann	# 411 Length Required is specified in RFC2616, section 10.4.12 (HTTP/1.1)
542	64357ed8	Michael Hanselmann	if content_length is None:
543	64357ed8	Michael Hanselmann	raise HTTPLengthRequired("Missing Content-Length header or"
544	64357ed8	Michael Hanselmann	" invalid format")
545	42242313	Michael Hanselmann
546	42242313	Michael Hanselmann	data = self.rfile.read(content_length)
547	42242313	Michael Hanselmann
548	42242313	Michael Hanselmann	# TODO: Content-type, error handling
549	42242313	Michael Hanselmann	self.request_post_data = HTTPJsonConverter().Decode(data)
550	42242313	Michael Hanselmann
551	42242313	Michael Hanselmann	logging.debug("HTTP POST data: %s", self.request_post_data)
552	42242313	Michael Hanselmann
553	42242313	Michael Hanselmann
554	b14f759e	Michael Hanselmann	class HttpServer(_HttpSocketBase):
555	42242313	Michael Hanselmann	"""Generic HTTP server class
556	42242313	Michael Hanselmann
557	42242313	Michael Hanselmann	Users of this class must subclass it and override the HandleRequest function.
558	42242313	Michael Hanselmann
559	42242313	Michael Hanselmann	"""
560	42242313	Michael Hanselmann	MAX_CHILDREN = 20
561	42242313	Michael Hanselmann
562	f2a6fc9e	Michael Hanselmann	def __init__(self, mainloop, local_address, port,
563	f2a6fc9e	Michael Hanselmann	ssl_key_path=None, ssl_cert_path=None, ssl_verify_peer=False):
564	23e46494	Michael Hanselmann	"""Initializes the HTTP server
565	23e46494	Michael Hanselmann
566	23e46494	Michael Hanselmann	@type mainloop: ganeti.daemon.Mainloop
567	23e46494	Michael Hanselmann	@param mainloop: Mainloop used to poll for I/O events
568	23e46494	Michael Hanselmann	@type local_addess: string
569	23e46494	Michael Hanselmann	@param local_address: Local IP address to bind to
570	23e46494	Michael Hanselmann	@type port: int
571	23e46494	Michael Hanselmann	@param port: TCP port to listen on
572	f2a6fc9e	Michael Hanselmann	@type ssl_key_path: string
573	f2a6fc9e	Michael Hanselmann	@param ssl_key_path: Path to file containing SSL key in PEM format
574	f2a6fc9e	Michael Hanselmann	@type ssl_cert_path: string
575	f2a6fc9e	Michael Hanselmann	@param ssl_cert_path: Path to file containing SSL certificate in PEM format
576	f2a6fc9e	Michael Hanselmann	@type ssl_verify_peer: bool
577	f2a6fc9e	Michael Hanselmann	@param ssl_verify_peer: Whether to require client certificate and compare
578	f2a6fc9e	Michael Hanselmann	it with our certificate
579	23e46494	Michael Hanselmann
580	23e46494	Michael Hanselmann	"""
581	b14f759e	Michael Hanselmann	_HttpSocketBase.__init__(self)
582	b14f759e	Michael Hanselmann
583	42242313	Michael Hanselmann	self.mainloop = mainloop
584	23e46494	Michael Hanselmann	self.local_address = local_address
585	23e46494	Michael Hanselmann	self.port = port
586	42242313	Michael Hanselmann
587	b14f759e	Michael Hanselmann	self.socket = self._CreateSocket(ssl_key_path, ssl_cert_path, ssl_verify_peer)
588	f2a6fc9e	Michael Hanselmann
589	b14f759e	Michael Hanselmann	# Allow port to be reused
590	b14f759e	Michael Hanselmann	self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
591	42242313	Michael Hanselmann
592	b14f759e	Michael Hanselmann	if self._using_ssl:
593	42242313	Michael Hanselmann	self._fileio_class = _SSLFileObject
594	42242313	Michael Hanselmann	else:
595	42242313	Michael Hanselmann	self._fileio_class = socket._fileobject
596	42242313	Michael Hanselmann
597	42242313	Michael Hanselmann	self._children = []
598	42242313	Michael Hanselmann
599	42242313	Michael Hanselmann	mainloop.RegisterIO(self, self.socket.fileno(), select.POLLIN)
600	42242313	Michael Hanselmann	mainloop.RegisterSignal(self)
601	42242313	Michael Hanselmann
602	42242313	Michael Hanselmann	def Start(self):
603	23e46494	Michael Hanselmann	self.socket.bind((self.local_address, self.port))
604	42242313	Michael Hanselmann	self.socket.listen(5)
605	42242313	Michael Hanselmann
606	42242313	Michael Hanselmann	def Stop(self):
607	42242313	Michael Hanselmann	self.socket.close()
608	42242313	Michael Hanselmann
609	42242313	Michael Hanselmann	def OnIO(self, fd, condition):
610	42242313	Michael Hanselmann	if condition & select.POLLIN:
611	42242313	Michael Hanselmann	self._IncomingConnection()
612	42242313	Michael Hanselmann
613	42242313	Michael Hanselmann	def OnSignal(self, signum):
614	42242313	Michael Hanselmann	if signum == signal.SIGCHLD:
615	42242313	Michael Hanselmann	self._CollectChildren(True)
616	42242313	Michael Hanselmann
617	42242313	Michael Hanselmann	def _CollectChildren(self, quick):
618	42242313	Michael Hanselmann	"""Checks whether any child processes are done
619	42242313	Michael Hanselmann
620	42242313	Michael Hanselmann	@type quick: bool
621	42242313	Michael Hanselmann	@param quick: Whether to only use non-blocking functions
622	42242313	Michael Hanselmann
623	42242313	Michael Hanselmann	"""
624	42242313	Michael Hanselmann	if not quick:
625	42242313	Michael Hanselmann	# Don't wait for other processes if it should be a quick check
626	42242313	Michael Hanselmann	while len(self._children) > self.MAX_CHILDREN:
627	42242313	Michael Hanselmann	try:
628	6526ddcd	Michael Hanselmann	# Waiting without a timeout brings us into a potential DoS situation.
629	6526ddcd	Michael Hanselmann	# As soon as too many children run, we'll not respond to new
630	6526ddcd	Michael Hanselmann	# requests. The real solution would be to add a timeout for children
631	6526ddcd	Michael Hanselmann	# and killing them after some time.
632	42242313	Michael Hanselmann	pid, status = os.waitpid(0, 0)
633	42242313	Michael Hanselmann	except os.error:
634	42242313	Michael Hanselmann	pid = None
635	42242313	Michael Hanselmann	if pid and pid in self._children:
636	42242313	Michael Hanselmann	self._children.remove(pid)
637	42242313	Michael Hanselmann
638	42242313	Michael Hanselmann	for child in self._children:
639	42242313	Michael Hanselmann	try:
640	42242313	Michael Hanselmann	pid, status = os.waitpid(child, os.WNOHANG)
641	42242313	Michael Hanselmann	except os.error:
642	42242313	Michael Hanselmann	pid = None
643	42242313	Michael Hanselmann	if pid and pid in self._children:
644	42242313	Michael Hanselmann	self._children.remove(pid)
645	42242313	Michael Hanselmann
646	42242313	Michael Hanselmann	def _IncomingConnection(self):
647	6526ddcd	Michael Hanselmann	"""Called for each incoming connection
648	42242313	Michael Hanselmann
649	6526ddcd	Michael Hanselmann	"""
650	6526ddcd	Michael Hanselmann	(connection, client_addr) = self.socket.accept()
651	42242313	Michael Hanselmann
652	42242313	Michael Hanselmann	self._CollectChildren(False)
653	42242313	Michael Hanselmann
654	42242313	Michael Hanselmann	pid = os.fork()
655	42242313	Michael Hanselmann	if pid == 0:
656	42242313	Michael Hanselmann	# Child process
657	6526ddcd	Michael Hanselmann	logging.info("Connection from %s:%s", client_addr[0], client_addr[1])
658	6526ddcd	Michael Hanselmann
659	42242313	Michael Hanselmann	try:
660	6526ddcd	Michael Hanselmann	try:
661	6526ddcd	Michael Hanselmann	try:
662	6526ddcd	Michael Hanselmann	handler = None
663	6526ddcd	Michael Hanselmann	try:
664	6526ddcd	Michael Hanselmann	# Read, parse and handle request
665	6526ddcd	Michael Hanselmann	handler = _HttpConnectionHandler(self, connection, client_addr,
666	6526ddcd	Michael Hanselmann	self._fileio_class)
667	6526ddcd	Michael Hanselmann	handler.HandleRequest()
668	6526ddcd	Michael Hanselmann	finally:
669	6526ddcd	Michael Hanselmann	# Try to send a response
670	6526ddcd	Michael Hanselmann	if handler:
671	6526ddcd	Michael Hanselmann	handler.SendResponse()
672	6526ddcd	Michael Hanselmann	handler.Close()
673	6526ddcd	Michael Hanselmann	except SocketClosed:
674	6526ddcd	Michael Hanselmann	pass
675	6526ddcd	Michael Hanselmann	finally:
676	6526ddcd	Michael Hanselmann	logging.info("Disconnected %s:%s", client_addr[0], client_addr[1])
677	42242313	Michael Hanselmann	except:
678	42242313	Michael Hanselmann	logging.exception("Error while handling request from %s:%s",
679	42242313	Michael Hanselmann	client_addr[0], client_addr[1])
680	42242313	Michael Hanselmann	os._exit(1)
681	42242313	Michael Hanselmann	os._exit(0)
682	42242313	Michael Hanselmann	else:
683	42242313	Michael Hanselmann	self._children.append(pid)
684	42242313	Michael Hanselmann
685	42242313	Michael Hanselmann	def HandleRequest(self, req):
686	42242313	Michael Hanselmann	raise NotImplementedError()
687	42242313	Michael Hanselmann
688	42242313	Michael Hanselmann
689	8a0b06d2	Michael Hanselmann	class HttpClientRequest(object):
690	8a0b06d2	Michael Hanselmann	def __init__(self, host, port, method, path, headers=None, post_data=None):
691	8a0b06d2	Michael Hanselmann	"""Describes an HTTP request.
692	8a0b06d2	Michael Hanselmann
693	8a0b06d2	Michael Hanselmann	@type host: string
694	8a0b06d2	Michael Hanselmann	@param host: Hostname
695	8a0b06d2	Michael Hanselmann	@type port: int
696	8a0b06d2	Michael Hanselmann	@param port: Port
697	8a0b06d2	Michael Hanselmann	@type method: string
698	8a0b06d2	Michael Hanselmann	@param method: Method name
699	8a0b06d2	Michael Hanselmann	@type path: string
700	8a0b06d2	Michael Hanselmann	@param path: Request path
701	8a0b06d2	Michael Hanselmann	@type headers: dict or None
702	8a0b06d2	Michael Hanselmann	@param headers: Additional headers to send
703	8a0b06d2	Michael Hanselmann	@type post_data: string or None
704	8a0b06d2	Michael Hanselmann	@param post_data: Additional data to send
705	8a0b06d2	Michael Hanselmann
706	8a0b06d2	Michael Hanselmann	"""
707	8a0b06d2	Michael Hanselmann	if post_data is not None:
708	8a0b06d2	Michael Hanselmann	assert method.upper() in (HTTP_POST, HTTP_PUT), \
709	8a0b06d2	Michael Hanselmann	"Only POST and GET requests support sending data"
710	8a0b06d2	Michael Hanselmann
711	8a0b06d2	Michael Hanselmann	assert path.startswith("/"), "Path must start with slash (/)"
712	8a0b06d2	Michael Hanselmann
713	8a0b06d2	Michael Hanselmann	self.host = host
714	8a0b06d2	Michael Hanselmann	self.port = port
715	8a0b06d2	Michael Hanselmann	self.method = method
716	8a0b06d2	Michael Hanselmann	self.path = path
717	8a0b06d2	Michael Hanselmann	self.headers = headers
718	8a0b06d2	Michael Hanselmann	self.post_data = post_data
719	8a0b06d2	Michael Hanselmann
720	8a0b06d2	Michael Hanselmann	self.success = None
721	8a0b06d2	Michael Hanselmann	self.error = None
722	8a0b06d2	Michael Hanselmann
723	8a0b06d2	Michael Hanselmann	self.resp_status_line = None
724	8a0b06d2	Michael Hanselmann	self.resp_version = None
725	8a0b06d2	Michael Hanselmann	self.resp_status = None
726	8a0b06d2	Michael Hanselmann	self.resp_reason = None
727	8a0b06d2	Michael Hanselmann	self.resp_headers = None
728	8a0b06d2	Michael Hanselmann	self.resp_body = None
729	8a0b06d2	Michael Hanselmann
730	8a0b06d2	Michael Hanselmann
731	8a0b06d2	Michael Hanselmann	class HttpClientRequestExecutor(object):
732	8a0b06d2	Michael Hanselmann	# Default headers
733	8a0b06d2	Michael Hanselmann	DEFAULT_HEADERS = {
734	8a0b06d2	Michael Hanselmann	HTTP_USER_AGENT: HTTP_GANETI_VERSION,
735	8a0b06d2	Michael Hanselmann	# TODO: For keep-alive, don't send "Connection: close"
736	8a0b06d2	Michael Hanselmann	HTTP_CONNECTION: "close",
737	8a0b06d2	Michael Hanselmann	}
738	8a0b06d2	Michael Hanselmann
739	8a0b06d2	Michael Hanselmann	# Length limits
740	8a0b06d2	Michael Hanselmann	STATUS_LINE_LENGTH_MAX = 512
741	8a0b06d2	Michael Hanselmann	HEADER_LENGTH_MAX = 4 * 1024
742	8a0b06d2	Michael Hanselmann
743	8a0b06d2	Michael Hanselmann	# Timeouts in seconds
744	8a0b06d2	Michael Hanselmann	# TODO: Make read timeout configurable per OpCode
745	8a0b06d2	Michael Hanselmann	CONNECT_TIMEOUT = 5.0
746	8a0b06d2	Michael Hanselmann	WRITE_TIMEOUT = 10
747	8a0b06d2	Michael Hanselmann	READ_TIMEOUT = None
748	8a0b06d2	Michael Hanselmann	CLOSE_TIMEOUT = 1
749	8a0b06d2	Michael Hanselmann
750	8a0b06d2	Michael Hanselmann	# Parser state machine
751	8a0b06d2	Michael Hanselmann	PS_STATUS_LINE = "status-line"
752	8a0b06d2	Michael Hanselmann	PS_HEADERS = "headers"
753	8a0b06d2	Michael Hanselmann	PS_BODY = "body"
754	8a0b06d2	Michael Hanselmann	PS_COMPLETE = "complete"
755	8a0b06d2	Michael Hanselmann
756	8a0b06d2	Michael Hanselmann	def __init__(self, req):
757	8a0b06d2	Michael Hanselmann	"""Initializes the HttpClientRequestExecutor class.
758	8a0b06d2	Michael Hanselmann
759	8a0b06d2	Michael Hanselmann	@type req: HttpClientRequest
760	8a0b06d2	Michael Hanselmann	@param req: Request object
761	8a0b06d2	Michael Hanselmann
762	8a0b06d2	Michael Hanselmann	"""
763	8a0b06d2	Michael Hanselmann	self.request = req
764	8a0b06d2	Michael Hanselmann
765	8a0b06d2	Michael Hanselmann	self.parser_status = self.PS_STATUS_LINE
766	8a0b06d2	Michael Hanselmann	self.header_buffer = StringIO()
767	8a0b06d2	Michael Hanselmann	self.body_buffer = StringIO()
768	8a0b06d2	Michael Hanselmann	self.content_length = None
769	8a0b06d2	Michael Hanselmann	self.server_will_close = None
770	8a0b06d2	Michael Hanselmann
771	8a0b06d2	Michael Hanselmann	self.poller = select.poll()
772	8a0b06d2	Michael Hanselmann
773	8a0b06d2	Michael Hanselmann	# TODO: SSL
774	8a0b06d2	Michael Hanselmann
775	8a0b06d2	Michael Hanselmann	try:
776	8a0b06d2	Michael Hanselmann	# TODO: Implement connection caching/keep-alive
777	8a0b06d2	Michael Hanselmann	self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
778	8a0b06d2	Michael Hanselmann
779	8a0b06d2	Michael Hanselmann	# Disable Python's timeout
780	8a0b06d2	Michael Hanselmann	self.sock.settimeout(None)
781	8a0b06d2	Michael Hanselmann
782	8a0b06d2	Michael Hanselmann	# Operate in non-blocking mode
783	8a0b06d2	Michael Hanselmann	self.sock.setblocking(0)
784	8a0b06d2	Michael Hanselmann
785	8a0b06d2	Michael Hanselmann	force_close = True
786	8a0b06d2	Michael Hanselmann	self._Connect()
787	8a0b06d2	Michael Hanselmann	try:
788	8a0b06d2	Michael Hanselmann	self._SendRequest()
789	8a0b06d2	Michael Hanselmann	self._ReadResponse()
790	8a0b06d2	Michael Hanselmann
791	8a0b06d2	Michael Hanselmann	# Only wait for server to close if we didn't have any exception.
792	8a0b06d2	Michael Hanselmann	force_close = False
793	8a0b06d2	Michael Hanselmann	finally:
794	8a0b06d2	Michael Hanselmann	self._CloseConnection(force_close)
795	8a0b06d2	Michael Hanselmann
796	8a0b06d2	Michael Hanselmann	self.sock.close()
797	8a0b06d2	Michael Hanselmann	self.sock = None
798	8a0b06d2	Michael Hanselmann
799	8a0b06d2	Michael Hanselmann	req.resp_body = self.body_buffer.getvalue()
800	8a0b06d2	Michael Hanselmann
801	8a0b06d2	Michael Hanselmann	req.success = True
802	8a0b06d2	Michael Hanselmann	req.error = None
803	8a0b06d2	Michael Hanselmann
804	8a0b06d2	Michael Hanselmann	except ResponseError, err:
805	8a0b06d2	Michael Hanselmann	req.success = False
806	8a0b06d2	Michael Hanselmann	req.error = str(err)
807	8a0b06d2	Michael Hanselmann
808	8a0b06d2	Michael Hanselmann	def _BuildRequest(self):
809	8a0b06d2	Michael Hanselmann	"""Build HTTP request.
810	8a0b06d2	Michael Hanselmann
811	8a0b06d2	Michael Hanselmann	@rtype: string
812	8a0b06d2	Michael Hanselmann	@return: Complete request
813	8a0b06d2	Michael Hanselmann
814	8a0b06d2	Michael Hanselmann	"""
815	8a0b06d2	Michael Hanselmann	# Headers
816	8a0b06d2	Michael Hanselmann	send_headers = self.DEFAULT_HEADERS.copy()
817	8a0b06d2	Michael Hanselmann
818	8a0b06d2	Michael Hanselmann	if self.request.headers:
819	8a0b06d2	Michael Hanselmann	send_headers.update(req.headers)
820	8a0b06d2	Michael Hanselmann
821	8a0b06d2	Michael Hanselmann	send_headers[HTTP_HOST] = "%s:%s" % (self.request.host, self.request.port)
822	8a0b06d2	Michael Hanselmann
823	8a0b06d2	Michael Hanselmann	if self.request.post_data:
824	8a0b06d2	Michael Hanselmann	send_headers[HTTP_CONTENT_LENGTH] = len(self.request.post_data)
825	8a0b06d2	Michael Hanselmann
826	8a0b06d2	Michael Hanselmann	buf = StringIO()
827	8a0b06d2	Michael Hanselmann
828	8a0b06d2	Michael Hanselmann	# Add request line. We only support HTTP/1.0 (no chunked transfers and no
829	8a0b06d2	Michael Hanselmann	# keep-alive).
830	8a0b06d2	Michael Hanselmann	# TODO: For keep-alive, change to HTTP/1.1
831	8a0b06d2	Michael Hanselmann	buf.write("%s %s %s\r\n" % (self.request.method.upper(),
832	8a0b06d2	Michael Hanselmann	self.request.path, HTTP_1_0))
833	8a0b06d2	Michael Hanselmann
834	8a0b06d2	Michael Hanselmann	# Add headers
835	8a0b06d2	Michael Hanselmann	for name, value in send_headers.iteritems():
836	8a0b06d2	Michael Hanselmann	buf.write("%s: %s\r\n" % (name, value))
837	8a0b06d2	Michael Hanselmann
838	8a0b06d2	Michael Hanselmann	buf.write("\r\n")
839	8a0b06d2	Michael Hanselmann
840	8a0b06d2	Michael Hanselmann	if self.request.post_data:
841	8a0b06d2	Michael Hanselmann	buf.write(self.request.post_data)
842	8a0b06d2	Michael Hanselmann
843	8a0b06d2	Michael Hanselmann	return buf.getvalue()
844	8a0b06d2	Michael Hanselmann
845	8a0b06d2	Michael Hanselmann	def _ParseStatusLine(self):
846	8a0b06d2	Michael Hanselmann	"""Parses the status line sent by the server.
847	8a0b06d2	Michael Hanselmann
848	8a0b06d2	Michael Hanselmann	"""
849	8a0b06d2	Michael Hanselmann	line = self.request.resp_status_line
850	8a0b06d2	Michael Hanselmann
851	8a0b06d2	Michael Hanselmann	if not line:
852	8a0b06d2	Michael Hanselmann	raise ResponseError("Empty status line")
853	8a0b06d2	Michael Hanselmann
854	8a0b06d2	Michael Hanselmann	try:
855	8a0b06d2	Michael Hanselmann	[version, status, reason] = line.split(None, 2)
856	8a0b06d2	Michael Hanselmann	except ValueError:
857	8a0b06d2	Michael Hanselmann	try:
858	8a0b06d2	Michael Hanselmann	[version, status] = line.split(None, 1)
859	8a0b06d2	Michael Hanselmann	reason = ""
860	8a0b06d2	Michael Hanselmann	except ValueError:
861	8a0b06d2	Michael Hanselmann	version = HTTP_9_0
862	8a0b06d2	Michael Hanselmann
863	8a0b06d2	Michael Hanselmann	if version:
864	8a0b06d2	Michael Hanselmann	version = version.upper()
865	8a0b06d2	Michael Hanselmann
866	8a0b06d2	Michael Hanselmann	if version not in (HTTP_1_0, HTTP_1_1):
867	8a0b06d2	Michael Hanselmann	# We do not support HTTP/0.9, despite the specification requiring it
868	8a0b06d2	Michael Hanselmann	# (RFC2616, section 19.6)
869	8a0b06d2	Michael Hanselmann	raise ResponseError("Only HTTP/1.0 and HTTP/1.1 are supported (%r)" %
870	8a0b06d2	Michael Hanselmann	line)
871	8a0b06d2	Michael Hanselmann
872	8a0b06d2	Michael Hanselmann	# The status code is a three-digit number
873	8a0b06d2	Michael Hanselmann	try:
874	8a0b06d2	Michael Hanselmann	status = int(status)
875	8a0b06d2	Michael Hanselmann	if status < 100 or status > 999:
876	8a0b06d2	Michael Hanselmann	status = -1
877	8a0b06d2	Michael Hanselmann	except ValueError:
878	8a0b06d2	Michael Hanselmann	status = -1
879	8a0b06d2	Michael Hanselmann
880	8a0b06d2	Michael Hanselmann	if status == -1:
881	8a0b06d2	Michael Hanselmann	raise ResponseError("Invalid status code (%r)" % line)
882	8a0b06d2	Michael Hanselmann
883	8a0b06d2	Michael Hanselmann	self.request.resp_version = version
884	8a0b06d2	Michael Hanselmann	self.request.resp_status = status
885	8a0b06d2	Michael Hanselmann	self.request.resp_reason = reason
886	8a0b06d2	Michael Hanselmann
887	8a0b06d2	Michael Hanselmann	def _WillServerCloseConnection(self):
888	8a0b06d2	Michael Hanselmann	"""Evaluate whether server will close the connection.
889	8a0b06d2	Michael Hanselmann
890	8a0b06d2	Michael Hanselmann	@rtype: bool
891	8a0b06d2	Michael Hanselmann	@return: Whether server will close the connection
892	8a0b06d2	Michael Hanselmann
893	8a0b06d2	Michael Hanselmann	"""
894	8a0b06d2	Michael Hanselmann	hdr_connection = self.request.resp_headers.get(HTTP_CONNECTION, None)
895	8a0b06d2	Michael Hanselmann	if hdr_connection:
896	8a0b06d2	Michael Hanselmann	hdr_connection = hdr_connection.lower()
897	8a0b06d2	Michael Hanselmann
898	8a0b06d2	Michael Hanselmann	# An HTTP/1.1 server is assumed to stay open unless explicitly closed.
899	8a0b06d2	Michael Hanselmann	if self.request.resp_version == HTTP_1_1:
900	8a0b06d2	Michael Hanselmann	return (hdr_connection and "close" in hdr_connection)
901	8a0b06d2	Michael Hanselmann
902	8a0b06d2	Michael Hanselmann	# Some HTTP/1.0 implementations have support for persistent connections,
903	8a0b06d2	Michael Hanselmann	# using rules different than HTTP/1.1.
904	8a0b06d2	Michael Hanselmann
905	8a0b06d2	Michael Hanselmann	# For older HTTP, Keep-Alive indicates persistent connection.
906	8a0b06d2	Michael Hanselmann	if self.request.resp_headers.get(HTTP_KEEP_ALIVE):
907	8a0b06d2	Michael Hanselmann	return False
908	8a0b06d2	Michael Hanselmann
909	8a0b06d2	Michael Hanselmann	# At least Akamai returns a "Connection: Keep-Alive" header, which was
910	8a0b06d2	Michael Hanselmann	# supposed to be sent by the client.
911	8a0b06d2	Michael Hanselmann	if hdr_connection and "keep-alive" in hdr_connection:
912	8a0b06d2	Michael Hanselmann	return False
913	8a0b06d2	Michael Hanselmann
914	8a0b06d2	Michael Hanselmann	return True
915	8a0b06d2	Michael Hanselmann
916	8a0b06d2	Michael Hanselmann	def _ParseHeaders(self):
917	8a0b06d2	Michael Hanselmann	"""Parses the headers sent by the server.
918	8a0b06d2	Michael Hanselmann
919	8a0b06d2	Michael Hanselmann	This function also adjusts internal variables based on the header values.
920	8a0b06d2	Michael Hanselmann
921	8a0b06d2	Michael Hanselmann	"""
922	8a0b06d2	Michael Hanselmann	req = self.request
923	8a0b06d2	Michael Hanselmann
924	8a0b06d2	Michael Hanselmann	# Parse headers
925	8a0b06d2	Michael Hanselmann	self.header_buffer.seek(0, 0)
926	8a0b06d2	Michael Hanselmann	req.resp_headers = mimetools.Message(self.header_buffer, 0)
927	8a0b06d2	Michael Hanselmann
928	8a0b06d2	Michael Hanselmann	self.server_will_close = self._WillServerCloseConnection()
929	8a0b06d2	Michael Hanselmann
930	8a0b06d2	Michael Hanselmann	# Do we have a Content-Length header?
931	8a0b06d2	Michael Hanselmann	hdr_content_length = req.resp_headers.get(HTTP_CONTENT_LENGTH, None)
932	8a0b06d2	Michael Hanselmann	if hdr_content_length:
933	8a0b06d2	Michael Hanselmann	try:
934	8a0b06d2	Michael Hanselmann	self.content_length = int(hdr_content_length)
935	8a0b06d2	Michael Hanselmann	except ValueError:
936	8a0b06d2	Michael Hanselmann	pass
937	8a0b06d2	Michael Hanselmann	if self.content_length is not None and self.content_length < 0:
938	8a0b06d2	Michael Hanselmann	self.content_length = None
939	8a0b06d2	Michael Hanselmann
940	8a0b06d2	Michael Hanselmann	# does the body have a fixed length? (of zero)
941	8a0b06d2	Michael Hanselmann	if (req.resp_status in (HTTP_NO_CONTENT, HTTP_NOT_MODIFIED) or
942	8a0b06d2	Michael Hanselmann	100 <= req.resp_status < 200 or req.method == HTTP_HEAD):
943	8a0b06d2	Michael Hanselmann	self.content_length = 0
944	8a0b06d2	Michael Hanselmann
945	8a0b06d2	Michael Hanselmann	# if the connection remains open and a content-length was not provided,
946	8a0b06d2	Michael Hanselmann	# then assume that the connection WILL close.
947	8a0b06d2	Michael Hanselmann	if self.content_length is None:
948	8a0b06d2	Michael Hanselmann	self.server_will_close = True
949	8a0b06d2	Michael Hanselmann
950	8a0b06d2	Michael Hanselmann	def _CheckStatusLineLength(self, length):
951	8a0b06d2	Michael Hanselmann	if length > self.STATUS_LINE_LENGTH_MAX:
952	8a0b06d2	Michael Hanselmann	raise ResponseError("Status line longer than %d chars" %
953	8a0b06d2	Michael Hanselmann	self.STATUS_LINE_LENGTH_MAX)
954	8a0b06d2	Michael Hanselmann
955	8a0b06d2	Michael Hanselmann	def _CheckHeaderLength(self, length):
956	8a0b06d2	Michael Hanselmann	if length > self.HEADER_LENGTH_MAX:
957	8a0b06d2	Michael Hanselmann	raise ResponseError("Headers longer than %d chars" %
958	8a0b06d2	Michael Hanselmann	self.HEADER_LENGTH_MAX)
959	8a0b06d2	Michael Hanselmann
960	8a0b06d2	Michael Hanselmann	def _ParseBuffer(self, buf, eof):
961	8a0b06d2	Michael Hanselmann	"""Main function for HTTP response state machine.
962	8a0b06d2	Michael Hanselmann
963	8a0b06d2	Michael Hanselmann	@type buf: string
964	8a0b06d2	Michael Hanselmann	@param buf: Receive buffer
965	8a0b06d2	Michael Hanselmann	@type eof: bool
966	8a0b06d2	Michael Hanselmann	@param eof: Whether we've reached EOF on the socket
967	8a0b06d2	Michael Hanselmann	@rtype: string
968	8a0b06d2	Michael Hanselmann	@return: Updated receive buffer
969	8a0b06d2	Michael Hanselmann
970	8a0b06d2	Michael Hanselmann	"""
971	8a0b06d2	Michael Hanselmann	if self.parser_status == self.PS_STATUS_LINE:
972	8a0b06d2	Michael Hanselmann	# Expect status line
973	8a0b06d2	Michael Hanselmann	idx = buf.find("\r\n")
974	8a0b06d2	Michael Hanselmann	if idx >= 0:
975	8a0b06d2	Michael Hanselmann	self.request.resp_status_line = buf[:idx]
976	8a0b06d2	Michael Hanselmann
977	8a0b06d2	Michael Hanselmann	self._CheckStatusLineLength(len(self.request.resp_status_line))
978	8a0b06d2	Michael Hanselmann
979	8a0b06d2	Michael Hanselmann	# Remove status line, including CRLF
980	8a0b06d2	Michael Hanselmann	buf = buf[idx + 2:]
981	8a0b06d2	Michael Hanselmann
982	8a0b06d2	Michael Hanselmann	self._ParseStatusLine()
983	8a0b06d2	Michael Hanselmann
984	8a0b06d2	Michael Hanselmann	self.parser_status = self.PS_HEADERS
985	8a0b06d2	Michael Hanselmann	else:
986	8a0b06d2	Michael Hanselmann	# Check whether incoming data is getting too large, otherwise we just
987	8a0b06d2	Michael Hanselmann	# fill our read buffer.
988	8a0b06d2	Michael Hanselmann	self._CheckStatusLineLength(len(buf))
989	8a0b06d2	Michael Hanselmann
990	8a0b06d2	Michael Hanselmann	if self.parser_status == self.PS_HEADERS:
991	8a0b06d2	Michael Hanselmann	# Wait for header end
992	8a0b06d2	Michael Hanselmann	idx = buf.find("\r\n\r\n")
993	8a0b06d2	Michael Hanselmann	if idx >= 0:
994	8a0b06d2	Michael Hanselmann	self.header_buffer.write(buf[:idx + 2])
995	8a0b06d2	Michael Hanselmann
996	8a0b06d2	Michael Hanselmann	self._CheckHeaderLength(self.header_buffer.tell())
997	8a0b06d2	Michael Hanselmann
998	8a0b06d2	Michael Hanselmann	# Remove headers, including CRLF
999	8a0b06d2	Michael Hanselmann	buf = buf[idx + 4:]
1000	8a0b06d2	Michael Hanselmann
1001	8a0b06d2	Michael Hanselmann	self._ParseHeaders()
1002	8a0b06d2	Michael Hanselmann
1003	8a0b06d2	Michael Hanselmann	self.parser_status = self.PS_BODY
1004	8a0b06d2	Michael Hanselmann	else:
1005	8a0b06d2	Michael Hanselmann	# Check whether incoming data is getting too large, otherwise we just
1006	8a0b06d2	Michael Hanselmann	# fill our read buffer.
1007	8a0b06d2	Michael Hanselmann	self._CheckHeaderLength(len(buf))
1008	8a0b06d2	Michael Hanselmann
1009	8a0b06d2	Michael Hanselmann	if self.parser_status == self.PS_BODY:
1010	8a0b06d2	Michael Hanselmann	self.body_buffer.write(buf)
1011	8a0b06d2	Michael Hanselmann	buf = ""
1012	8a0b06d2	Michael Hanselmann
1013	8a0b06d2	Michael Hanselmann	# Check whether we've read everything
1014	8a0b06d2	Michael Hanselmann	if (eof or
1015	8a0b06d2	Michael Hanselmann	(self.content_length is not None and
1016	8a0b06d2	Michael Hanselmann	self.body_buffer.tell() >= self.content_length)):
1017	8a0b06d2	Michael Hanselmann	self.parser_status = self.PS_COMPLETE
1018	8a0b06d2	Michael Hanselmann
1019	8a0b06d2	Michael Hanselmann	return buf
1020	8a0b06d2	Michael Hanselmann
1021	8a0b06d2	Michael Hanselmann	def _WaitForCondition(self, event, timeout):
1022	8a0b06d2	Michael Hanselmann	"""Waits for a condition to occur on the socket.
1023	8a0b06d2	Michael Hanselmann
1024	8a0b06d2	Michael Hanselmann	@type event: int
1025	8a0b06d2	Michael Hanselmann	@param event: ORed condition (see select module)
1026	8a0b06d2	Michael Hanselmann	@type timeout: float or None
1027	8a0b06d2	Michael Hanselmann	@param timeout: Timeout in seconds
1028	8a0b06d2	Michael Hanselmann	@rtype: int or None
1029	8a0b06d2	Michael Hanselmann	@return: None for timeout, otherwise occured conditions
1030	8a0b06d2	Michael Hanselmann
1031	8a0b06d2	Michael Hanselmann	"""
1032	8a0b06d2	Michael Hanselmann	check = (event \| select.POLLPRI \|
1033	8a0b06d2	Michael Hanselmann	select.POLLNVAL \| select.POLLHUP \| select.POLLERR)
1034	8a0b06d2	Michael Hanselmann
1035	8a0b06d2	Michael Hanselmann	if timeout is not None:
1036	8a0b06d2	Michael Hanselmann	# Poller object expects milliseconds
1037	8a0b06d2	Michael Hanselmann	timeout *= 1000
1038	8a0b06d2	Michael Hanselmann
1039	8a0b06d2	Michael Hanselmann	self.poller.register(self.sock, event)
1040	8a0b06d2	Michael Hanselmann	try:
1041	8a0b06d2	Michael Hanselmann	while True:
1042	8a0b06d2	Michael Hanselmann	# TODO: If the main thread receives a signal and we have no timeout, we
1043	8a0b06d2	Michael Hanselmann	# could wait forever. This should check a global "quit" flag or
1044	8a0b06d2	Michael Hanselmann	# something every so often.
1045	8a0b06d2	Michael Hanselmann	io_events = self.poller.poll(timeout)
1046	8a0b06d2	Michael Hanselmann	if io_events:
1047	8a0b06d2	Michael Hanselmann	for (evfd, evcond) in io_events:
1048	8a0b06d2	Michael Hanselmann	if evcond & check:
1049	8a0b06d2	Michael Hanselmann	return evcond
1050	8a0b06d2	Michael Hanselmann	else:
1051	8a0b06d2	Michael Hanselmann	# Timeout
1052	8a0b06d2	Michael Hanselmann	return None
1053	8a0b06d2	Michael Hanselmann	finally:
1054	8a0b06d2	Michael Hanselmann	self.poller.unregister(self.sock)
1055	8a0b06d2	Michael Hanselmann
1056	8a0b06d2	Michael Hanselmann	def _Connect(self):
1057	8a0b06d2	Michael Hanselmann	"""Non-blocking connect to host with timeout.
1058	8a0b06d2	Michael Hanselmann
1059	8a0b06d2	Michael Hanselmann	"""
1060	8a0b06d2	Michael Hanselmann	connected = False
1061	8a0b06d2	Michael Hanselmann	while True:
1062	8a0b06d2	Michael Hanselmann	connect_error = self.sock.connect_ex((self.request.host,
1063	8a0b06d2	Michael Hanselmann	self.request.port))
1064	8a0b06d2	Michael Hanselmann	if connect_error == errno.EINTR:
1065	8a0b06d2	Michael Hanselmann	# Mask signals
1066	8a0b06d2	Michael Hanselmann	pass
1067	8a0b06d2	Michael Hanselmann
1068	8a0b06d2	Michael Hanselmann	elif connect_error == 0:
1069	8a0b06d2	Michael Hanselmann	# Connection established
1070	8a0b06d2	Michael Hanselmann	connected = True
1071	8a0b06d2	Michael Hanselmann	break
1072	8a0b06d2	Michael Hanselmann
1073	8a0b06d2	Michael Hanselmann	elif connect_error == errno.EINPROGRESS:
1074	8a0b06d2	Michael Hanselmann	# Connection started
1075	8a0b06d2	Michael Hanselmann	break
1076	8a0b06d2	Michael Hanselmann
1077	8a0b06d2	Michael Hanselmann	raise ResponseError("Connection failed (%s: %s)" %
1078	8a0b06d2	Michael Hanselmann	(connect_error, os.strerror(connect_error)))
1079	8a0b06d2	Michael Hanselmann
1080	8a0b06d2	Michael Hanselmann	if not connected:
1081	8a0b06d2	Michael Hanselmann	# Wait for connection
1082	8a0b06d2	Michael Hanselmann	event = self._WaitForCondition(select.POLLOUT, self.CONNECT_TIMEOUT)
1083	8a0b06d2	Michael Hanselmann	if event is None:
1084	8a0b06d2	Michael Hanselmann	raise ResponseError("Timeout while connecting to server")
1085	8a0b06d2	Michael Hanselmann
1086	8a0b06d2	Michael Hanselmann	# Get error code
1087	8a0b06d2	Michael Hanselmann	connect_error = self.sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
1088	8a0b06d2	Michael Hanselmann	if connect_error != 0:
1089	8a0b06d2	Michael Hanselmann	raise ResponseError("Connection failed (%s: %s)" %
1090	8a0b06d2	Michael Hanselmann	(connect_error, os.strerror(connect_error)))
1091	8a0b06d2	Michael Hanselmann
1092	8a0b06d2	Michael Hanselmann	# Enable TCP keep-alive
1093	8a0b06d2	Michael Hanselmann	self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
1094	8a0b06d2	Michael Hanselmann
1095	8a0b06d2	Michael Hanselmann	# If needed, Linux specific options are available to change the TCP
1096	8a0b06d2	Michael Hanselmann	# keep-alive settings, see "man 7 tcp" for TCP_KEEPCNT, TCP_KEEPIDLE and
1097	8a0b06d2	Michael Hanselmann	# TCP_KEEPINTVL.
1098	8a0b06d2	Michael Hanselmann
1099	8a0b06d2	Michael Hanselmann	def _SendRequest(self):
1100	8a0b06d2	Michael Hanselmann	"""Sends request to server.
1101	8a0b06d2	Michael Hanselmann
1102	8a0b06d2	Michael Hanselmann	"""
1103	8a0b06d2	Michael Hanselmann	buf = self._BuildRequest()
1104	8a0b06d2	Michael Hanselmann
1105	8a0b06d2	Michael Hanselmann	while buf:
1106	8a0b06d2	Michael Hanselmann	event = self._WaitForCondition(select.POLLOUT, self.WRITE_TIMEOUT)
1107	8a0b06d2	Michael Hanselmann	if event is None:
1108	8a0b06d2	Michael Hanselmann	raise ResponseError("Timeout while sending request")
1109	8a0b06d2	Michael Hanselmann
1110	8a0b06d2	Michael Hanselmann	try:
1111	8a0b06d2	Michael Hanselmann	# Only send 4 KB at a time
1112	8a0b06d2	Michael Hanselmann	data = buf[:4096]
1113	8a0b06d2	Michael Hanselmann
1114	8a0b06d2	Michael Hanselmann	sent = self.sock.send(data)
1115	8a0b06d2	Michael Hanselmann	except socket.error, err:
1116	8a0b06d2	Michael Hanselmann	if err.args and err.args[0] == errno.EAGAIN:
1117	8a0b06d2	Michael Hanselmann	# Ignore EAGAIN
1118	8a0b06d2	Michael Hanselmann	continue
1119	8a0b06d2	Michael Hanselmann
1120	8a0b06d2	Michael Hanselmann	raise ResponseError("Sending request failed: %s" % str(err))
1121	8a0b06d2	Michael Hanselmann
1122	8a0b06d2	Michael Hanselmann	# Remove sent bytes
1123	8a0b06d2	Michael Hanselmann	buf = buf[sent:]
1124	8a0b06d2	Michael Hanselmann
1125	8a0b06d2	Michael Hanselmann	assert not buf, "Request wasn't sent completely"
1126	8a0b06d2	Michael Hanselmann
1127	8a0b06d2	Michael Hanselmann	def _ReadResponse(self):
1128	8a0b06d2	Michael Hanselmann	"""Read response from server.
1129	8a0b06d2	Michael Hanselmann
1130	8a0b06d2	Michael Hanselmann	Calls the parser function after reading a chunk of data.
1131	8a0b06d2	Michael Hanselmann
1132	8a0b06d2	Michael Hanselmann	"""
1133	8a0b06d2	Michael Hanselmann	buf = ""
1134	8a0b06d2	Michael Hanselmann	eof = False
1135	8a0b06d2	Michael Hanselmann	while self.parser_status != self.PS_COMPLETE:
1136	8a0b06d2	Michael Hanselmann	event = self._WaitForCondition(select.POLLIN, self.READ_TIMEOUT)
1137	8a0b06d2	Michael Hanselmann	if event is None:
1138	8a0b06d2	Michael Hanselmann	raise ResponseError("Timeout while reading response")
1139	8a0b06d2	Michael Hanselmann
1140	8a0b06d2	Michael Hanselmann	if event & (select.POLLIN \| select.POLLPRI):
1141	8a0b06d2	Michael Hanselmann	try:
1142	8a0b06d2	Michael Hanselmann	data = self.sock.recv(4096)
1143	8a0b06d2	Michael Hanselmann	except socket.error, err:
1144	8a0b06d2	Michael Hanselmann	if err.args and err.args[0] == errno.EAGAIN:
1145	8a0b06d2	Michael Hanselmann	# Ignore EAGAIN
1146	8a0b06d2	Michael Hanselmann	continue
1147	8a0b06d2	Michael Hanselmann
1148	8a0b06d2	Michael Hanselmann	raise ResponseError("Reading response failed: %s" % str(err))
1149	8a0b06d2	Michael Hanselmann
1150	8a0b06d2	Michael Hanselmann	if data:
1151	8a0b06d2	Michael Hanselmann	buf += data
1152	8a0b06d2	Michael Hanselmann	else:
1153	8a0b06d2	Michael Hanselmann	eof = True
1154	8a0b06d2	Michael Hanselmann
1155	8a0b06d2	Michael Hanselmann	if event & (select.POLLNVAL \| select.POLLHUP \| select.POLLERR):
1156	8a0b06d2	Michael Hanselmann	eof = True
1157	8a0b06d2	Michael Hanselmann
1158	8a0b06d2	Michael Hanselmann	# Do some parsing and error checking while more data arrives
1159	8a0b06d2	Michael Hanselmann	buf = self._ParseBuffer(buf, eof)
1160	8a0b06d2	Michael Hanselmann
1161	8a0b06d2	Michael Hanselmann	# Must be done only after the buffer has been evaluated
1162	8a0b06d2	Michael Hanselmann	if (eof and
1163	8a0b06d2	Michael Hanselmann	self.parser_status in (self.PS_STATUS_LINE,
1164	8a0b06d2	Michael Hanselmann	self.PS_HEADERS)):
1165	8a0b06d2	Michael Hanselmann	raise ResponseError("Connection closed prematurely")
1166	8a0b06d2	Michael Hanselmann
1167	8a0b06d2	Michael Hanselmann	# Parse rest
1168	8a0b06d2	Michael Hanselmann	buf = self._ParseBuffer(buf, True)
1169	8a0b06d2	Michael Hanselmann
1170	8a0b06d2	Michael Hanselmann	assert self.parser_status == self.PS_COMPLETE
1171	8a0b06d2	Michael Hanselmann	assert not buf, "Parser didn't read full response"
1172	8a0b06d2	Michael Hanselmann
1173	8a0b06d2	Michael Hanselmann	def _CloseConnection(self, force):
1174	8a0b06d2	Michael Hanselmann	"""Closes the connection.
1175	8a0b06d2	Michael Hanselmann
1176	8a0b06d2	Michael Hanselmann	"""
1177	8a0b06d2	Michael Hanselmann	if self.server_will_close and not force:
1178	8a0b06d2	Michael Hanselmann	# Wait for server to close
1179	8a0b06d2	Michael Hanselmann	event = self._WaitForCondition(select.POLLIN, self.CLOSE_TIMEOUT)
1180	8a0b06d2	Michael Hanselmann	if event is None:
1181	8a0b06d2	Michael Hanselmann	# Server didn't close connection within CLOSE_TIMEOUT
1182	8a0b06d2	Michael Hanselmann	pass
1183	8a0b06d2	Michael Hanselmann	else:
1184	8a0b06d2	Michael Hanselmann	try:
1185	8a0b06d2	Michael Hanselmann	# Check whether it's actually closed
1186	8a0b06d2	Michael Hanselmann	if not self.sock.recv(1):
1187	8a0b06d2	Michael Hanselmann	return
1188	8a0b06d2	Michael Hanselmann	except socket.error, err:
1189	8a0b06d2	Michael Hanselmann	# Ignore errors at this stage
1190	8a0b06d2	Michael Hanselmann	pass
1191	8a0b06d2	Michael Hanselmann
1192	8a0b06d2	Michael Hanselmann	# Close the connection from our side
1193	8a0b06d2	Michael Hanselmann	self.sock.shutdown(socket.SHUT_RDWR)
1194	8a0b06d2	Michael Hanselmann
1195	8a0b06d2	Michael Hanselmann
1196	8a0b06d2	Michael Hanselmann	class HttpClientWorker(workerpool.BaseWorker):
1197	8a0b06d2	Michael Hanselmann	"""HTTP client worker class.
1198	8a0b06d2	Michael Hanselmann
1199	8a0b06d2	Michael Hanselmann	"""
1200	8a0b06d2	Michael Hanselmann	def RunTask(self, req):
1201	8a0b06d2	Michael Hanselmann	HttpClientRequestExecutor(req)
1202	8a0b06d2	Michael Hanselmann
1203	8a0b06d2	Michael Hanselmann
1204	8a0b06d2	Michael Hanselmann	class HttpClientWorkerPool(workerpool.WorkerPool):
1205	8a0b06d2	Michael Hanselmann	def __init__(self, manager):
1206	8a0b06d2	Michael Hanselmann	workerpool.WorkerPool.__init__(self, HTTP_CLIENT_THREADS,
1207	8a0b06d2	Michael Hanselmann	HttpClientWorker)
1208	8a0b06d2	Michael Hanselmann	self.manager = manager
1209	8a0b06d2	Michael Hanselmann
1210	8a0b06d2	Michael Hanselmann
1211	8a0b06d2	Michael Hanselmann	class HttpClientManager(object):
1212	8a0b06d2	Michael Hanselmann	def __init__(self):
1213	8a0b06d2	Michael Hanselmann	self._wpool = HttpClientWorkerPool(self)
1214	8a0b06d2	Michael Hanselmann
1215	8a0b06d2	Michael Hanselmann	def __del__(self):
1216	8a0b06d2	Michael Hanselmann	self.Shutdown()
1217	8a0b06d2	Michael Hanselmann
1218	8a0b06d2	Michael Hanselmann	def ExecRequests(self, requests):
1219	8a0b06d2	Michael Hanselmann	# Add requests to queue
1220	8a0b06d2	Michael Hanselmann	for req in requests:
1221	8a0b06d2	Michael Hanselmann	self._wpool.AddTask(req)
1222	8a0b06d2	Michael Hanselmann
1223	8a0b06d2	Michael Hanselmann	# And wait for them to finish
1224	8a0b06d2	Michael Hanselmann	self._wpool.Quiesce()
1225	8a0b06d2	Michael Hanselmann
1226	8a0b06d2	Michael Hanselmann	return requests
1227	8a0b06d2	Michael Hanselmann
1228	8a0b06d2	Michael Hanselmann	def Shutdown(self):
1229	8a0b06d2	Michael Hanselmann	self._wpool.TerminateWorkers()
1230	8a0b06d2	Michael Hanselmann
1231	8a0b06d2	Michael Hanselmann
1232	42242313	Michael Hanselmann	class _SSLFileObject(object):
1233	42242313	Michael Hanselmann	"""Wrapper around socket._fileobject
1234	42242313	Michael Hanselmann
1235	42242313	Michael Hanselmann	This wrapper is required to handle OpenSSL exceptions.
1236	42242313	Michael Hanselmann
1237	42242313	Michael Hanselmann	"""
1238	42242313	Michael Hanselmann	def _RequireOpenSocket(fn):
1239	42242313	Michael Hanselmann	def wrapper(self, args, *kwargs):
1240	42242313	Michael Hanselmann	if self.closed:
1241	42242313	Michael Hanselmann	raise SocketClosed("Socket is closed")
1242	42242313	Michael Hanselmann	return fn(self, args, *kwargs)
1243	42242313	Michael Hanselmann	return wrapper
1244	42242313	Michael Hanselmann
1245	42242313	Michael Hanselmann	def __init__(self, sock, mode='rb', bufsize=-1):
1246	42242313	Michael Hanselmann	self._base = socket._fileobject(sock, mode=mode, bufsize=bufsize)
1247	42242313	Michael Hanselmann
1248	42242313	Michael Hanselmann	def _ConnectionLost(self):
1249	42242313	Michael Hanselmann	self._base = None
1250	42242313	Michael Hanselmann
1251	42242313	Michael Hanselmann	def _getclosed(self):
1252	42242313	Michael Hanselmann	return self._base is None or self._base.closed
1253	42242313	Michael Hanselmann	closed = property(_getclosed, doc="True if the file is closed")
1254	42242313	Michael Hanselmann
1255	42242313	Michael Hanselmann	@_RequireOpenSocket
1256	42242313	Michael Hanselmann	def close(self):
1257	42242313	Michael Hanselmann	return self._base.close()
1258	42242313	Michael Hanselmann
1259	42242313	Michael Hanselmann	@_RequireOpenSocket
1260	42242313	Michael Hanselmann	def flush(self):
1261	42242313	Michael Hanselmann	return self._base.flush()
1262	42242313	Michael Hanselmann
1263	42242313	Michael Hanselmann	@_RequireOpenSocket
1264	42242313	Michael Hanselmann	def fileno(self):
1265	42242313	Michael Hanselmann	return self._base.fileno()
1266	42242313	Michael Hanselmann
1267	42242313	Michael Hanselmann	@_RequireOpenSocket
1268	42242313	Michael Hanselmann	def read(self, size=-1):
1269	42242313	Michael Hanselmann	return self._ReadWrapper(self._base.read, size=size)
1270	42242313	Michael Hanselmann
1271	42242313	Michael Hanselmann	@_RequireOpenSocket
1272	42242313	Michael Hanselmann	def readline(self, size=-1):
1273	42242313	Michael Hanselmann	return self._ReadWrapper(self._base.readline, size=size)
1274	42242313	Michael Hanselmann
1275	42242313	Michael Hanselmann	def _ReadWrapper(self, fn, args, *kwargs):
1276	42242313	Michael Hanselmann	while True:
1277	42242313	Michael Hanselmann	try:
1278	42242313	Michael Hanselmann	return fn(args, *kwargs)
1279	42242313	Michael Hanselmann
1280	42242313	Michael Hanselmann	except OpenSSL.SSL.ZeroReturnError, err:
1281	42242313	Michael Hanselmann	self._ConnectionLost()
1282	42242313	Michael Hanselmann	return ""
1283	42242313	Michael Hanselmann
1284	42242313	Michael Hanselmann	except OpenSSL.SSL.WantReadError:
1285	42242313	Michael Hanselmann	continue
1286	42242313	Michael Hanselmann
1287	42242313	Michael Hanselmann	#except OpenSSL.SSL.WantWriteError:
1288	42242313	Michael Hanselmann	# TODO
1289	42242313	Michael Hanselmann
1290	42242313	Michael Hanselmann	except OpenSSL.SSL.SysCallError, (retval, desc):
1291	d7bace1b	Michael Hanselmann	if ((retval == -1 and desc == _SSL_UNEXPECTED_EOF)
1292	42242313	Michael Hanselmann	or retval > 0):
1293	42242313	Michael Hanselmann	self._ConnectionLost()
1294	42242313	Michael Hanselmann	return ""
1295	42242313	Michael Hanselmann
1296	42242313	Michael Hanselmann	logging.exception("Error in OpenSSL")
1297	42242313	Michael Hanselmann	self._ConnectionLost()
1298	42242313	Michael Hanselmann	raise socket.error(err.args)
1299	42242313	Michael Hanselmann
1300	42242313	Michael Hanselmann	except OpenSSL.SSL.Error, err:
1301	42242313	Michael Hanselmann	self._ConnectionLost()
1302	42242313	Michael Hanselmann	raise socket.error(err.args)
1303	42242313	Michael Hanselmann
1304	42242313	Michael Hanselmann	@_RequireOpenSocket
1305	42242313	Michael Hanselmann	def write(self, data):
1306	42242313	Michael Hanselmann	return self._WriteWrapper(self._base.write, data)
1307	42242313	Michael Hanselmann
1308	42242313	Michael Hanselmann	def _WriteWrapper(self, fn, args, *kwargs):
1309	42242313	Michael Hanselmann	while True:
1310	42242313	Michael Hanselmann	try:
1311	42242313	Michael Hanselmann	return fn(args, *kwargs)
1312	42242313	Michael Hanselmann	except OpenSSL.SSL.ZeroReturnError, err:
1313	42242313	Michael Hanselmann	self._ConnectionLost()
1314	42242313	Michael Hanselmann	return 0
1315	42242313	Michael Hanselmann
1316	42242313	Michael Hanselmann	except OpenSSL.SSL.WantWriteError:
1317	42242313	Michael Hanselmann	continue
1318	42242313	Michael Hanselmann
1319	42242313	Michael Hanselmann	#except OpenSSL.SSL.WantReadError:
1320	42242313	Michael Hanselmann	# TODO
1321	42242313	Michael Hanselmann
1322	42242313	Michael Hanselmann	except OpenSSL.SSL.SysCallError, err:
1323	42242313	Michael Hanselmann	if err.args[0] == -1 and data == "":
1324	42242313	Michael Hanselmann	# errors when writing empty strings are expected
1325	42242313	Michael Hanselmann	# and can be ignored
1326	42242313	Michael Hanselmann	return 0
1327	42242313	Michael Hanselmann
1328	42242313	Michael Hanselmann	self._ConnectionLost()
1329	42242313	Michael Hanselmann	raise socket.error(err.args)
1330	42242313	Michael Hanselmann
1331	42242313	Michael Hanselmann	except OpenSSL.SSL.Error, err:
1332	42242313	Michael Hanselmann	self._ConnectionLost()
1333	42242313	Michael Hanselmann	raise socket.error(err.args)

Synnefo » snf-ganeti

root / lib / http.py @ cb4e8387