Revision b956618e pithos/api/util.py
b/pithos/api/util.py | ||
---|---|---|
1 |
# |
|
2 |
# Copyright (c) 2011 Greek Research and Technology Network |
|
3 |
# |
|
4 |
|
|
5 |
from functools import wraps |
|
6 |
|
|
7 |
from time import time |
|
8 |
from wsgiref.handlers import format_date_time |
|
9 |
|
|
10 |
from django.conf import settings |
|
11 |
from django.http import HttpResponse |
|
12 |
|
|
13 |
from pithos.api.faults import Fault, BadRequest, ServiceUnavailable |
|
14 |
|
|
15 |
import datetime |
|
16 |
import logging |
|
17 |
|
|
18 |
logger = logging.getLogger(__name__) |
|
19 |
|
|
20 |
def format_meta_key(k): |
|
21 |
""" |
|
22 |
Convert underscores to dashes and capitalize intra-dash strings. |
|
23 |
""" |
|
24 |
return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')]) |
|
25 |
|
|
26 |
def get_meta(request, prefix): |
|
27 |
""" |
|
28 |
Get all prefix-* request headers in a dict. Reformat keys with format_meta_key(). |
|
29 |
""" |
|
30 |
prefix = 'HTTP_' + prefix.upper().replace('-', '_') |
|
31 |
return dict([(format_meta_key(k[5:]), v) for k, v in request.META.iteritems() if k.startswith(prefix)]) |
|
32 |
|
|
33 |
def get_range(request): |
|
34 |
""" |
|
35 |
Parse a Range header from the request. |
|
36 |
Either returns None, or an (offset, length) tuple. |
|
37 |
If no offset is defined offset equals 0. |
|
38 |
If no length is defined length is None. |
|
39 |
""" |
|
40 |
|
|
41 |
range = request.GET.get('range') |
|
42 |
if not range: |
|
43 |
return None |
|
44 |
|
|
45 |
range = range.replace(' ', '') |
|
46 |
if not range.startswith('bytes='): |
|
47 |
return None |
|
48 |
|
|
49 |
parts = range.split('-') |
|
50 |
if len(parts) != 2: |
|
51 |
return None |
|
52 |
|
|
53 |
offset, length = parts |
|
54 |
if offset == '' and length == '': |
|
55 |
return None |
|
56 |
|
|
57 |
if offset != '': |
|
58 |
try: |
|
59 |
offset = int(offset) |
|
60 |
except ValueError: |
|
61 |
return None |
|
62 |
else: |
|
63 |
offset = 0 |
|
64 |
|
|
65 |
if length != '': |
|
66 |
try: |
|
67 |
length = int(length) |
|
68 |
except ValueError: |
|
69 |
return None |
|
70 |
else: |
|
71 |
length = None |
|
72 |
|
|
73 |
return (offset, length) |
|
74 |
|
|
75 |
def update_response_headers(request, response): |
|
76 |
if request.serialization == 'xml': |
|
77 |
response['Content-Type'] = 'application/xml; charset=UTF-8' |
|
78 |
elif request.serialization == 'json': |
|
79 |
response['Content-Type'] = 'application/json; charset=UTF-8' |
|
80 |
else: |
|
81 |
response['Content-Type'] = 'text/plain; charset=UTF-8' |
|
82 |
|
|
83 |
if settings.TEST: |
|
84 |
response['Date'] = format_date_time(time()) |
|
85 |
|
|
86 |
def render_fault(request, fault): |
|
87 |
response = HttpResponse(status = fault.code) |
|
88 |
update_response_headers(request, response) |
|
89 |
return response |
|
90 |
|
|
91 |
def request_serialization(request, format_allowed=False): |
|
92 |
""" |
|
93 |
Return the serialization format requested. |
|
94 |
|
|
95 |
Valid formats are 'text' and 'json', 'xml' if `format_allowed` is True. |
|
96 |
""" |
|
97 |
|
|
98 |
if not format_allowed: |
|
99 |
return 'text' |
|
100 |
|
|
101 |
format = request.GET.get('format') |
|
102 |
if format == 'json': |
|
103 |
return 'json' |
|
104 |
elif format == 'xml': |
|
105 |
return 'xml' |
|
106 |
|
|
107 |
for item in request.META.get('HTTP_ACCEPT', '').split(','): |
|
108 |
accept, sep, rest = item.strip().partition(';') |
|
109 |
if accept == 'text/plain': |
|
110 |
return 'text' |
|
111 |
elif accept == 'application/json': |
|
112 |
return 'json' |
|
113 |
elif accept == 'application/xml' or accept == 'text/xml': |
|
114 |
return 'xml' |
|
115 |
|
|
116 |
return 'text' |
|
117 |
|
|
118 |
def api_method(http_method = None, format_allowed = False): |
|
119 |
""" |
|
120 |
Decorator function for views that implement an API method. |
|
121 |
""" |
|
122 |
|
|
123 |
def decorator(func): |
|
124 |
@wraps(func) |
|
125 |
def wrapper(request, *args, **kwargs): |
|
126 |
try: |
|
127 |
if http_method and request.method != http_method: |
|
128 |
raise BadRequest('Method not allowed.') |
|
129 |
|
|
130 |
# The args variable may contain up to (account, container, object). |
|
131 |
if len(args) > 1 and len(args[1]) > 256: |
|
132 |
raise BadRequest('Container name too large.') |
|
133 |
if len(args) > 2 and len(args[2]) > 1024: |
|
134 |
raise BadRequest('Object name too large.') |
|
135 |
|
|
136 |
# Fill in custom request variables. |
|
137 |
request.serialization = request_serialization(request, format_allowed) |
|
138 |
# TODO: Authenticate. |
|
139 |
request.user = "test" |
|
140 |
|
|
141 |
response = func(request, *args, **kwargs) |
|
142 |
update_response_headers(request, response) |
|
143 |
return response |
|
144 |
except Fault, fault: |
|
145 |
return render_fault(request, fault) |
|
146 |
except BaseException, e: |
|
147 |
logger.exception('Unexpected error: %s' % e) |
|
148 |
fault = ServiceUnavailable('Unexpected error') |
|
149 |
return render_fault(request, fault) |
|
150 |
return wrapper |
|
151 |
return decorator |
|
1 |
from functools import wraps |
|
2 |
from time import time |
|
3 |
from traceback import format_exc |
|
4 |
from wsgiref.handlers import format_date_time |
|
5 |
|
|
6 |
from django.conf import settings |
|
7 |
from django.http import HttpResponse |
|
8 |
from django.utils.http import http_date |
|
9 |
|
|
10 |
from pithos.api.compat import parse_http_date_safe |
|
11 |
from pithos.api.faults import (Fault, NotModified, BadRequest, ItemNotFound, PreconditionFailed, |
|
12 |
ServiceUnavailable) |
|
13 |
from pithos.backends import backend |
|
14 |
|
|
15 |
import datetime |
|
16 |
import logging |
|
17 |
|
|
18 |
|
|
19 |
logger = logging.getLogger(__name__) |
|
20 |
|
|
21 |
|
|
22 |
def printable_meta_dict(d): |
|
23 |
"""Format a meta dictionary for printing out json/xml. |
|
24 |
|
|
25 |
Convert all keys to lower case and replace dashes to underscores. |
|
26 |
Change 'modified' key from backend to 'last_modified' and format date. |
|
27 |
""" |
|
28 |
if 'modified' in d: |
|
29 |
d['last_modified'] = datetime.datetime.fromtimestamp(int(d['modified'])).isoformat() |
|
30 |
del(d['modified']) |
|
31 |
return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()]) |
|
32 |
|
|
33 |
def format_meta_key(k): |
|
34 |
"""Convert underscores to dashes and capitalize intra-dash strings""" |
|
35 |
return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')]) |
|
36 |
|
|
37 |
def get_meta_prefix(request, prefix): |
|
38 |
"""Get all prefix-* request headers in a dict. Reformat keys with format_meta_key()""" |
|
39 |
prefix = 'HTTP_' + prefix.upper().replace('-', '_') |
|
40 |
return dict([(format_meta_key(k[5:]), v) for k, v in request.META.iteritems() if k.startswith(prefix)]) |
|
41 |
|
|
42 |
def get_account_meta(request): |
|
43 |
"""Get metadata from an account request""" |
|
44 |
meta = get_meta_prefix(request, 'X-Account-Meta-') |
|
45 |
return meta |
|
46 |
|
|
47 |
def put_account_meta(response, meta): |
|
48 |
"""Put metadata in an account response""" |
|
49 |
response['X-Account-Container-Count'] = meta['count'] |
|
50 |
response['X-Account-Bytes-Used'] = meta['bytes'] |
|
51 |
if 'modified' in meta: |
|
52 |
response['Last-Modified'] = http_date(int(meta['modified'])) |
|
53 |
for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]: |
|
54 |
response[k.encode('utf-8')] = meta[k].encode('utf-8') |
|
55 |
|
|
56 |
def get_container_meta(request): |
|
57 |
"""Get metadata from a container request""" |
|
58 |
meta = get_meta_prefix(request, 'X-Container-Meta-') |
|
59 |
return meta |
|
60 |
|
|
61 |
def put_container_meta(response, meta): |
|
62 |
"""Put metadata in a container response""" |
|
63 |
response['X-Container-Object-Count'] = meta['count'] |
|
64 |
response['X-Container-Bytes-Used'] = meta['bytes'] |
|
65 |
if 'modified' in meta: |
|
66 |
response['Last-Modified'] = http_date(int(meta['modified'])) |
|
67 |
for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]: |
|
68 |
response[k.encode('utf-8')] = meta[k].encode('utf-8') |
|
69 |
|
|
70 |
def get_object_meta(request): |
|
71 |
"""Get metadata from an object request""" |
|
72 |
meta = get_meta_prefix(request, 'X-Object-Meta-') |
|
73 |
if request.META.get('CONTENT_TYPE'): |
|
74 |
meta['Content-Type'] = request.META['CONTENT_TYPE'] |
|
75 |
if request.META.get('HTTP_CONTENT_ENCODING'): |
|
76 |
meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING'] |
|
77 |
if request.META.get('HTTP_X_OBJECT_MANIFEST'): |
|
78 |
meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST'] |
|
79 |
return meta |
|
80 |
|
|
81 |
def put_object_meta(response, meta): |
|
82 |
"""Put metadata in an object response""" |
|
83 |
response['ETag'] = meta['hash'] |
|
84 |
response['Content-Length'] = meta['bytes'] |
|
85 |
response['Content-Type'] = meta.get('Content-Type', 'application/octet-stream') |
|
86 |
response['Last-Modified'] = http_date(int(meta['modified'])) |
|
87 |
for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]: |
|
88 |
response[k.encode('utf-8')] = meta[k].encode('utf-8') |
|
89 |
for k in ('Content-Encoding', 'X-Object-Manifest'): |
|
90 |
if k in meta: |
|
91 |
response[k] = meta[k] |
|
92 |
|
|
93 |
def validate_modification_preconditions(request, meta): |
|
94 |
"""Check that the modified timestamp conforms with the preconditions set""" |
|
95 |
if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE') |
|
96 |
if if_modified_since is not None: |
|
97 |
if_modified_since = parse_http_date_safe(if_modified_since) |
|
98 |
if if_modified_since is not None and 'modified' in meta and int(meta['modified']) <= if_modified_since: |
|
99 |
raise NotModified('Object has not been modified') |
|
100 |
|
|
101 |
if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE') |
|
102 |
if if_unmodified_since is not None: |
|
103 |
if_unmodified_since = parse_http_date_safe(if_unmodified_since) |
|
104 |
if if_unmodified_since is not None and 'modified' in meta and int(meta['modified']) > if_unmodified_since: |
|
105 |
raise PreconditionFailed('Object has been modified') |
|
106 |
|
|
107 |
def copy_or_move_object(request, src_path, dest_path, move=False): |
|
108 |
"""Copy or move an object""" |
|
109 |
if type(src_path) == str: |
|
110 |
parts = src_path.split('/') |
|
111 |
if len(parts) < 3 or parts[0] != '': |
|
112 |
raise BadRequest('Invalid X-Copy-From or X-Move-From header') |
|
113 |
src_container = parts[1] |
|
114 |
src_name = '/'.join(parts[2:]) |
|
115 |
elif type(src_path) == tuple and len(src_path) == 2: |
|
116 |
src_container, src_name = src_path |
|
117 |
|
|
118 |
if type(dest_path) == str: |
|
119 |
parts = dest_path.split('/') |
|
120 |
if len(parts) < 3 or parts[0] != '': |
|
121 |
raise BadRequest('Invalid Destination header') |
|
122 |
dest_container = parts[1] |
|
123 |
dest_name = '/'.join(parts[2:]) |
|
124 |
elif type(dest_path) == tuple and len(dest_path) == 2: |
|
125 |
dest_container, dest_name = dest_path |
|
126 |
|
|
127 |
meta = get_object_meta(request) |
|
128 |
try: |
|
129 |
if move: |
|
130 |
backend.move_object(request.user, src_container, src_name, dest_container, dest_name, meta) |
|
131 |
else: |
|
132 |
backend.copy_object(request.user, src_container, src_name, dest_container, dest_name, meta) |
|
133 |
except NameError: |
|
134 |
raise ItemNotFound('Container or object does not exist') |
|
135 |
|
|
136 |
def get_range(request): |
|
137 |
"""Parse a Range header from the request |
|
138 |
|
|
139 |
Either returns None, or an (offset, length) tuple. |
|
140 |
If no length is defined length is None. |
|
141 |
May return a negative offset (offset from the end). |
|
142 |
""" |
|
143 |
range = request.META.get('HTTP_RANGE', '').replace(' ', '') |
|
144 |
if not range.startswith('bytes='): |
|
145 |
return None |
|
146 |
|
|
147 |
parts = range[6:].split('-') |
|
148 |
if len(parts) != 2: |
|
149 |
return None |
|
150 |
|
|
151 |
offset, upto = parts |
|
152 |
if offset == '' and upto == '': |
|
153 |
return None |
|
154 |
if offset != '': |
|
155 |
try: |
|
156 |
offset = int(offset) |
|
157 |
except ValueError: |
|
158 |
return None |
|
159 |
|
|
160 |
if upto != '': |
|
161 |
try: |
|
162 |
upto = int(upto) |
|
163 |
except ValueError: |
|
164 |
return None |
|
165 |
else: |
|
166 |
return (offset, None) |
|
167 |
|
|
168 |
if offset > upto: |
|
169 |
return None |
|
170 |
return (offset, upto - offset + 1) |
|
171 |
else: |
|
172 |
try: |
|
173 |
offset = -int(upto) |
|
174 |
except ValueError: |
|
175 |
return None |
|
176 |
return (offset, None) |
|
177 |
|
|
178 |
def raw_input_socket(request): |
|
179 |
"""Return the socket for reading the rest of the request""" |
|
180 |
server_software = request.META.get('SERVER_SOFTWARE') |
|
181 |
if not server_software: |
|
182 |
if 'wsgi.input' in request.environ: |
|
183 |
return request.environ['wsgi.input'] |
|
184 |
raise ServiceUnavailable('Unknown server software') |
|
185 |
if server_software.startswith('WSGIServer'): |
|
186 |
return request.environ['wsgi.input'] |
|
187 |
elif server_software.startswith('mod_python'): |
|
188 |
return request._req |
|
189 |
raise ServiceUnavailable('Unknown server software') |
|
190 |
|
|
191 |
MAX_UPLOAD_SIZE = 10 * (1024 * 1024) # 10MB |
|
192 |
|
|
193 |
def socket_read_iterator(sock, length=-1, blocksize=4096): |
|
194 |
"""Return a maximum of blocksize data read from the socket in each iteration |
|
195 |
|
|
196 |
Read up to 'length'. If no 'length' is defined, will attempt a chunked read. |
|
197 |
The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE. |
|
198 |
""" |
|
199 |
if length < 0: # Chunked transfers |
|
200 |
while length < MAX_UPLOAD_SIZE: |
|
201 |
chunk_length = sock.readline() |
|
202 |
pos = chunk_length.find(';') |
|
203 |
if pos >= 0: |
|
204 |
chunk_length = chunk_length[:pos] |
|
205 |
try: |
|
206 |
chunk_length = int(chunk_length, 16) |
|
207 |
except Exception, e: |
|
208 |
raise BadRequest('Bad chunk size') # TODO: Change to something more appropriate. |
|
209 |
if chunk_length == 0: |
|
210 |
return |
|
211 |
while chunk_length > 0: |
|
212 |
data = sock.read(min(chunk_length, blocksize)) |
|
213 |
chunk_length -= len(data) |
|
214 |
length += len(data) |
|
215 |
yield data |
|
216 |
data = sock.read(2) # CRLF |
|
217 |
# TODO: Raise something to note that maximum size is reached. |
|
218 |
else: |
|
219 |
if length > MAX_UPLOAD_SIZE: |
|
220 |
# TODO: Raise something to note that maximum size is reached. |
|
221 |
pass |
|
222 |
while length > 0: |
|
223 |
data = sock.read(min(length, blocksize)) |
|
224 |
length -= len(data) |
|
225 |
yield data |
|
226 |
|
|
227 |
def update_response_headers(request, response): |
|
228 |
if request.serialization == 'xml': |
|
229 |
response['Content-Type'] = 'application/xml; charset=UTF-8' |
|
230 |
elif request.serialization == 'json': |
|
231 |
response['Content-Type'] = 'application/json; charset=UTF-8' |
|
232 |
else: |
|
233 |
response['Content-Type'] = 'text/plain; charset=UTF-8' |
|
234 |
|
|
235 |
if settings.TEST: |
|
236 |
response['Date'] = format_date_time(time()) |
|
237 |
|
|
238 |
def render_fault(request, fault): |
|
239 |
if settings.DEBUG or settings.TEST: |
|
240 |
fault.details = format_exc(fault) |
|
241 |
|
|
242 |
request.serialization = 'text' |
|
243 |
data = '\n'.join((fault.message, fault.details)) + '\n' |
|
244 |
response = HttpResponse(data, status=fault.code) |
|
245 |
update_response_headers(request, response) |
|
246 |
return response |
|
247 |
|
|
248 |
def request_serialization(request, format_allowed=False): |
|
249 |
"""Return the serialization format requested |
|
250 |
|
|
251 |
Valid formats are 'text' and 'json', 'xml' if 'format_allowed' is True. |
|
252 |
""" |
|
253 |
if not format_allowed: |
|
254 |
return 'text' |
|
255 |
|
|
256 |
format = request.GET.get('format') |
|
257 |
if format == 'json': |
|
258 |
return 'json' |
|
259 |
elif format == 'xml': |
|
260 |
return 'xml' |
|
261 |
|
|
262 |
for item in request.META.get('HTTP_ACCEPT', '').split(','): |
|
263 |
accept, sep, rest = item.strip().partition(';') |
|
264 |
if accept == 'text/plain': |
|
265 |
return 'text' |
|
266 |
elif accept == 'application/json': |
|
267 |
return 'json' |
|
268 |
elif accept == 'application/xml' or accept == 'text/xml': |
|
269 |
return 'xml' |
|
270 |
|
|
271 |
return 'text' |
|
272 |
|
|
273 |
def api_method(http_method=None, format_allowed=False): |
|
274 |
"""Decorator function for views that implement an API method""" |
|
275 |
def decorator(func): |
|
276 |
@wraps(func) |
|
277 |
def wrapper(request, *args, **kwargs): |
|
278 |
try: |
|
279 |
if http_method and request.method != http_method: |
|
280 |
raise BadRequest('Method not allowed.') |
|
281 |
|
|
282 |
# The args variable may contain up to (account, container, object). |
|
283 |
if len(args) > 1 and len(args[1]) > 256: |
|
284 |
raise BadRequest('Container name too large.') |
|
285 |
if len(args) > 2 and len(args[2]) > 1024: |
|
286 |
raise BadRequest('Object name too large.') |
|
287 |
|
|
288 |
# Fill in custom request variables. |
|
289 |
request.serialization = request_serialization(request, format_allowed) |
|
290 |
# TODO: Authenticate. |
|
291 |
request.user = "test" |
|
292 |
|
|
293 |
response = func(request, *args, **kwargs) |
|
294 |
update_response_headers(request, response) |
|
295 |
return response |
|
296 |
except Fault, fault: |
|
297 |
return render_fault(request, fault) |
|
298 |
except BaseException, e: |
|
299 |
logger.exception('Unexpected error: %s' % e) |
|
300 |
fault = ServiceUnavailable('Unexpected error') |
|
301 |
return render_fault(request, fault) |
|
302 |
return wrapper |
|
303 |
return decorator |
Also available in: Unified diff