f1488b8a0b8ccd62042b49455bc17062da5497b3
[pithos] / snf-pithos-app / pithos / api / functions.py
1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
2
3 # Redistribution and use in source and binary forms, with or
4 # without modification, are permitted provided that the following
5 # conditions are met:
6
7 #   1. Redistributions of source code must retain the above
8 #      copyright notice, this list of conditions and the following
9 #      disclaimer.
10
11 #   2. Redistributions in binary form must reproduce the above
12 #      copyright notice, this list of conditions and the following
13 #      disclaimer in the documentation and/or other materials
14 #      provided with the distribution.
15
16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 # POSSIBILITY OF SUCH DAMAGE.
28
29 # The views and conclusions contained in the software and
30 # documentation are those of the authors and should not be
31 # interpreted as representing official policies, either expressed
32 # or implied, of GRNET S.A.
33
34 from xml.dom import minidom
35 from urllib import unquote
36
37 from django.conf import settings
38 from django.http import HttpResponse
39 from django.template.loader import render_to_string
40 from django.utils import simplejson as json
41 from django.utils.http import parse_etags
42 from django.utils.encoding import smart_str
43 from django.views.decorators.csrf import csrf_exempt
44
45 from synnefo.lib.astakos import get_user
46
47 from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
48     LengthRequired, PreconditionFailed, RequestEntityTooLarge, RangeNotSatisfiable, UnprocessableEntity)
49 from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
50     get_account_headers, put_account_headers, get_container_headers, put_container_headers, get_object_headers,
51     put_object_headers, update_manifest_meta, update_sharing_meta, update_public_meta,
52     validate_modification_preconditions, validate_matching_preconditions, split_container_object_string,
53     copy_or_move_object, get_int_parameter, get_content_length, get_content_range, socket_read_iterator,
54     SaveToBackendHandler, object_data_response, put_object_block, hashmap_md5, simple_list_response, api_method)
55 from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS, COOKIE_NAME, UPDATE_MD5
56
57 from pithos.backends.base import NotAllowedError, QuotaError
58 from pithos.backends.filter import parse_filters
59
60 import logging
61 import hashlib
62
63
64 logger = logging.getLogger(__name__)
65
66
67 @csrf_exempt
68 def top_demux(request):
69     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
70     if request.method == 'GET':
71         if getattr(request, 'user', None) is not None:
72             return account_list(request)
73         return authenticate(request)
74     else:
75         return method_not_allowed(request)
76
77 @csrf_exempt
78 def account_demux(request, v_account):
79     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
80     if request.method == 'HEAD':
81         return account_meta(request, v_account)
82     elif request.method == 'POST':
83         return account_update(request, v_account)
84     elif request.method == 'GET':
85         return container_list(request, v_account)
86     else:
87         return method_not_allowed(request)
88
89 @csrf_exempt
90 def container_demux(request, v_account, v_container):
91     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
92     if request.method == 'HEAD':
93         return container_meta(request, v_account, v_container)
94     elif request.method == 'PUT':
95         return container_create(request, v_account, v_container)
96     elif request.method == 'POST':
97         return container_update(request, v_account, v_container)
98     elif request.method == 'DELETE':
99         return container_delete(request, v_account, v_container)
100     elif request.method == 'GET':
101         return object_list(request, v_account, v_container)
102     else:
103         return method_not_allowed(request)
104
105 @csrf_exempt
106 def object_demux(request, v_account, v_container, v_object):
107     # Helper to avoid placing the token in the URL when loading objects from a browser.
108     token = None
109     if request.method in ('HEAD', 'GET') and COOKIE_NAME in request.COOKIES:
110         cookie_value = unquote(request.COOKIES.get(COOKIE_NAME, ''))
111         if cookie_value and '|' in cookie_value:
112             token = cookie_value.split('|', 1)[1]
113     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS, token)
114     if request.method == 'HEAD':
115         return object_meta(request, v_account, v_container, v_object)
116     elif request.method == 'GET':
117         return object_read(request, v_account, v_container, v_object)
118     elif request.method == 'PUT':
119         return object_write(request, v_account, v_container, v_object)
120     elif request.method == 'COPY':
121         return object_copy(request, v_account, v_container, v_object)
122     elif request.method == 'MOVE':
123         return object_move(request, v_account, v_container, v_object)
124     elif request.method == 'POST':
125         if request.META.get('CONTENT_TYPE', '').startswith('multipart/form-data'):
126             return object_write_form(request, v_account, v_container, v_object)
127         return object_update(request, v_account, v_container, v_object)
128     elif request.method == 'DELETE':
129         return object_delete(request, v_account, v_container, v_object)
130     else:
131         return method_not_allowed(request)
132
133 @api_method('GET', user_required=False)
134 def authenticate(request):
135     # Normal Response Codes: 204
136     # Error Response Codes: internalServerError (500),
137     #                       forbidden (403),
138     #                       badRequest (400)
139     
140     x_auth_user = request.META.get('HTTP_X_AUTH_USER')
141     x_auth_key = request.META.get('HTTP_X_AUTH_KEY')
142     if not x_auth_user or not x_auth_key:
143         raise BadRequest('Missing X-Auth-User or X-Auth-Key header')
144     response = HttpResponse(status=204)
145     
146     uri = request.build_absolute_uri()
147     if '?' in uri:
148         uri = uri[:uri.find('?')]
149     
150     response['X-Auth-Token'] = x_auth_key
151     response['X-Storage-Url'] = uri + ('' if uri.endswith('/') else '/') + x_auth_user
152     return response
153
154 @api_method('GET', format_allowed=True)
155 def account_list(request):
156     # Normal Response Codes: 200, 204
157     # Error Response Codes: internalServerError (500),
158     #                       badRequest (400)
159     
160     response = HttpResponse()
161     
162     marker = request.GET.get('marker')
163     limit = get_int_parameter(request.GET.get('limit'))
164     if not limit:
165         limit = 10000
166     
167     accounts = request.backend.list_accounts(request.user_uniq, marker, limit)
168     
169     if request.serialization == 'text':
170         if len(accounts) == 0:
171             # The cloudfiles python bindings expect 200 if json/xml.
172             response.status_code = 204
173             return response
174         response.status_code = 200
175         response.content = '\n'.join(accounts) + '\n'
176         return response
177     
178     account_meta = []
179     for x in accounts:
180         if x == request.user_uniq:
181             continue
182         try:
183             meta = request.backend.get_account_meta(request.user_uniq, x, 'pithos', include_user_defined=False)
184             groups = request.backend.get_account_groups(request.user_uniq, x)
185         except NotAllowedError:
186             raise Forbidden('Not allowed')
187         else:
188             rename_meta_key(meta, 'modified', 'last_modified')
189             rename_meta_key(meta, 'until_timestamp', 'x_account_until_timestamp')
190             if groups:
191                 meta['X-Account-Group'] = printable_header_dict(dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
192             account_meta.append(printable_header_dict(meta))
193     if request.serialization == 'xml':
194         data = render_to_string('accounts.xml', {'accounts': account_meta})
195     elif request.serialization  == 'json':
196         data = json.dumps(account_meta)
197     response.status_code = 200
198     response.content = data
199     return response
200
201 @api_method('HEAD')
202 def account_meta(request, v_account):
203     # Normal Response Codes: 204
204     # Error Response Codes: internalServerError (500),
205     #                       forbidden (403),
206     #                       badRequest (400)
207     
208     until = get_int_parameter(request.GET.get('until'))
209     try:
210         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
211         groups = request.backend.get_account_groups(request.user_uniq, v_account)
212         policy = request.backend.get_account_policy(request.user_uniq, v_account)
213     except NotAllowedError:
214         raise Forbidden('Not allowed')
215     
216     validate_modification_preconditions(request, meta)
217     
218     response = HttpResponse(status=204)
219     put_account_headers(response, meta, groups, policy)
220     return response
221
222 @api_method('POST')
223 def account_update(request, v_account):
224     # Normal Response Codes: 202
225     # Error Response Codes: internalServerError (500),
226     #                       forbidden (403),
227     #                       badRequest (400)
228     
229     meta, groups = get_account_headers(request)
230     replace = True
231     if 'update' in request.GET:
232         replace = False
233     if groups:
234         try:
235             request.backend.update_account_groups(request.user_uniq, v_account,
236                                                     groups, replace)
237         except NotAllowedError:
238             raise Forbidden('Not allowed')
239         except ValueError:
240             raise BadRequest('Invalid groups header')
241     if meta or replace:
242         try:
243             request.backend.update_account_meta(request.user_uniq, v_account,
244                                                 'pithos', meta, replace)
245         except NotAllowedError:
246             raise Forbidden('Not allowed')
247     return HttpResponse(status=202)
248
249 @api_method('GET', format_allowed=True)
250 def container_list(request, v_account):
251     # Normal Response Codes: 200, 204
252     # Error Response Codes: internalServerError (500),
253     #                       itemNotFound (404),
254     #                       forbidden (403),
255     #                       badRequest (400)
256     
257     until = get_int_parameter(request.GET.get('until'))
258     try:
259         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
260         groups = request.backend.get_account_groups(request.user_uniq, v_account)
261         policy = request.backend.get_account_policy(request.user_uniq, v_account)
262     except NotAllowedError:
263         raise Forbidden('Not allowed')
264     
265     validate_modification_preconditions(request, meta)
266     
267     response = HttpResponse()
268     put_account_headers(response, meta, groups, policy)
269     
270     marker = request.GET.get('marker')
271     limit = get_int_parameter(request.GET.get('limit'))
272     if not limit:
273         limit = 10000
274     
275     shared = False
276     if 'shared' in request.GET:
277         shared = True
278     public = False
279     if 'public' in request.GET:
280         public = True
281     
282     try:
283         containers = request.backend.list_containers(request.user_uniq, v_account,
284                                                 marker, limit, shared, until, public)
285     except NotAllowedError:
286         raise Forbidden('Not allowed')
287     except NameError:
288         containers = []
289     
290     if request.serialization == 'text':
291         if len(containers) == 0:
292             # The cloudfiles python bindings expect 200 if json/xml.
293             response.status_code = 204
294             return response
295         response.status_code = 200
296         response.content = '\n'.join(containers) + '\n'
297         return response
298     
299     container_meta = []
300     for x in containers:
301         try:
302             meta = request.backend.get_container_meta(request.user_uniq, v_account,
303                                                         x, 'pithos', until, include_user_defined=False)
304             policy = request.backend.get_container_policy(request.user_uniq,
305                                                             v_account, x)
306         except NotAllowedError:
307             raise Forbidden('Not allowed')
308         except NameError:
309             pass
310         else:
311             rename_meta_key(meta, 'modified', 'last_modified')
312             rename_meta_key(meta, 'until_timestamp', 'x_container_until_timestamp')
313             if policy:
314                 meta['X-Container-Policy'] = printable_header_dict(dict([(k, v) for k, v in policy.iteritems()]))
315             container_meta.append(printable_header_dict(meta))
316     if request.serialization == 'xml':
317         data = render_to_string('containers.xml', {'account': v_account, 'containers': container_meta})
318     elif request.serialization  == 'json':
319         data = json.dumps(container_meta)
320     response.status_code = 200
321     response.content = data
322     return response
323
324 @api_method('HEAD')
325 def container_meta(request, v_account, v_container):
326     # Normal Response Codes: 204
327     # Error Response Codes: internalServerError (500),
328     #                       itemNotFound (404),
329     #                       forbidden (403),
330     #                       badRequest (400)
331     
332     until = get_int_parameter(request.GET.get('until'))
333     try:
334         meta = request.backend.get_container_meta(request.user_uniq, v_account,
335                                                     v_container, 'pithos', until)
336         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
337                                                 v_account, v_container, 'pithos', until)
338         policy = request.backend.get_container_policy(request.user_uniq, v_account,
339                                                         v_container)
340     except NotAllowedError:
341         raise Forbidden('Not allowed')
342     except NameError:
343         raise ItemNotFound('Container does not exist')
344     
345     validate_modification_preconditions(request, meta)
346     
347     response = HttpResponse(status=204)
348     put_container_headers(request, response, meta, policy)
349     return response
350
351 @api_method('PUT')
352 def container_create(request, v_account, v_container):
353     # Normal Response Codes: 201, 202
354     # Error Response Codes: internalServerError (500),
355     #                       itemNotFound (404),
356     #                       forbidden (403),
357     #                       badRequest (400)
358     
359     meta, policy = get_container_headers(request)
360     
361     try:
362         request.backend.put_container(request.user_uniq, v_account, v_container, policy)
363         ret = 201
364     except NotAllowedError:
365         raise Forbidden('Not allowed')
366     except ValueError:
367         raise BadRequest('Invalid policy header')
368     except NameError:
369         ret = 202
370     
371     if ret == 202 and policy:
372         try:
373             request.backend.update_container_policy(request.user_uniq, v_account,
374                                             v_container, policy, replace=False)
375         except NotAllowedError:
376             raise Forbidden('Not allowed')
377         except NameError:
378             raise ItemNotFound('Container does not exist')
379         except ValueError:
380             raise BadRequest('Invalid policy header')
381     if meta:
382         try:
383             request.backend.update_container_meta(request.user_uniq, v_account,
384                                             v_container, 'pithos', meta, replace=False)
385         except NotAllowedError:
386             raise Forbidden('Not allowed')
387         except NameError:
388             raise ItemNotFound('Container does not exist')
389     
390     return HttpResponse(status=ret)
391
392 @api_method('POST', format_allowed=True)
393 def container_update(request, v_account, v_container):
394     # Normal Response Codes: 202
395     # Error Response Codes: internalServerError (500),
396     #                       itemNotFound (404),
397     #                       forbidden (403),
398     #                       badRequest (400)
399     
400     meta, policy = get_container_headers(request)
401     replace = True
402     if 'update' in request.GET:
403         replace = False
404     if policy:
405         try:
406             request.backend.update_container_policy(request.user_uniq, v_account,
407                                                 v_container, policy, replace)
408         except NotAllowedError:
409             raise Forbidden('Not allowed')
410         except NameError:
411             raise ItemNotFound('Container does not exist')
412         except ValueError:
413             raise BadRequest('Invalid policy header')
414     if meta or replace:
415         try:
416             request.backend.update_container_meta(request.user_uniq, v_account,
417                                                     v_container, 'pithos', meta, replace)
418         except NotAllowedError:
419             raise Forbidden('Not allowed')
420         except NameError:
421             raise ItemNotFound('Container does not exist')
422     
423     content_length = -1
424     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
425         content_length = get_int_parameter(request.META.get('CONTENT_LENGTH', 0))
426     content_type = request.META.get('CONTENT_TYPE')
427     hashmap = []
428     if content_type and content_type == 'application/octet-stream' and content_length != 0:
429         for data in socket_read_iterator(request, content_length,
430                                             request.backend.block_size):
431             # TODO: Raise 408 (Request Timeout) if this takes too long.
432             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
433             hashmap.append(request.backend.put_block(data))
434     
435     response = HttpResponse(status=202)
436     if hashmap:
437         response.content = simple_list_response(request, hashmap)
438     return response
439
440 @api_method('DELETE')
441 def container_delete(request, v_account, v_container):
442     # Normal Response Codes: 204
443     # Error Response Codes: internalServerError (500),
444     #                       conflict (409),
445     #                       itemNotFound (404),
446     #                       forbidden (403),
447     #                       badRequest (400)
448     
449     until = get_int_parameter(request.GET.get('until'))
450     try:
451         request.backend.delete_container(request.user_uniq, v_account, v_container,
452                                             until)
453     except NotAllowedError:
454         raise Forbidden('Not allowed')
455     except NameError:
456         raise ItemNotFound('Container does not exist')
457     except IndexError:
458         raise Conflict('Container is not empty')
459     return HttpResponse(status=204)
460
461 @api_method('GET', format_allowed=True)
462 def object_list(request, v_account, v_container):
463     # Normal Response Codes: 200, 204
464     # Error Response Codes: internalServerError (500),
465     #                       itemNotFound (404),
466     #                       forbidden (403),
467     #                       badRequest (400)
468     
469     until = get_int_parameter(request.GET.get('until'))
470     try:
471         meta = request.backend.get_container_meta(request.user_uniq, v_account,
472                                                     v_container, 'pithos', until)
473         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
474                                                 v_account, v_container, 'pithos', until)
475         policy = request.backend.get_container_policy(request.user_uniq, v_account,
476                                                         v_container)
477     except NotAllowedError:
478         raise Forbidden('Not allowed')
479     except NameError:
480         raise ItemNotFound('Container does not exist')
481     
482     validate_modification_preconditions(request, meta)
483     
484     response = HttpResponse()
485     put_container_headers(request, response, meta, policy)
486     
487     path = request.GET.get('path')
488     prefix = request.GET.get('prefix')
489     delimiter = request.GET.get('delimiter')
490     
491     # Path overrides prefix and delimiter.
492     virtual = True
493     if path:
494         prefix = path
495         delimiter = '/'
496         virtual = False
497     
498     # Naming policy.
499     if prefix and delimiter and not prefix.endswith(delimiter):
500         prefix = prefix + delimiter
501     if not prefix:
502         prefix = ''
503     prefix = prefix.lstrip('/')
504     
505     marker = request.GET.get('marker')
506     limit = get_int_parameter(request.GET.get('limit'))
507     if not limit:
508         limit = 10000
509     
510     keys = request.GET.get('meta')
511     if keys:
512         keys = [smart_str(x.strip()) for x in keys.split(',') if x.strip() != '']
513         included, excluded, opers = parse_filters(keys)
514         keys = []
515         keys += [format_header_key('X-Object-Meta-' + x) for x in included]
516         keys += ['!'+format_header_key('X-Object-Meta-' + x) for x in excluded]
517         keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
518     else:
519         keys = []
520     
521     shared = False
522     if 'shared' in request.GET:
523         shared = True
524     public = False
525     if 'public' in request.GET:
526         public = True
527     
528     if request.serialization == 'text':
529         try:
530             objects = request.backend.list_objects(request.user_uniq, v_account,
531                                         v_container, prefix, delimiter, marker,
532                                         limit, virtual, 'pithos', keys, shared,
533                                         until, None, public)
534         except NotAllowedError:
535             raise Forbidden('Not allowed')
536         except NameError:
537             raise ItemNotFound('Container does not exist')
538         
539         if len(objects) == 0:
540             # The cloudfiles python bindings expect 200 if json/xml.
541             response.status_code = 204
542             return response
543         response.status_code = 200
544         response.content = '\n'.join([x[0] for x in objects]) + '\n'
545         return response
546
547     try:
548         objects = request.backend.list_object_meta(request.user_uniq, v_account,
549                                     v_container, prefix, delimiter, marker,
550                                     limit, virtual, 'pithos', keys, shared, until, None, public)
551         object_permissions = {}
552         object_public = {}
553         if until is None:
554             name_idx = len('/'.join((v_account, v_container, '')))
555             for x in request.backend.list_object_permissions(request.user_uniq,
556                                     v_account, v_container, prefix):
557                 object = x[name_idx:]
558                 object_permissions[object] = request.backend.get_object_permissions(
559                                     request.user_uniq, v_account, v_container, object)
560             for k, v in request.backend.list_object_public(request.user_uniq,
561                                     v_account, v_container, prefix).iteritems():
562                 object_public[k[name_idx:]] = v
563     except NotAllowedError:
564         raise Forbidden('Not allowed')
565     except NameError:
566         raise ItemNotFound('Container does not exist')
567     
568     object_meta = []
569     for meta in objects:
570         if len(meta) == 1:
571             # Virtual objects/directories.
572             object_meta.append(meta)
573         else:
574             rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
575             rename_meta_key(meta, 'checksum', 'hash')
576             rename_meta_key(meta, 'type', 'content_type')
577             rename_meta_key(meta, 'uuid', 'x_object_uuid')
578             if until is not None and 'modified' in meta:
579                 del(meta['modified'])
580             else:
581                 rename_meta_key(meta, 'modified', 'last_modified')
582             rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
583             rename_meta_key(meta, 'version', 'x_object_version')
584             rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
585             permissions = object_permissions.get(meta['name'], None)
586             if permissions:
587                 update_sharing_meta(request, permissions, v_account, v_container, meta['name'], meta)
588             public = object_public.get(meta['name'], None)
589             if public:
590                 update_public_meta(public, meta)
591             object_meta.append(printable_header_dict(meta))
592     if request.serialization == 'xml':
593         data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
594     elif request.serialization  == 'json':
595         data = json.dumps(object_meta, default=json_encode_decimal)
596     response.status_code = 200
597     response.content = data
598     return response
599
600 @api_method('HEAD')
601 def object_meta(request, v_account, v_container, v_object):
602     # Normal Response Codes: 204
603     # Error Response Codes: internalServerError (500),
604     #                       itemNotFound (404),
605     #                       forbidden (403),
606     #                       badRequest (400)
607     
608     version = request.GET.get('version')
609     try:
610         meta = request.backend.get_object_meta(request.user_uniq, v_account,
611                                                 v_container, v_object, 'pithos', version)
612         if version is None:
613             permissions = request.backend.get_object_permissions(request.user_uniq,
614                                             v_account, v_container, v_object)
615             public = request.backend.get_object_public(request.user_uniq, v_account,
616                                                         v_container, v_object)
617         else:
618             permissions = None
619             public = None
620     except NotAllowedError:
621         raise Forbidden('Not allowed')
622     except NameError:
623         raise ItemNotFound('Object does not exist')
624     except IndexError:
625         raise ItemNotFound('Version does not exist')
626     
627     update_manifest_meta(request, v_account, meta)
628     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
629     update_public_meta(public, meta)
630     
631     # Evaluate conditions.
632     validate_modification_preconditions(request, meta)
633     try:
634         validate_matching_preconditions(request, meta)
635     except NotModified:
636         response = HttpResponse(status=304)
637         response['ETag'] = meta['checksum']
638         return response
639     
640     response = HttpResponse(status=200)
641     put_object_headers(response, meta)
642     return response
643
644 @api_method('GET', format_allowed=True)
645 def object_read(request, v_account, v_container, v_object):
646     # Normal Response Codes: 200, 206
647     # Error Response Codes: internalServerError (500),
648     #                       rangeNotSatisfiable (416),
649     #                       preconditionFailed (412),
650     #                       itemNotFound (404),
651     #                       forbidden (403),
652     #                       badRequest (400),
653     #                       notModified (304)
654     
655     version = request.GET.get('version')
656     
657     # Reply with the version list. Do this first, as the object may be deleted.
658     if version == 'list':
659         if request.serialization == 'text':
660             raise BadRequest('No format specified for version list.')
661         
662         try:
663             v = request.backend.list_versions(request.user_uniq, v_account,
664                                                 v_container, v_object)
665         except NotAllowedError:
666             raise Forbidden('Not allowed')
667         d = {'versions': v}
668         if request.serialization == 'xml':
669             d['object'] = v_object
670             data = render_to_string('versions.xml', d)
671         elif request.serialization  == 'json':
672             data = json.dumps(d, default=json_encode_decimal)
673         
674         response = HttpResponse(data, status=200)
675         response['Content-Length'] = len(data)
676         return response
677     
678     try:
679         meta = request.backend.get_object_meta(request.user_uniq, v_account,
680                                                 v_container, v_object, 'pithos', version)
681         if version is None:
682             permissions = request.backend.get_object_permissions(request.user_uniq,
683                                             v_account, v_container, v_object)
684             public = request.backend.get_object_public(request.user_uniq, v_account,
685                                                         v_container, v_object)
686         else:
687             permissions = None
688             public = None
689     except NotAllowedError:
690         raise Forbidden('Not allowed')
691     except NameError:
692         raise ItemNotFound('Object does not exist')
693     except IndexError:
694         raise ItemNotFound('Version does not exist')
695     
696     update_manifest_meta(request, v_account, meta)
697     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
698     update_public_meta(public, meta)
699     
700     # Evaluate conditions.
701     validate_modification_preconditions(request, meta)
702     try:
703         validate_matching_preconditions(request, meta)
704     except NotModified:
705         response = HttpResponse(status=304)
706         response['ETag'] = meta['checksum']
707         return response
708     
709     hashmap_reply = False
710     if 'hashmap' in request.GET and request.serialization != 'text':
711         hashmap_reply = True
712     
713     sizes = []
714     hashmaps = []
715     if 'X-Object-Manifest' in meta and not hashmap_reply:
716         try:
717             src_container, src_name = split_container_object_string('/' + meta['X-Object-Manifest'])
718             objects = request.backend.list_objects(request.user_uniq, v_account,
719                                 src_container, prefix=src_name, virtual=False)
720         except NotAllowedError:
721             raise Forbidden('Not allowed')
722         except ValueError:
723             raise BadRequest('Invalid X-Object-Manifest header')
724         except NameError:
725             raise ItemNotFound('Container does not exist')
726         
727         try:
728             for x in objects:
729                 s, h = request.backend.get_object_hashmap(request.user_uniq,
730                                         v_account, src_container, x[0], x[1])
731                 sizes.append(s)
732                 hashmaps.append(h)
733         except NotAllowedError:
734             raise Forbidden('Not allowed')
735         except NameError:
736             raise ItemNotFound('Object does not exist')
737         except IndexError:
738             raise ItemNotFound('Version does not exist')
739     else:
740         try:
741             s, h = request.backend.get_object_hashmap(request.user_uniq, v_account,
742                                                 v_container, v_object, version)
743             sizes.append(s)
744             hashmaps.append(h)
745         except NotAllowedError:
746             raise Forbidden('Not allowed')
747         except NameError:
748             raise ItemNotFound('Object does not exist')
749         except IndexError:
750             raise ItemNotFound('Version does not exist')
751     
752     # Reply with the hashmap.
753     if hashmap_reply:
754         size = sum(sizes)
755         hashmap = sum(hashmaps, [])
756         d = {
757             'block_size': request.backend.block_size,
758             'block_hash': request.backend.hash_algorithm,
759             'bytes': size,
760             'hashes': hashmap}
761         if request.serialization == 'xml':
762             d['object'] = v_object
763             data = render_to_string('hashes.xml', d)
764         elif request.serialization  == 'json':
765             data = json.dumps(d)
766         
767         response = HttpResponse(data, status=200)
768         put_object_headers(response, meta)
769         response['Content-Length'] = len(data)
770         return response
771     
772     request.serialization = 'text' # Unset.
773     return object_data_response(request, sizes, hashmaps, meta)
774
775 @api_method('PUT', format_allowed=True)
776 def object_write(request, v_account, v_container, v_object):
777     # Normal Response Codes: 201
778     # Error Response Codes: internalServerError (500),
779     #                       unprocessableEntity (422),
780     #                       lengthRequired (411),
781     #                       conflict (409),
782     #                       itemNotFound (404),
783     #                       forbidden (403),
784     #                       badRequest (400)
785     
786     # Evaluate conditions.
787     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
788         try:
789             meta = request.backend.get_object_meta(request.user_uniq, v_account,
790                                                         v_container, v_object, 'pithos')
791         except NotAllowedError:
792             raise Forbidden('Not allowed')
793         except NameError:
794             meta = {}
795         validate_matching_preconditions(request, meta)
796     
797     copy_from = request.META.get('HTTP_X_COPY_FROM')
798     move_from = request.META.get('HTTP_X_MOVE_FROM')
799     if copy_from or move_from:
800         content_length = get_content_length(request) # Required by the API.
801         
802         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
803         if not src_account:
804             src_account = request.user_uniq
805         if move_from:
806             try:
807                 src_container, src_name = split_container_object_string(move_from)
808             except ValueError:
809                 raise BadRequest('Invalid X-Move-From header')
810             version_id = copy_or_move_object(request, src_account, src_container, src_name,
811                                                 v_account, v_container, v_object, move=True)
812         else:
813             try:
814                 src_container, src_name = split_container_object_string(copy_from)
815             except ValueError:
816                 raise BadRequest('Invalid X-Copy-From header')
817             version_id = copy_or_move_object(request, src_account, src_container, src_name,
818                                                 v_account, v_container, v_object, move=False)
819         response = HttpResponse(status=201)
820         response['X-Object-Version'] = version_id
821         return response
822     
823     content_type, meta, permissions, public = get_object_headers(request)
824     content_length = -1
825     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
826         content_length = get_content_length(request)
827     # Should be BadRequest, but API says otherwise.
828     if content_type is None:
829         raise LengthRequired('Missing Content-Type header')
830     
831     if 'hashmap' in request.GET:
832         if request.serialization not in ('json', 'xml'):
833             raise BadRequest('Invalid hashmap format')
834         
835         data = ''
836         for block in socket_read_iterator(request, content_length,
837                                             request.backend.block_size):
838             data = '%s%s' % (data, block)
839         
840         if request.serialization == 'json':
841             d = json.loads(data)
842             if not hasattr(d, '__getitem__'):
843                 raise BadRequest('Invalid data formating')
844             try:
845                 hashmap = d['hashes']
846                 size = int(d['bytes'])
847             except:
848                 raise BadRequest('Invalid data formatting')
849         elif request.serialization == 'xml':
850             try:
851                 xml = minidom.parseString(data)
852                 obj = xml.getElementsByTagName('object')[0]
853                 size = int(obj.attributes['bytes'].value)
854                 
855                 hashes = xml.getElementsByTagName('hash')
856                 hashmap = []
857                 for hash in hashes:
858                     hashmap.append(hash.firstChild.data)
859             except:
860                 raise BadRequest('Invalid data formatting')
861         
862         checksum = '' # Do not set to None (will copy previous value).
863     else:
864         md5 = hashlib.md5()
865         size = 0
866         hashmap = []
867         for data in socket_read_iterator(request, content_length,
868                                             request.backend.block_size):
869             # TODO: Raise 408 (Request Timeout) if this takes too long.
870             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
871             size += len(data)
872             hashmap.append(request.backend.put_block(data))
873             md5.update(data)
874         
875         checksum = md5.hexdigest().lower()
876         etag = request.META.get('HTTP_ETAG')
877         if etag and parse_etags(etag)[0].lower() != checksum:
878             raise UnprocessableEntity('Object ETag does not match')
879     
880     try:
881         version_id = request.backend.update_object_hashmap(request.user_uniq,
882                         v_account, v_container, v_object, size, content_type,
883                         hashmap, checksum, 'pithos', meta, True, permissions)
884     except NotAllowedError:
885         raise Forbidden('Not allowed')
886     except IndexError, e:
887         raise Conflict(simple_list_response(request, e.data))
888     except NameError:
889         raise ItemNotFound('Container does not exist')
890     except ValueError:
891         raise BadRequest('Invalid sharing header')
892     except QuotaError:
893         raise RequestEntityTooLarge('Quota exceeded')
894     if not checksum and UPDATE_MD5:
895         # Update the MD5 after the hashmap, as there may be missing hashes.
896         checksum = hashmap_md5(request.backend, hashmap, size)
897         try:
898             request.backend.update_object_checksum(request.user_uniq,
899               v_account, v_container, v_object, version_id, checksum)
900         except NotAllowedError:
901             raise Forbidden('Not allowed')
902     if public is not None:
903         try:
904             request.backend.update_object_public(request.user_uniq, v_account,
905                                                 v_container, v_object, public)
906         except NotAllowedError:
907             raise Forbidden('Not allowed')
908         except NameError:
909             raise ItemNotFound('Object does not exist')
910     
911     response = HttpResponse(status=201)
912     if checksum:
913         response['ETag'] = checksum
914     response['X-Object-Version'] = version_id
915     return response
916
917 @api_method('POST')
918 def object_write_form(request, v_account, v_container, v_object):
919     # Normal Response Codes: 201
920     # Error Response Codes: internalServerError (500),
921     #                       itemNotFound (404),
922     #                       forbidden (403),
923     #                       badRequest (400)
924     
925     request.upload_handlers = [SaveToBackendHandler(request)]
926     if not request.FILES.has_key('X-Object-Data'):
927         raise BadRequest('Missing X-Object-Data field')
928     file = request.FILES['X-Object-Data']
929     
930     checksum = file.etag
931     try:
932         version_id = request.backend.update_object_hashmap(request.user_uniq,
933                         v_account, v_container, v_object, file.size, file.content_type,
934                         file.hashmap, checksum, 'pithos', {}, True)
935     except NotAllowedError:
936         raise Forbidden('Not allowed')
937     except NameError:
938         raise ItemNotFound('Container does not exist')
939     except QuotaError:
940         raise RequestEntityTooLarge('Quota exceeded')
941     
942     response = HttpResponse(status=201)
943     response['ETag'] = checksum
944     response['X-Object-Version'] = version_id
945     response.content = checksum
946     return response
947
948 @api_method('COPY', format_allowed=True)
949 def object_copy(request, v_account, v_container, v_object):
950     # Normal Response Codes: 201
951     # Error Response Codes: internalServerError (500),
952     #                       itemNotFound (404),
953     #                       forbidden (403),
954     #                       badRequest (400)
955     
956     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
957     if not dest_account:
958         dest_account = request.user_uniq
959     dest_path = request.META.get('HTTP_DESTINATION')
960     if not dest_path:
961         raise BadRequest('Missing Destination header')
962     try:
963         dest_container, dest_name = split_container_object_string(dest_path)
964     except ValueError:
965         raise BadRequest('Invalid Destination header')
966     
967     # Evaluate conditions.
968     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
969         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
970         try:
971             meta = request.backend.get_object_meta(request.user_uniq, v_account,
972                                             v_container, v_object, 'pithos', src_version)
973         except NotAllowedError:
974             raise Forbidden('Not allowed')
975         except (NameError, IndexError):
976             raise ItemNotFound('Container or object does not exist')
977         validate_matching_preconditions(request, meta)
978     
979     version_id = copy_or_move_object(request, v_account, v_container, v_object,
980                                         dest_account, dest_container, dest_name, move=False)
981     response = HttpResponse(status=201)
982     response['X-Object-Version'] = version_id
983     return response
984
985 @api_method('MOVE', format_allowed=True)
986 def object_move(request, v_account, v_container, v_object):
987     # Normal Response Codes: 201
988     # Error Response Codes: internalServerError (500),
989     #                       itemNotFound (404),
990     #                       forbidden (403),
991     #                       badRequest (400)
992     
993     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
994     if not dest_account:
995         dest_account = request.user_uniq
996     dest_path = request.META.get('HTTP_DESTINATION')
997     if not dest_path:
998         raise BadRequest('Missing Destination header')
999     try:
1000         dest_container, dest_name = split_container_object_string(dest_path)
1001     except ValueError:
1002         raise BadRequest('Invalid Destination header')
1003     
1004     # Evaluate conditions.
1005     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1006         try:
1007             meta = request.backend.get_object_meta(request.user_uniq, v_account,
1008                                                     v_container, v_object, 'pithos')
1009         except NotAllowedError:
1010             raise Forbidden('Not allowed')
1011         except NameError:
1012             raise ItemNotFound('Container or object does not exist')
1013         validate_matching_preconditions(request, meta)
1014     
1015     version_id = copy_or_move_object(request, v_account, v_container, v_object,
1016                                         dest_account, dest_container, dest_name, move=True)
1017     response = HttpResponse(status=201)
1018     response['X-Object-Version'] = version_id
1019     return response
1020
1021 @api_method('POST', format_allowed=True)
1022 def object_update(request, v_account, v_container, v_object):
1023     # Normal Response Codes: 202, 204
1024     # Error Response Codes: internalServerError (500),
1025     #                       conflict (409),
1026     #                       itemNotFound (404),
1027     #                       forbidden (403),
1028     #                       badRequest (400)
1029     
1030     content_type, meta, permissions, public = get_object_headers(request)
1031     
1032     try:
1033         prev_meta = request.backend.get_object_meta(request.user_uniq, v_account,
1034                                                     v_container, v_object, 'pithos')
1035     except NotAllowedError:
1036         raise Forbidden('Not allowed')
1037     except NameError:
1038         raise ItemNotFound('Object does not exist')
1039     
1040     # Evaluate conditions.
1041     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1042         validate_matching_preconditions(request, prev_meta)
1043     
1044     replace = True
1045     if 'update' in request.GET:
1046         replace = False
1047     
1048     # A Content-Type or X-Source-Object header indicates data updates.
1049     src_object = request.META.get('HTTP_X_SOURCE_OBJECT')
1050     if (not content_type or content_type != 'application/octet-stream') and not src_object:
1051         response = HttpResponse(status=202)
1052         
1053         # Do permissions first, as it may fail easier.
1054         if permissions is not None:
1055             try:
1056                 request.backend.update_object_permissions(request.user_uniq,
1057                                 v_account, v_container, v_object, permissions)
1058             except NotAllowedError:
1059                 raise Forbidden('Not allowed')
1060             except NameError:
1061                 raise ItemNotFound('Object does not exist')
1062             except ValueError:
1063                 raise BadRequest('Invalid sharing header')
1064         if public is not None:
1065             try:
1066                 request.backend.update_object_public(request.user_uniq, v_account,
1067                                                 v_container, v_object, public)
1068             except NotAllowedError:
1069                 raise Forbidden('Not allowed')
1070             except NameError:
1071                 raise ItemNotFound('Object does not exist')
1072         if meta or replace:
1073             try:
1074                 version_id = request.backend.update_object_meta(request.user_uniq,
1075                                 v_account, v_container, v_object, 'pithos', meta, replace)
1076             except NotAllowedError:
1077                 raise Forbidden('Not allowed')
1078             except NameError:
1079                 raise ItemNotFound('Object does not exist')        
1080             response['X-Object-Version'] = version_id
1081         
1082         return response
1083     
1084     # Single range update. Range must be in Content-Range.
1085     # Based on: http://code.google.com/p/gears/wiki/ContentRangePostProposal
1086     # (with the addition that '*' is allowed for the range - will append).
1087     content_range = request.META.get('HTTP_CONTENT_RANGE')
1088     if not content_range:
1089         raise BadRequest('Missing Content-Range header')
1090     ranges = get_content_range(request)
1091     if not ranges:
1092         raise RangeNotSatisfiable('Invalid Content-Range header')
1093     
1094     try:
1095         size, hashmap = request.backend.get_object_hashmap(request.user_uniq,
1096                                             v_account, v_container, v_object)
1097     except NotAllowedError:
1098         raise Forbidden('Not allowed')
1099     except NameError:
1100         raise ItemNotFound('Object does not exist')
1101     
1102     offset, length, total = ranges
1103     if offset is None:
1104         offset = size
1105     elif offset > size:
1106         raise RangeNotSatisfiable('Supplied offset is beyond object limits')
1107     if src_object:
1108         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
1109         if not src_account:
1110             src_account = request.user_uniq
1111         src_container, src_name = split_container_object_string(src_object)
1112         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
1113         try:
1114             src_size, src_hashmap = request.backend.get_object_hashmap(request.user_uniq,
1115                                         src_account, src_container, src_name, src_version)
1116         except NotAllowedError:
1117             raise Forbidden('Not allowed')
1118         except NameError:
1119             raise ItemNotFound('Source object does not exist')
1120         
1121         if length is None:
1122             length = src_size
1123         elif length > src_size:
1124             raise BadRequest('Object length is smaller than range length')
1125     else:
1126         # Require either a Content-Length, or 'chunked' Transfer-Encoding.
1127         content_length = -1
1128         if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
1129             content_length = get_content_length(request)
1130         
1131         if length is None:
1132             length = content_length
1133         else:
1134             if content_length == -1:
1135                 # TODO: Get up to length bytes in chunks.
1136                 length = content_length
1137             elif length != content_length:
1138                 raise BadRequest('Content length does not match range length')
1139     if total is not None and (total != size or offset >= size or (length > 0 and offset + length >= size)):
1140         raise RangeNotSatisfiable('Supplied range will change provided object limits')
1141     
1142     dest_bytes = request.META.get('HTTP_X_OBJECT_BYTES')
1143     if dest_bytes is not None:
1144         dest_bytes = get_int_parameter(dest_bytes)
1145         if dest_bytes is None:
1146             raise BadRequest('Invalid X-Object-Bytes header')
1147     
1148     if src_object:
1149         if offset % request.backend.block_size == 0:
1150             # Update the hashes only.
1151             sbi = 0
1152             while length > 0:
1153                 bi = int(offset / request.backend.block_size)
1154                 bl = min(length, request.backend.block_size)
1155                 if bi < len(hashmap):
1156                     if bl == request.backend.block_size:
1157                         hashmap[bi] = src_hashmap[sbi]
1158                     else:
1159                         data = request.backend.get_block(src_hashmap[sbi])
1160                         hashmap[bi] = request.backend.update_block(hashmap[bi],
1161                                                                 data[:bl], 0)
1162                 else:
1163                     hashmap.append(src_hashmap[sbi])
1164                 offset += bl
1165                 length -= bl
1166                 sbi += 1
1167         else:
1168             data = ''
1169             sbi = 0
1170             while length > 0:
1171                 data += request.backend.get_block(src_hashmap[sbi])
1172                 if length < request.backend.block_size:
1173                     data = data[:length]
1174                 bytes = put_object_block(request, hashmap, data, offset)
1175                 offset += bytes
1176                 data = data[bytes:]
1177                 length -= bytes
1178                 sbi += 1
1179     else:
1180         data = ''
1181         for d in socket_read_iterator(request, length,
1182                                         request.backend.block_size):
1183             # TODO: Raise 408 (Request Timeout) if this takes too long.
1184             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
1185             data += d
1186             bytes = put_object_block(request, hashmap, data, offset)
1187             offset += bytes
1188             data = data[bytes:]
1189         if len(data) > 0:
1190             put_object_block(request, hashmap, data, offset)
1191     
1192     if offset > size:
1193         size = offset
1194     if dest_bytes is not None and dest_bytes < size:
1195         size = dest_bytes
1196         hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)]
1197     checksum = hashmap_md5(request.backend, hashmap, size) if UPDATE_MD5 else ''
1198     try:
1199         version_id = request.backend.update_object_hashmap(request.user_uniq,
1200                         v_account, v_container, v_object, size, prev_meta['type'],
1201                         hashmap, checksum, 'pithos', meta, replace, permissions)
1202     except NotAllowedError:
1203         raise Forbidden('Not allowed')
1204     except NameError:
1205         raise ItemNotFound('Container does not exist')
1206     except ValueError:
1207         raise BadRequest('Invalid sharing header')
1208     except QuotaError:
1209         raise RequestEntityTooLarge('Quota exceeded')
1210     if public is not None:
1211         try:
1212             request.backend.update_object_public(request.user_uniq, v_account,
1213                                                 v_container, v_object, public)
1214         except NotAllowedError:
1215             raise Forbidden('Not allowed')
1216         except NameError:
1217             raise ItemNotFound('Object does not exist')
1218     
1219     response = HttpResponse(status=204)
1220     response['ETag'] = checksum
1221     response['X-Object-Version'] = version_id
1222     return response
1223
1224 @api_method('DELETE')
1225 def object_delete(request, v_account, v_container, v_object):
1226     # Normal Response Codes: 204
1227     # Error Response Codes: internalServerError (500),
1228     #                       itemNotFound (404),
1229     #                       forbidden (403),
1230     #                       badRequest (400)
1231     
1232     until = get_int_parameter(request.GET.get('until'))
1233     try:
1234         request.backend.delete_object(request.user_uniq, v_account, v_container,
1235                                         v_object, until)
1236     except NotAllowedError:
1237         raise Forbidden('Not allowed')
1238     except NameError:
1239         raise ItemNotFound('Object does not exist')
1240     return HttpResponse(status=204)
1241
1242 @api_method()
1243 def method_not_allowed(request):
1244     raise BadRequest('Method not allowed')