Add dispatcher function for updating object checksums asynchronously.
[pithos] / snf-pithos-app / pithos / api / functions.py
1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
2
3 # Redistribution and use in source and binary forms, with or
4 # without modification, are permitted provided that the following
5 # conditions are met:
6
7 #   1. Redistributions of source code must retain the above
8 #      copyright notice, this list of conditions and the following
9 #      disclaimer.
10
11 #   2. Redistributions in binary form must reproduce the above
12 #      copyright notice, this list of conditions and the following
13 #      disclaimer in the documentation and/or other materials
14 #      provided with the distribution.
15
16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 # POSSIBILITY OF SUCH DAMAGE.
28
29 # The views and conclusions contained in the software and
30 # documentation are those of the authors and should not be
31 # interpreted as representing official policies, either expressed
32 # or implied, of GRNET S.A.
33
34 from xml.dom import minidom
35 from urllib import unquote
36
37 from django.conf import settings
38 from django.http import HttpResponse
39 from django.template.loader import render_to_string
40 from django.utils import simplejson as json
41 from django.utils.http import parse_etags
42 from django.utils.encoding import smart_str
43 from django.views.decorators.csrf import csrf_exempt
44
45 from synnefo.lib.astakos import get_user
46
47 from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
48     LengthRequired, PreconditionFailed, RequestEntityTooLarge, RangeNotSatisfiable, UnprocessableEntity)
49 from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
50     get_account_headers, put_account_headers, get_container_headers, put_container_headers, get_object_headers,
51     put_object_headers, update_manifest_meta, update_sharing_meta, update_public_meta,
52     validate_modification_preconditions, validate_matching_preconditions, split_container_object_string,
53     copy_or_move_object, get_int_parameter, get_content_length, get_content_range, socket_read_iterator,
54     SaveToBackendHandler, object_data_response, put_object_block, hashmap_md5, simple_list_response, api_method)
55 from pithos.api.settings import AUTHENTICATION_URL, AUTHENTICATION_USERS, COOKIE_NAME, UPDATE_MD5
56
57 from pithos.backends.base import NotAllowedError, QuotaError
58 from pithos.backends.filter import parse_filters
59
60 import logging
61 import hashlib
62
63
64 logger = logging.getLogger(__name__)
65
66
67 @csrf_exempt
68 def top_demux(request):
69     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
70     if request.method == 'GET':
71         if getattr(request, 'user', None) is not None:
72             return account_list(request)
73         return authenticate(request)
74     else:
75         return method_not_allowed(request)
76
77 @csrf_exempt
78 def account_demux(request, v_account):
79     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
80     if request.method == 'HEAD':
81         return account_meta(request, v_account)
82     elif request.method == 'POST':
83         return account_update(request, v_account)
84     elif request.method == 'GET':
85         return container_list(request, v_account)
86     else:
87         return method_not_allowed(request)
88
89 @csrf_exempt
90 def container_demux(request, v_account, v_container):
91     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS)
92     if request.method == 'HEAD':
93         return container_meta(request, v_account, v_container)
94     elif request.method == 'PUT':
95         return container_create(request, v_account, v_container)
96     elif request.method == 'POST':
97         return container_update(request, v_account, v_container)
98     elif request.method == 'DELETE':
99         return container_delete(request, v_account, v_container)
100     elif request.method == 'GET':
101         return object_list(request, v_account, v_container)
102     else:
103         return method_not_allowed(request)
104
105 @csrf_exempt
106 def object_demux(request, v_account, v_container, v_object):
107     # Helper to avoid placing the token in the URL when loading objects from a browser.
108     token = None
109     if request.method in ('HEAD', 'GET') and COOKIE_NAME in request.COOKIES:
110         cookie_value = unquote(request.COOKIES.get(COOKIE_NAME, ''))
111         if cookie_value and '|' in cookie_value:
112             token = cookie_value.split('|', 1)[1]
113     get_user(request, AUTHENTICATION_URL, AUTHENTICATION_USERS, token)
114     if request.method == 'HEAD':
115         return object_meta(request, v_account, v_container, v_object)
116     elif request.method == 'GET':
117         return object_read(request, v_account, v_container, v_object)
118     elif request.method == 'PUT':
119         return object_write(request, v_account, v_container, v_object)
120     elif request.method == 'COPY':
121         return object_copy(request, v_account, v_container, v_object)
122     elif request.method == 'MOVE':
123         return object_move(request, v_account, v_container, v_object)
124     elif request.method == 'POST':
125         if request.META.get('CONTENT_TYPE', '').startswith('multipart/form-data'):
126             return object_write_form(request, v_account, v_container, v_object)
127         return object_update(request, v_account, v_container, v_object)
128     elif request.method == 'DELETE':
129         return object_delete(request, v_account, v_container, v_object)
130     else:
131         return method_not_allowed(request)
132
133 @api_method('GET', user_required=False)
134 def authenticate(request):
135     # Normal Response Codes: 204
136     # Error Response Codes: internalServerError (500),
137     #                       forbidden (403),
138     #                       badRequest (400)
139     
140     x_auth_user = request.META.get('HTTP_X_AUTH_USER')
141     x_auth_key = request.META.get('HTTP_X_AUTH_KEY')
142     if not x_auth_user or not x_auth_key:
143         raise BadRequest('Missing X-Auth-User or X-Auth-Key header')
144     response = HttpResponse(status=204)
145     
146     uri = request.build_absolute_uri()
147     if '?' in uri:
148         uri = uri[:uri.find('?')]
149     
150     response['X-Auth-Token'] = x_auth_key
151     response['X-Storage-Url'] = uri + ('' if uri.endswith('/') else '/') + x_auth_user
152     return response
153
154 @api_method('GET', format_allowed=True)
155 def account_list(request):
156     # Normal Response Codes: 200, 204
157     # Error Response Codes: internalServerError (500),
158     #                       badRequest (400)
159     
160     response = HttpResponse()
161     
162     marker = request.GET.get('marker')
163     limit = get_int_parameter(request.GET.get('limit'))
164     if not limit:
165         limit = 10000
166     
167     accounts = request.backend.list_accounts(request.user_uniq, marker, limit)
168     
169     if request.serialization == 'text':
170         if len(accounts) == 0:
171             # The cloudfiles python bindings expect 200 if json/xml.
172             response.status_code = 204
173             return response
174         response.status_code = 200
175         response.content = '\n'.join(accounts) + '\n'
176         return response
177     
178     account_meta = []
179     for x in accounts:
180         if x == request.user_uniq:
181             continue
182         try:
183             meta = request.backend.get_account_meta(request.user_uniq, x, 'pithos', include_user_defined=False)
184             groups = request.backend.get_account_groups(request.user_uniq, x)
185         except NotAllowedError:
186             raise Forbidden('Not allowed')
187         else:
188             rename_meta_key(meta, 'modified', 'last_modified')
189             rename_meta_key(meta, 'until_timestamp', 'x_account_until_timestamp')
190             if groups:
191                 meta['X-Account-Group'] = printable_header_dict(dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
192             account_meta.append(printable_header_dict(meta))
193     if request.serialization == 'xml':
194         data = render_to_string('accounts.xml', {'accounts': account_meta})
195     elif request.serialization  == 'json':
196         data = json.dumps(account_meta)
197     response.status_code = 200
198     response.content = data
199     return response
200
201 @api_method('HEAD')
202 def account_meta(request, v_account):
203     # Normal Response Codes: 204
204     # Error Response Codes: internalServerError (500),
205     #                       forbidden (403),
206     #                       badRequest (400)
207     
208     until = get_int_parameter(request.GET.get('until'))
209     try:
210         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
211         groups = request.backend.get_account_groups(request.user_uniq, v_account)
212         policy = request.backend.get_account_policy(request.user_uniq, v_account)
213     except NotAllowedError:
214         raise Forbidden('Not allowed')
215     
216     validate_modification_preconditions(request, meta)
217     
218     response = HttpResponse(status=204)
219     put_account_headers(response, meta, groups, policy)
220     return response
221
222 @api_method('POST')
223 def account_update(request, v_account):
224     # Normal Response Codes: 202
225     # Error Response Codes: internalServerError (500),
226     #                       forbidden (403),
227     #                       badRequest (400)
228     
229     meta, groups = get_account_headers(request)
230     replace = True
231     if 'update' in request.GET:
232         replace = False
233     if groups:
234         try:
235             request.backend.update_account_groups(request.user_uniq, v_account,
236                                                     groups, replace)
237         except NotAllowedError:
238             raise Forbidden('Not allowed')
239         except ValueError:
240             raise BadRequest('Invalid groups header')
241     if meta or replace:
242         try:
243             request.backend.update_account_meta(request.user_uniq, v_account,
244                                                 'pithos', meta, replace)
245         except NotAllowedError:
246             raise Forbidden('Not allowed')
247     return HttpResponse(status=202)
248
249 @api_method('GET', format_allowed=True)
250 def container_list(request, v_account):
251     # Normal Response Codes: 200, 204
252     # Error Response Codes: internalServerError (500),
253     #                       itemNotFound (404),
254     #                       forbidden (403),
255     #                       badRequest (400)
256     
257     until = get_int_parameter(request.GET.get('until'))
258     try:
259         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
260         groups = request.backend.get_account_groups(request.user_uniq, v_account)
261         policy = request.backend.get_account_policy(request.user_uniq, v_account)
262     except NotAllowedError:
263         raise Forbidden('Not allowed')
264     
265     validate_modification_preconditions(request, meta)
266     
267     response = HttpResponse()
268     put_account_headers(response, meta, groups, policy)
269     
270     marker = request.GET.get('marker')
271     limit = get_int_parameter(request.GET.get('limit'))
272     if not limit:
273         limit = 10000
274     
275     shared = False
276     if 'shared' in request.GET:
277         shared = True
278     
279     try:
280         containers = request.backend.list_containers(request.user_uniq, v_account,
281                                                 marker, limit, shared, until)
282     except NotAllowedError:
283         raise Forbidden('Not allowed')
284     except NameError:
285         containers = []
286     
287     if request.serialization == 'text':
288         if len(containers) == 0:
289             # The cloudfiles python bindings expect 200 if json/xml.
290             response.status_code = 204
291             return response
292         response.status_code = 200
293         response.content = '\n'.join(containers) + '\n'
294         return response
295     
296     container_meta = []
297     for x in containers:
298         try:
299             meta = request.backend.get_container_meta(request.user_uniq, v_account,
300                                                         x, 'pithos', until, include_user_defined=False)
301             policy = request.backend.get_container_policy(request.user_uniq,
302                                                             v_account, x)
303         except NotAllowedError:
304             raise Forbidden('Not allowed')
305         except NameError:
306             pass
307         else:
308             rename_meta_key(meta, 'modified', 'last_modified')
309             rename_meta_key(meta, 'until_timestamp', 'x_container_until_timestamp')
310             if policy:
311                 meta['X-Container-Policy'] = printable_header_dict(dict([(k, v) for k, v in policy.iteritems()]))
312             container_meta.append(printable_header_dict(meta))
313     if request.serialization == 'xml':
314         data = render_to_string('containers.xml', {'account': v_account, 'containers': container_meta})
315     elif request.serialization  == 'json':
316         data = json.dumps(container_meta)
317     response.status_code = 200
318     response.content = data
319     return response
320
321 @api_method('HEAD')
322 def container_meta(request, v_account, v_container):
323     # Normal Response Codes: 204
324     # Error Response Codes: internalServerError (500),
325     #                       itemNotFound (404),
326     #                       forbidden (403),
327     #                       badRequest (400)
328     
329     until = get_int_parameter(request.GET.get('until'))
330     try:
331         meta = request.backend.get_container_meta(request.user_uniq, v_account,
332                                                     v_container, 'pithos', until)
333         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
334                                                 v_account, v_container, 'pithos', until)
335         policy = request.backend.get_container_policy(request.user_uniq, v_account,
336                                                         v_container)
337     except NotAllowedError:
338         raise Forbidden('Not allowed')
339     except NameError:
340         raise ItemNotFound('Container does not exist')
341     
342     validate_modification_preconditions(request, meta)
343     
344     response = HttpResponse(status=204)
345     put_container_headers(request, response, meta, policy)
346     return response
347
348 @api_method('PUT')
349 def container_create(request, v_account, v_container):
350     # Normal Response Codes: 201, 202
351     # Error Response Codes: internalServerError (500),
352     #                       itemNotFound (404),
353     #                       forbidden (403),
354     #                       badRequest (400)
355     
356     meta, policy = get_container_headers(request)
357     
358     try:
359         request.backend.put_container(request.user_uniq, v_account, v_container, policy)
360         ret = 201
361     except NotAllowedError:
362         raise Forbidden('Not allowed')
363     except ValueError:
364         raise BadRequest('Invalid policy header')
365     except NameError:
366         ret = 202
367     
368     if ret == 202 and policy:
369         try:
370             request.backend.update_container_policy(request.user_uniq, v_account,
371                                             v_container, policy, replace=False)
372         except NotAllowedError:
373             raise Forbidden('Not allowed')
374         except NameError:
375             raise ItemNotFound('Container does not exist')
376         except ValueError:
377             raise BadRequest('Invalid policy header')
378     if meta:
379         try:
380             request.backend.update_container_meta(request.user_uniq, v_account,
381                                             v_container, 'pithos', meta, replace=False)
382         except NotAllowedError:
383             raise Forbidden('Not allowed')
384         except NameError:
385             raise ItemNotFound('Container does not exist')
386     
387     return HttpResponse(status=ret)
388
389 @api_method('POST', format_allowed=True)
390 def container_update(request, v_account, v_container):
391     # Normal Response Codes: 202
392     # Error Response Codes: internalServerError (500),
393     #                       itemNotFound (404),
394     #                       forbidden (403),
395     #                       badRequest (400)
396     
397     meta, policy = get_container_headers(request)
398     replace = True
399     if 'update' in request.GET:
400         replace = False
401     if policy:
402         try:
403             request.backend.update_container_policy(request.user_uniq, v_account,
404                                                 v_container, policy, replace)
405         except NotAllowedError:
406             raise Forbidden('Not allowed')
407         except NameError:
408             raise ItemNotFound('Container does not exist')
409         except ValueError:
410             raise BadRequest('Invalid policy header')
411     if meta or replace:
412         try:
413             request.backend.update_container_meta(request.user_uniq, v_account,
414                                                     v_container, 'pithos', meta, replace)
415         except NotAllowedError:
416             raise Forbidden('Not allowed')
417         except NameError:
418             raise ItemNotFound('Container does not exist')
419     
420     content_length = -1
421     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
422         content_length = get_int_parameter(request.META.get('CONTENT_LENGTH', 0))
423     content_type = request.META.get('CONTENT_TYPE')
424     hashmap = []
425     if content_type and content_type == 'application/octet-stream' and content_length != 0:
426         for data in socket_read_iterator(request, content_length,
427                                             request.backend.block_size):
428             # TODO: Raise 408 (Request Timeout) if this takes too long.
429             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
430             hashmap.append(request.backend.put_block(data))
431     
432     response = HttpResponse(status=202)
433     if hashmap:
434         response.content = simple_list_response(request, hashmap)
435     return response
436
437 @api_method('DELETE')
438 def container_delete(request, v_account, v_container):
439     # Normal Response Codes: 204
440     # Error Response Codes: internalServerError (500),
441     #                       conflict (409),
442     #                       itemNotFound (404),
443     #                       forbidden (403),
444     #                       badRequest (400)
445     
446     until = get_int_parameter(request.GET.get('until'))
447     try:
448         request.backend.delete_container(request.user_uniq, v_account, v_container,
449                                             until)
450     except NotAllowedError:
451         raise Forbidden('Not allowed')
452     except NameError:
453         raise ItemNotFound('Container does not exist')
454     except IndexError:
455         raise Conflict('Container is not empty')
456     return HttpResponse(status=204)
457
458 @api_method('GET', format_allowed=True)
459 def object_list(request, v_account, v_container):
460     # Normal Response Codes: 200, 204
461     # Error Response Codes: internalServerError (500),
462     #                       itemNotFound (404),
463     #                       forbidden (403),
464     #                       badRequest (400)
465     
466     until = get_int_parameter(request.GET.get('until'))
467     try:
468         meta = request.backend.get_container_meta(request.user_uniq, v_account,
469                                                     v_container, 'pithos', until)
470         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
471                                                 v_account, v_container, 'pithos', until)
472         policy = request.backend.get_container_policy(request.user_uniq, v_account,
473                                                         v_container)
474     except NotAllowedError:
475         raise Forbidden('Not allowed')
476     except NameError:
477         raise ItemNotFound('Container does not exist')
478     
479     validate_modification_preconditions(request, meta)
480     
481     response = HttpResponse()
482     put_container_headers(request, response, meta, policy)
483     
484     path = request.GET.get('path')
485     prefix = request.GET.get('prefix')
486     delimiter = request.GET.get('delimiter')
487     
488     # Path overrides prefix and delimiter.
489     virtual = True
490     if path:
491         prefix = path
492         delimiter = '/'
493         virtual = False
494     
495     # Naming policy.
496     if prefix and delimiter:
497         prefix = prefix + delimiter
498     if not prefix:
499         prefix = ''
500     prefix = prefix.lstrip('/')
501     
502     marker = request.GET.get('marker')
503     limit = get_int_parameter(request.GET.get('limit'))
504     if not limit:
505         limit = 10000
506     
507     keys = request.GET.get('meta')
508     if keys:
509         keys = [smart_str(x.strip()) for x in keys.split(',') if x.strip() != '']
510         included, excluded, opers = parse_filters(keys)
511         keys = []
512         keys += [format_header_key('X-Object-Meta-' + x) for x in included]
513         keys += ['!'+format_header_key('X-Object-Meta-' + x) for x in excluded]
514         keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
515     else:
516         keys = []
517     
518     shared = False
519     if 'shared' in request.GET:
520         shared = True
521     
522     if request.serialization == 'text':
523         try:
524             objects = request.backend.list_objects(request.user_uniq, v_account,
525                                         v_container, prefix, delimiter, marker,
526                                         limit, virtual, 'pithos', keys, shared, until)
527         except NotAllowedError:
528             raise Forbidden('Not allowed')
529         except NameError:
530             raise ItemNotFound('Container does not exist')
531         
532         if len(objects) == 0:
533             # The cloudfiles python bindings expect 200 if json/xml.
534             response.status_code = 204
535             return response
536         response.status_code = 200
537         response.content = '\n'.join([x[0] for x in objects]) + '\n'
538         return response
539     
540     try:
541         objects = request.backend.list_object_meta(request.user_uniq, v_account,
542                                     v_container, prefix, delimiter, marker,
543                                     limit, virtual, 'pithos', keys, shared, until)
544         object_permissions = {}
545         object_public = {}
546         if until is None:
547             name_idx = len('/'.join((v_account, v_container, '')))
548             for x in request.backend.list_object_permissions(request.user_uniq,
549                                     v_account, v_container, prefix):
550                 object = x[name_idx:]
551                 object_permissions[object] = request.backend.get_object_permissions(
552                                     request.user_uniq, v_account, v_container, object)
553             for k, v in request.backend.list_object_public(request.user_uniq,
554                                     v_account, v_container, prefix).iteritems():
555                 object_public[k[name_idx:]] = v
556     except NotAllowedError:
557         raise Forbidden('Not allowed')
558     except NameError:
559         raise ItemNotFound('Container does not exist')
560     
561     object_meta = []
562     for meta in objects:
563         if len(meta) == 1:
564             # Virtual objects/directories.
565             object_meta.append(meta)
566         else:
567             rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
568             rename_meta_key(meta, 'checksum', 'hash')
569             rename_meta_key(meta, 'type', 'content_type')
570             rename_meta_key(meta, 'uuid', 'x_object_uuid')
571             if until is not None and 'modified' in meta:
572                 del(meta['modified'])
573             else:
574                 rename_meta_key(meta, 'modified', 'last_modified')
575             rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
576             rename_meta_key(meta, 'version', 'x_object_version')
577             rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
578             permissions = object_permissions.get(meta['name'], None)
579             if permissions:
580                 update_sharing_meta(request, permissions, v_account, v_container, meta['name'], meta)
581             public = object_public.get(meta['name'], None)
582             if public:
583                 update_public_meta(public, meta)
584             object_meta.append(printable_header_dict(meta))
585     if request.serialization == 'xml':
586         data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
587     elif request.serialization  == 'json':
588         data = json.dumps(object_meta, default=json_encode_decimal)
589     response.status_code = 200
590     response.content = data
591     return response
592
593 @api_method('HEAD')
594 def object_meta(request, v_account, v_container, v_object):
595     # Normal Response Codes: 204
596     # Error Response Codes: internalServerError (500),
597     #                       itemNotFound (404),
598     #                       forbidden (403),
599     #                       badRequest (400)
600     
601     version = request.GET.get('version')
602     try:
603         meta = request.backend.get_object_meta(request.user_uniq, v_account,
604                                                 v_container, v_object, 'pithos', version)
605         if version is None:
606             permissions = request.backend.get_object_permissions(request.user_uniq,
607                                             v_account, v_container, v_object)
608             public = request.backend.get_object_public(request.user_uniq, v_account,
609                                                         v_container, v_object)
610         else:
611             permissions = None
612             public = None
613     except NotAllowedError:
614         raise Forbidden('Not allowed')
615     except NameError:
616         raise ItemNotFound('Object does not exist')
617     except IndexError:
618         raise ItemNotFound('Version does not exist')
619     
620     update_manifest_meta(request, v_account, meta)
621     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
622     update_public_meta(public, meta)
623     
624     # Evaluate conditions.
625     validate_modification_preconditions(request, meta)
626     try:
627         validate_matching_preconditions(request, meta)
628     except NotModified:
629         response = HttpResponse(status=304)
630         response['ETag'] = meta['checksum']
631         return response
632     
633     response = HttpResponse(status=200)
634     put_object_headers(response, meta)
635     return response
636
637 @api_method('GET', format_allowed=True)
638 def object_read(request, v_account, v_container, v_object):
639     # Normal Response Codes: 200, 206
640     # Error Response Codes: internalServerError (500),
641     #                       rangeNotSatisfiable (416),
642     #                       preconditionFailed (412),
643     #                       itemNotFound (404),
644     #                       forbidden (403),
645     #                       badRequest (400),
646     #                       notModified (304)
647     
648     version = request.GET.get('version')
649     
650     # Reply with the version list. Do this first, as the object may be deleted.
651     if version == 'list':
652         if request.serialization == 'text':
653             raise BadRequest('No format specified for version list.')
654         
655         try:
656             v = request.backend.list_versions(request.user_uniq, v_account,
657                                                 v_container, v_object)
658         except NotAllowedError:
659             raise Forbidden('Not allowed')
660         d = {'versions': v}
661         if request.serialization == 'xml':
662             d['object'] = v_object
663             data = render_to_string('versions.xml', d)
664         elif request.serialization  == 'json':
665             data = json.dumps(d, default=json_encode_decimal)
666         
667         response = HttpResponse(data, status=200)
668         response['Content-Length'] = len(data)
669         return response
670     
671     try:
672         meta = request.backend.get_object_meta(request.user_uniq, v_account,
673                                                 v_container, v_object, 'pithos', version)
674         if version is None:
675             permissions = request.backend.get_object_permissions(request.user_uniq,
676                                             v_account, v_container, v_object)
677             public = request.backend.get_object_public(request.user_uniq, v_account,
678                                                         v_container, v_object)
679         else:
680             permissions = None
681             public = None
682     except NotAllowedError:
683         raise Forbidden('Not allowed')
684     except NameError:
685         raise ItemNotFound('Object does not exist')
686     except IndexError:
687         raise ItemNotFound('Version does not exist')
688     
689     update_manifest_meta(request, v_account, meta)
690     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
691     update_public_meta(public, meta)
692     
693     # Evaluate conditions.
694     validate_modification_preconditions(request, meta)
695     try:
696         validate_matching_preconditions(request, meta)
697     except NotModified:
698         response = HttpResponse(status=304)
699         response['ETag'] = meta['checksum']
700         return response
701     
702     hashmap_reply = False
703     if 'hashmap' in request.GET and request.serialization != 'text':
704         hashmap_reply = True
705     
706     sizes = []
707     hashmaps = []
708     if 'X-Object-Manifest' in meta and not hashmap_reply:
709         try:
710             src_container, src_name = split_container_object_string('/' + meta['X-Object-Manifest'])
711             objects = request.backend.list_objects(request.user_uniq, v_account,
712                                 src_container, prefix=src_name, virtual=False)
713         except NotAllowedError:
714             raise Forbidden('Not allowed')
715         except ValueError:
716             raise BadRequest('Invalid X-Object-Manifest header')
717         except NameError:
718             raise ItemNotFound('Container does not exist')
719         
720         try:
721             for x in objects:
722                 s, h = request.backend.get_object_hashmap(request.user_uniq,
723                                         v_account, src_container, x[0], x[1])
724                 sizes.append(s)
725                 hashmaps.append(h)
726         except NotAllowedError:
727             raise Forbidden('Not allowed')
728         except NameError:
729             raise ItemNotFound('Object does not exist')
730         except IndexError:
731             raise ItemNotFound('Version does not exist')
732     else:
733         try:
734             s, h = request.backend.get_object_hashmap(request.user_uniq, v_account,
735                                                 v_container, v_object, version)
736             sizes.append(s)
737             hashmaps.append(h)
738         except NotAllowedError:
739             raise Forbidden('Not allowed')
740         except NameError:
741             raise ItemNotFound('Object does not exist')
742         except IndexError:
743             raise ItemNotFound('Version does not exist')
744     
745     # Reply with the hashmap.
746     if hashmap_reply:
747         size = sum(sizes)
748         hashmap = sum(hashmaps, [])
749         d = {
750             'block_size': request.backend.block_size,
751             'block_hash': request.backend.hash_algorithm,
752             'bytes': size,
753             'hashes': hashmap}
754         if request.serialization == 'xml':
755             d['object'] = v_object
756             data = render_to_string('hashes.xml', d)
757         elif request.serialization  == 'json':
758             data = json.dumps(d)
759         
760         response = HttpResponse(data, status=200)
761         put_object_headers(response, meta)
762         response['Content-Length'] = len(data)
763         return response
764     
765     request.serialization = 'text' # Unset.
766     return object_data_response(request, sizes, hashmaps, meta)
767
768 @api_method('PUT', format_allowed=True)
769 def object_write(request, v_account, v_container, v_object):
770     # Normal Response Codes: 201
771     # Error Response Codes: internalServerError (500),
772     #                       unprocessableEntity (422),
773     #                       lengthRequired (411),
774     #                       conflict (409),
775     #                       itemNotFound (404),
776     #                       forbidden (403),
777     #                       badRequest (400)
778     
779     # Evaluate conditions.
780     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
781         try:
782             meta = request.backend.get_object_meta(request.user_uniq, v_account,
783                                                         v_container, v_object, 'pithos')
784         except NotAllowedError:
785             raise Forbidden('Not allowed')
786         except NameError:
787             meta = {}
788         validate_matching_preconditions(request, meta)
789     
790     copy_from = request.META.get('HTTP_X_COPY_FROM')
791     move_from = request.META.get('HTTP_X_MOVE_FROM')
792     if copy_from or move_from:
793         content_length = get_content_length(request) # Required by the API.
794         
795         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
796         if not src_account:
797             src_account = request.user_uniq
798         if move_from:
799             try:
800                 src_container, src_name = split_container_object_string(move_from)
801             except ValueError:
802                 raise BadRequest('Invalid X-Move-From header')
803             version_id = copy_or_move_object(request, src_account, src_container, src_name,
804                                                 v_account, v_container, v_object, move=True)
805         else:
806             try:
807                 src_container, src_name = split_container_object_string(copy_from)
808             except ValueError:
809                 raise BadRequest('Invalid X-Copy-From header')
810             version_id = copy_or_move_object(request, src_account, src_container, src_name,
811                                                 v_account, v_container, v_object, move=False)
812         response = HttpResponse(status=201)
813         response['X-Object-Version'] = version_id
814         return response
815     
816     content_type, meta, permissions, public = get_object_headers(request)
817     content_length = -1
818     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
819         content_length = get_content_length(request)
820     # Should be BadRequest, but API says otherwise.
821     if not content_type:
822         raise LengthRequired('Missing Content-Type header')
823     
824     if 'hashmap' in request.GET:
825         if request.serialization not in ('json', 'xml'):
826             raise BadRequest('Invalid hashmap format')
827         
828         data = ''
829         for block in socket_read_iterator(request, content_length,
830                                             request.backend.block_size):
831             data = '%s%s' % (data, block)
832         
833         if request.serialization == 'json':
834             d = json.loads(data)
835             if not hasattr(d, '__getitem__'):
836                 raise BadRequest('Invalid data formating')
837             try:
838                 hashmap = d['hashes']
839                 size = int(d['bytes'])
840             except:
841                 raise BadRequest('Invalid data formatting')
842         elif request.serialization == 'xml':
843             try:
844                 xml = minidom.parseString(data)
845                 obj = xml.getElementsByTagName('object')[0]
846                 size = int(obj.attributes['bytes'].value)
847                 
848                 hashes = xml.getElementsByTagName('hash')
849                 hashmap = []
850                 for hash in hashes:
851                     hashmap.append(hash.firstChild.data)
852             except:
853                 raise BadRequest('Invalid data formatting')
854         
855         checksum = '' # Do not set to None (will copy previous value).
856     else:
857         md5 = hashlib.md5()
858         size = 0
859         hashmap = []
860         for data in socket_read_iterator(request, content_length,
861                                             request.backend.block_size):
862             # TODO: Raise 408 (Request Timeout) if this takes too long.
863             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
864             size += len(data)
865             hashmap.append(request.backend.put_block(data))
866             md5.update(data)
867         
868         checksum = md5.hexdigest().lower()
869         etag = request.META.get('HTTP_ETAG')
870         if etag and parse_etags(etag)[0].lower() != checksum:
871             raise UnprocessableEntity('Object ETag does not match')
872     
873     try:
874         version_id = request.backend.update_object_hashmap(request.user_uniq,
875                         v_account, v_container, v_object, size, content_type,
876                         hashmap, checksum, 'pithos', meta, True, permissions)
877     except NotAllowedError:
878         raise Forbidden('Not allowed')
879     except IndexError, e:
880         raise Conflict(simple_list_response(request, e.data))
881     except NameError:
882         raise ItemNotFound('Container does not exist')
883     except ValueError:
884         raise BadRequest('Invalid sharing header')
885     except QuotaError:
886         raise RequestEntityTooLarge('Quota exceeded')
887     if not checksum and UPDATE_MD5:
888         # Update the MD5 after the hashmap, as there may be missing hashes.
889         checksum = hashmap_md5(request.backend, hashmap, size)
890         try:
891             request.backend.update_object_checksum(request.user_uniq,
892               v_account, v_container, v_object, version_id, checksum)
893         except NotAllowedError:
894             raise Forbidden('Not allowed')
895     if public is not None:
896         try:
897             request.backend.update_object_public(request.user_uniq, v_account,
898                                                 v_container, v_object, public)
899         except NotAllowedError:
900             raise Forbidden('Not allowed')
901         except NameError:
902             raise ItemNotFound('Object does not exist')
903     
904     response = HttpResponse(status=201)
905     if checksum:
906         response['ETag'] = checksum
907     response['X-Object-Version'] = version_id
908     return response
909
910 @api_method('POST')
911 def object_write_form(request, v_account, v_container, v_object):
912     # Normal Response Codes: 201
913     # Error Response Codes: internalServerError (500),
914     #                       itemNotFound (404),
915     #                       forbidden (403),
916     #                       badRequest (400)
917     
918     request.upload_handlers = [SaveToBackendHandler(request)]
919     if not request.FILES.has_key('X-Object-Data'):
920         raise BadRequest('Missing X-Object-Data field')
921     file = request.FILES['X-Object-Data']
922     
923     checksum = file.etag
924     try:
925         version_id = request.backend.update_object_hashmap(request.user_uniq,
926                         v_account, v_container, v_object, file.size, file.content_type,
927                         file.hashmap, checksum, 'pithos', {}, True)
928     except NotAllowedError:
929         raise Forbidden('Not allowed')
930     except NameError:
931         raise ItemNotFound('Container does not exist')
932     except QuotaError:
933         raise RequestEntityTooLarge('Quota exceeded')
934     
935     response = HttpResponse(status=201)
936     response['ETag'] = checksum
937     response['X-Object-Version'] = version_id
938     response.content = checksum
939     return response
940
941 @api_method('COPY', format_allowed=True)
942 def object_copy(request, v_account, v_container, v_object):
943     # Normal Response Codes: 201
944     # Error Response Codes: internalServerError (500),
945     #                       itemNotFound (404),
946     #                       forbidden (403),
947     #                       badRequest (400)
948     
949     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
950     if not dest_account:
951         dest_account = request.user_uniq
952     dest_path = request.META.get('HTTP_DESTINATION')
953     if not dest_path:
954         raise BadRequest('Missing Destination header')
955     try:
956         dest_container, dest_name = split_container_object_string(dest_path)
957     except ValueError:
958         raise BadRequest('Invalid Destination header')
959     
960     # Evaluate conditions.
961     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
962         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
963         try:
964             meta = request.backend.get_object_meta(request.user_uniq, v_account,
965                                             v_container, v_object, 'pithos', src_version)
966         except NotAllowedError:
967             raise Forbidden('Not allowed')
968         except (NameError, IndexError):
969             raise ItemNotFound('Container or object does not exist')
970         validate_matching_preconditions(request, meta)
971     
972     version_id = copy_or_move_object(request, v_account, v_container, v_object,
973                                         dest_account, dest_container, dest_name, move=False)
974     response = HttpResponse(status=201)
975     response['X-Object-Version'] = version_id
976     return response
977
978 @api_method('MOVE', format_allowed=True)
979 def object_move(request, v_account, v_container, v_object):
980     # Normal Response Codes: 201
981     # Error Response Codes: internalServerError (500),
982     #                       itemNotFound (404),
983     #                       forbidden (403),
984     #                       badRequest (400)
985     
986     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
987     if not dest_account:
988         dest_account = request.user_uniq
989     dest_path = request.META.get('HTTP_DESTINATION')
990     if not dest_path:
991         raise BadRequest('Missing Destination header')
992     try:
993         dest_container, dest_name = split_container_object_string(dest_path)
994     except ValueError:
995         raise BadRequest('Invalid Destination header')
996     
997     # Evaluate conditions.
998     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
999         try:
1000             meta = request.backend.get_object_meta(request.user_uniq, v_account,
1001                                                     v_container, v_object, 'pithos')
1002         except NotAllowedError:
1003             raise Forbidden('Not allowed')
1004         except NameError:
1005             raise ItemNotFound('Container or object does not exist')
1006         validate_matching_preconditions(request, meta)
1007     
1008     version_id = copy_or_move_object(request, v_account, v_container, v_object,
1009                                         dest_account, dest_container, dest_name, move=True)
1010     response = HttpResponse(status=201)
1011     response['X-Object-Version'] = version_id
1012     return response
1013
1014 @api_method('POST', format_allowed=True)
1015 def object_update(request, v_account, v_container, v_object):
1016     # Normal Response Codes: 202, 204
1017     # Error Response Codes: internalServerError (500),
1018     #                       conflict (409),
1019     #                       itemNotFound (404),
1020     #                       forbidden (403),
1021     #                       badRequest (400)
1022     
1023     content_type, meta, permissions, public = get_object_headers(request)
1024     
1025     try:
1026         prev_meta = request.backend.get_object_meta(request.user_uniq, v_account,
1027                                                     v_container, v_object, 'pithos')
1028     except NotAllowedError:
1029         raise Forbidden('Not allowed')
1030     except NameError:
1031         raise ItemNotFound('Object does not exist')
1032     
1033     # Evaluate conditions.
1034     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1035         validate_matching_preconditions(request, prev_meta)
1036     
1037     replace = True
1038     if 'update' in request.GET:
1039         replace = False
1040     
1041     # A Content-Type or X-Source-Object header indicates data updates.
1042     src_object = request.META.get('HTTP_X_SOURCE_OBJECT')
1043     if (not content_type or content_type != 'application/octet-stream') and not src_object:
1044         response = HttpResponse(status=202)
1045         
1046         # Do permissions first, as it may fail easier.
1047         if permissions is not None:
1048             try:
1049                 request.backend.update_object_permissions(request.user_uniq,
1050                                 v_account, v_container, v_object, permissions)
1051             except NotAllowedError:
1052                 raise Forbidden('Not allowed')
1053             except NameError:
1054                 raise ItemNotFound('Object does not exist')
1055             except ValueError:
1056                 raise BadRequest('Invalid sharing header')
1057         if public is not None:
1058             try:
1059                 request.backend.update_object_public(request.user_uniq, v_account,
1060                                                 v_container, v_object, public)
1061             except NotAllowedError:
1062                 raise Forbidden('Not allowed')
1063             except NameError:
1064                 raise ItemNotFound('Object does not exist')
1065         if meta or replace:
1066             try:
1067                 version_id = request.backend.update_object_meta(request.user_uniq,
1068                                 v_account, v_container, v_object, 'pithos', meta, replace)
1069             except NotAllowedError:
1070                 raise Forbidden('Not allowed')
1071             except NameError:
1072                 raise ItemNotFound('Object does not exist')        
1073             response['X-Object-Version'] = version_id
1074         
1075         return response
1076     
1077     # Single range update. Range must be in Content-Range.
1078     # Based on: http://code.google.com/p/gears/wiki/ContentRangePostProposal
1079     # (with the addition that '*' is allowed for the range - will append).
1080     content_range = request.META.get('HTTP_CONTENT_RANGE')
1081     if not content_range:
1082         raise BadRequest('Missing Content-Range header')
1083     ranges = get_content_range(request)
1084     if not ranges:
1085         raise RangeNotSatisfiable('Invalid Content-Range header')
1086     
1087     try:
1088         size, hashmap = request.backend.get_object_hashmap(request.user_uniq,
1089                                             v_account, v_container, v_object)
1090     except NotAllowedError:
1091         raise Forbidden('Not allowed')
1092     except NameError:
1093         raise ItemNotFound('Object does not exist')
1094     
1095     offset, length, total = ranges
1096     if offset is None:
1097         offset = size
1098     elif offset > size:
1099         raise RangeNotSatisfiable('Supplied offset is beyond object limits')
1100     if src_object:
1101         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
1102         if not src_account:
1103             src_account = request.user_uniq
1104         src_container, src_name = split_container_object_string(src_object)
1105         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
1106         try:
1107             src_size, src_hashmap = request.backend.get_object_hashmap(request.user_uniq,
1108                                         src_account, src_container, src_name, src_version)
1109         except NotAllowedError:
1110             raise Forbidden('Not allowed')
1111         except NameError:
1112             raise ItemNotFound('Source object does not exist')
1113         
1114         if length is None:
1115             length = src_size
1116         elif length > src_size:
1117             raise BadRequest('Object length is smaller than range length')
1118     else:
1119         # Require either a Content-Length, or 'chunked' Transfer-Encoding.
1120         content_length = -1
1121         if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
1122             content_length = get_content_length(request)
1123         
1124         if length is None:
1125             length = content_length
1126         else:
1127             if content_length == -1:
1128                 # TODO: Get up to length bytes in chunks.
1129                 length = content_length
1130             elif length != content_length:
1131                 raise BadRequest('Content length does not match range length')
1132     if total is not None and (total != size or offset >= size or (length > 0 and offset + length >= size)):
1133         raise RangeNotSatisfiable('Supplied range will change provided object limits')
1134     
1135     dest_bytes = request.META.get('HTTP_X_OBJECT_BYTES')
1136     if dest_bytes is not None:
1137         dest_bytes = get_int_parameter(dest_bytes)
1138         if dest_bytes is None:
1139             raise BadRequest('Invalid X-Object-Bytes header')
1140     
1141     if src_object:
1142         if offset % request.backend.block_size == 0:
1143             # Update the hashes only.
1144             sbi = 0
1145             while length > 0:
1146                 bi = int(offset / request.backend.block_size)
1147                 bl = min(length, request.backend.block_size)
1148                 if bi < len(hashmap):
1149                     if bl == request.backend.block_size:
1150                         hashmap[bi] = src_hashmap[sbi]
1151                     else:
1152                         data = request.backend.get_block(src_hashmap[sbi])
1153                         hashmap[bi] = request.backend.update_block(hashmap[bi],
1154                                                                 data[:bl], 0)
1155                 else:
1156                     hashmap.append(src_hashmap[sbi])
1157                 offset += bl
1158                 length -= bl
1159                 sbi += 1
1160         else:
1161             data = ''
1162             sbi = 0
1163             while length > 0:
1164                 data += request.backend.get_block(src_hashmap[sbi])
1165                 if length < request.backend.block_size:
1166                     data = data[:length]
1167                 bytes = put_object_block(request, hashmap, data, offset)
1168                 offset += bytes
1169                 data = data[bytes:]
1170                 length -= bytes
1171                 sbi += 1
1172     else:
1173         data = ''
1174         for d in socket_read_iterator(request, length,
1175                                         request.backend.block_size):
1176             # TODO: Raise 408 (Request Timeout) if this takes too long.
1177             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
1178             data += d
1179             bytes = put_object_block(request, hashmap, data, offset)
1180             offset += bytes
1181             data = data[bytes:]
1182         if len(data) > 0:
1183             put_object_block(request, hashmap, data, offset)
1184     
1185     if offset > size:
1186         size = offset
1187     if dest_bytes is not None and dest_bytes < size:
1188         size = dest_bytes
1189         hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)]
1190     checksum = hashmap_md5(request.backend, hashmap, size) if UPDATE_MD5 else ''
1191     try:
1192         version_id = request.backend.update_object_hashmap(request.user_uniq,
1193                         v_account, v_container, v_object, size, prev_meta['type'],
1194                         hashmap, checksum, 'pithos', meta, replace, permissions)
1195     except NotAllowedError:
1196         raise Forbidden('Not allowed')
1197     except NameError:
1198         raise ItemNotFound('Container does not exist')
1199     except ValueError:
1200         raise BadRequest('Invalid sharing header')
1201     except QuotaError:
1202         raise RequestEntityTooLarge('Quota exceeded')
1203     if public is not None:
1204         try:
1205             request.backend.update_object_public(request.user_uniq, v_account,
1206                                                 v_container, v_object, public)
1207         except NotAllowedError:
1208             raise Forbidden('Not allowed')
1209         except NameError:
1210             raise ItemNotFound('Object does not exist')
1211     
1212     response = HttpResponse(status=204)
1213     response['ETag'] = checksum
1214     response['X-Object-Version'] = version_id
1215     return response
1216
1217 @api_method('DELETE')
1218 def object_delete(request, v_account, v_container, v_object):
1219     # Normal Response Codes: 204
1220     # Error Response Codes: internalServerError (500),
1221     #                       itemNotFound (404),
1222     #                       forbidden (403),
1223     #                       badRequest (400)
1224     
1225     until = get_int_parameter(request.GET.get('until'))
1226     try:
1227         request.backend.delete_object(request.user_uniq, v_account, v_container,
1228                                         v_object, until)
1229     except NotAllowedError:
1230         raise Forbidden('Not allowed')
1231     except NameError:
1232         raise ItemNotFound('Object does not exist')
1233     return HttpResponse(status=204)
1234
1235 @api_method()
1236 def method_not_allowed(request):
1237     raise BadRequest('Method not allowed')