c27ad1cbb72b8210447d676cc2b70759f82f9625
[pithos] / snf-pithos-app / pithos / api / functions.py
1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or
4 # without modification, are permitted provided that the following
5 # conditions are met:
6 #
7 #   1. Redistributions of source code must retain the above
8 #      copyright notice, this list of conditions and the following
9 #      disclaimer.
10 #
11 #   2. Redistributions in binary form must reproduce the above
12 #      copyright notice, this list of conditions and the following
13 #      disclaimer in the documentation and/or other materials
14 #      provided with the distribution.
15 #
16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 # POSSIBILITY OF SUCH DAMAGE.
28 #
29 # The views and conclusions contained in the software and
30 # documentation are those of the authors and should not be
31 # interpreted as representing official policies, either expressed
32 # or implied, of GRNET S.A.
33
34 from xml.dom import minidom
35 from urllib import unquote
36
37 from django.conf import settings
38 from django.http import HttpResponse
39 from django.template.loader import render_to_string
40 from django.utils import simplejson as json
41 from django.utils.http import parse_etags
42 from django.utils.encoding import smart_str
43 from django.views.decorators.csrf import csrf_exempt
44
45 from synnefo.lib.astakos import get_user
46
47 from pithos.api.faults import (
48     Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
49     LengthRequired, PreconditionFailed, RequestEntityTooLarge, RangeNotSatisfiable, UnprocessableEntity)
50 from pithos.api.util import (
51     json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
52     get_account_headers, put_account_headers, get_container_headers, put_container_headers, get_object_headers,
53     put_object_headers, update_manifest_meta, update_sharing_meta, update_public_meta,
54     validate_modification_preconditions, validate_matching_preconditions, split_container_object_string,
55     copy_or_move_object, get_int_parameter, get_content_length, get_content_range, socket_read_iterator,
56     SaveToBackendHandler, object_data_response, put_object_block, hashmap_md5, simple_list_response, api_method)
57 from pithos.api.settings import UPDATE_MD5
58
59 from pithos.backends.base import NotAllowedError, QuotaError, ContainerNotEmpty, ItemNotExists, VersionNotExists
60
61 from pithos.backends.filter import parse_filters
62
63 import logging
64 import hashlib
65
66
67 logger = logging.getLogger(__name__)
68
69
70 @csrf_exempt
71 def top_demux(request):
72     if request.method == 'GET':
73         try:
74             request.GET['X-Auth-Token']
75         except KeyError:
76             try:
77                 request.META['HTTP_X_AUTH_TOKEN']
78             except KeyError:
79                 return authenticate(request)
80         return account_list(request)
81     else:
82         return method_not_allowed(request)
83
84
85 @csrf_exempt
86 def account_demux(request, v_account):
87     if request.method == 'HEAD':
88         return account_meta(request, v_account)
89     elif request.method == 'POST':
90         return account_update(request, v_account)
91     elif request.method == 'GET':
92         return container_list(request, v_account)
93     else:
94         return method_not_allowed(request)
95
96
97 @csrf_exempt
98 def container_demux(request, v_account, v_container):
99     if request.method == 'HEAD':
100         return container_meta(request, v_account, v_container)
101     elif request.method == 'PUT':
102         return container_create(request, v_account, v_container)
103     elif request.method == 'POST':
104         return container_update(request, v_account, v_container)
105     elif request.method == 'DELETE':
106         return container_delete(request, v_account, v_container)
107     elif request.method == 'GET':
108         return object_list(request, v_account, v_container)
109     else:
110         return method_not_allowed(request)
111
112
113 @csrf_exempt
114 def object_demux(request, v_account, v_container, v_object):
115     # Helper to avoid placing the token in the URL when loading objects from a browser.
116     if request.method == 'HEAD':
117         return object_meta(request, v_account, v_container, v_object)
118     elif request.method == 'GET':
119         return object_read(request, v_account, v_container, v_object)
120     elif request.method == 'PUT':
121         return object_write(request, v_account, v_container, v_object)
122     elif request.method == 'COPY':
123         return object_copy(request, v_account, v_container, v_object)
124     elif request.method == 'MOVE':
125         return object_move(request, v_account, v_container, v_object)
126     elif request.method == 'POST':
127         if request.META.get('CONTENT_TYPE', '').startswith('multipart/form-data'):
128             return object_write_form(request, v_account, v_container, v_object)
129         return object_update(request, v_account, v_container, v_object)
130     elif request.method == 'DELETE':
131         return object_delete(request, v_account, v_container, v_object)
132     else:
133         return method_not_allowed(request)
134
135
136 @api_method('GET', user_required=False)
137 def authenticate(request):
138     # Normal Response Codes: 204
139     # Error Response Codes: internalServerError (500),
140     #                       forbidden (403),
141     #                       badRequest (400)
142
143     x_auth_user = request.META.get('HTTP_X_AUTH_USER')
144     x_auth_key = request.META.get('HTTP_X_AUTH_KEY')
145     if not x_auth_user or not x_auth_key:
146         raise BadRequest('Missing X-Auth-User or X-Auth-Key header')
147     response = HttpResponse(status=204)
148
149     uri = request.build_absolute_uri()
150     if '?' in uri:
151         uri = uri[:uri.find('?')]
152
153     response['X-Auth-Token'] = x_auth_key
154     response['X-Storage-Url'] = uri + ('' if uri.endswith('/')
155                                        else '/') + x_auth_user
156     return response
157
158
159 @api_method('GET', format_allowed=True)
160 def account_list(request):
161     # Normal Response Codes: 200, 204
162     # Error Response Codes: internalServerError (500),
163     #                       badRequest (400)
164     response = HttpResponse()
165
166     marker = request.GET.get('marker')
167     limit = get_int_parameter(request.GET.get('limit'))
168     if not limit:
169         limit = 10000
170
171     accounts = request.backend.list_accounts(request.user_uniq, marker, limit)
172
173     if request.serialization == 'text':
174         if len(accounts) == 0:
175             # The cloudfiles python bindings expect 200 if json/xml.
176             response.status_code = 204
177             return response
178         response.status_code = 200
179         response.content = '\n'.join(accounts) + '\n'
180         return response
181
182     account_meta = []
183     for x in accounts:
184         if x == request.user_uniq:
185             continue
186         try:
187             meta = request.backend.get_account_meta(
188                 request.user_uniq, x, 'pithos', include_user_defined=False)
189             groups = request.backend.get_account_groups(request.user_uniq, x)
190         except NotAllowedError:
191             raise Forbidden('Not allowed')
192         else:
193             rename_meta_key(meta, 'modified', 'last_modified')
194             rename_meta_key(
195                 meta, 'until_timestamp', 'x_account_until_timestamp')
196             if groups:
197                 meta['X-Account-Group'] = printable_header_dict(
198                     dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
199             account_meta.append(printable_header_dict(meta))
200     if request.serialization == 'xml':
201         data = render_to_string('accounts.xml', {'accounts': account_meta})
202     elif request.serialization == 'json':
203         data = json.dumps(account_meta)
204     response.status_code = 200
205     response.content = data
206     return response
207
208
209 @api_method('HEAD')
210 def account_meta(request, v_account):
211     # Normal Response Codes: 204
212     # Error Response Codes: internalServerError (500),
213     #                       forbidden (403),
214     #                       badRequest (400)
215
216     until = get_int_parameter(request.GET.get('until'))
217     try:
218         meta = request.backend.get_account_meta(
219             request.user_uniq, v_account, 'pithos', until)
220         groups = request.backend.get_account_groups(
221             request.user_uniq, v_account)
222         policy = request.backend.get_account_policy(
223             request.user_uniq, v_account)
224     except NotAllowedError:
225         raise Forbidden('Not allowed')
226
227     validate_modification_preconditions(request, meta)
228
229     response = HttpResponse(status=204)
230     put_account_headers(response, meta, groups, policy)
231     return response
232
233
234 @api_method('POST')
235 def account_update(request, v_account):
236     # Normal Response Codes: 202
237     # Error Response Codes: internalServerError (500),
238     #                       forbidden (403),
239     #                       badRequest (400)
240
241     meta, groups = get_account_headers(request)
242     replace = True
243     if 'update' in request.GET:
244         replace = False
245     if groups:
246         try:
247             request.backend.update_account_groups(request.user_uniq, v_account,
248                                                   groups, replace)
249         except NotAllowedError:
250             raise Forbidden('Not allowed')
251         except ValueError:
252             raise BadRequest('Invalid groups header')
253     if meta or replace:
254         try:
255             request.backend.update_account_meta(request.user_uniq, v_account,
256                                                 'pithos', meta, replace)
257         except NotAllowedError:
258             raise Forbidden('Not allowed')
259     return HttpResponse(status=202)
260
261
262 @api_method('GET', format_allowed=True)
263 def container_list(request, v_account):
264     # Normal Response Codes: 200, 204
265     # Error Response Codes: internalServerError (500),
266     #                       itemNotFound (404),
267     #                       forbidden (403),
268     #                       badRequest (400)
269
270     until = get_int_parameter(request.GET.get('until'))
271     try:
272         meta = request.backend.get_account_meta(
273             request.user_uniq, v_account, 'pithos', until)
274         groups = request.backend.get_account_groups(
275             request.user_uniq, v_account)
276         policy = request.backend.get_account_policy(
277             request.user_uniq, v_account)
278     except NotAllowedError:
279         raise Forbidden('Not allowed')
280
281     validate_modification_preconditions(request, meta)
282
283     response = HttpResponse()
284     put_account_headers(response, meta, groups, policy)
285
286     marker = request.GET.get('marker')
287     limit = get_int_parameter(request.GET.get('limit'))
288     if not limit:
289         limit = 10000
290
291     shared = False
292     if 'shared' in request.GET:
293         shared = True
294     public = False
295     if 'public' in request.GET:
296         public = True
297
298     try:
299         containers = request.backend.list_containers(
300             request.user_uniq, v_account,
301             marker, limit, shared, until, public)
302     except NotAllowedError:
303         raise Forbidden('Not allowed')
304     except NameError:
305         containers = []
306
307     if request.serialization == 'text':
308         if len(containers) == 0:
309             # The cloudfiles python bindings expect 200 if json/xml.
310             response.status_code = 204
311             return response
312         response.status_code = 200
313         response.content = '\n'.join(containers) + '\n'
314         return response
315
316     container_meta = []
317     for x in containers:
318         try:
319             meta = request.backend.get_container_meta(
320                 request.user_uniq, v_account,
321                 x, 'pithos', until, include_user_defined=False)
322             policy = request.backend.get_container_policy(request.user_uniq,
323                                                           v_account, x)
324         except NotAllowedError:
325             raise Forbidden('Not allowed')
326         except NameError:
327             pass
328         else:
329             rename_meta_key(meta, 'modified', 'last_modified')
330             rename_meta_key(
331                 meta, 'until_timestamp', 'x_container_until_timestamp')
332             if policy:
333                 meta['X-Container-Policy'] = printable_header_dict(
334                     dict([(k, v) for k, v in policy.iteritems()]))
335             container_meta.append(printable_header_dict(meta))
336     if request.serialization == 'xml':
337         data = render_to_string('containers.xml', {'account':
338                                 v_account, 'containers': container_meta})
339     elif request.serialization == 'json':
340         data = json.dumps(container_meta)
341     response.status_code = 200
342     response.content = data
343     return response
344
345
346 @api_method('HEAD')
347 def container_meta(request, v_account, v_container):
348     # Normal Response Codes: 204
349     # Error Response Codes: internalServerError (500),
350     #                       itemNotFound (404),
351     #                       forbidden (403),
352     #                       badRequest (400)
353
354     until = get_int_parameter(request.GET.get('until'))
355     try:
356         meta = request.backend.get_container_meta(request.user_uniq, v_account,
357                                                   v_container, 'pithos', until)
358         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
359                                                                   v_account, v_container, 'pithos', until)
360         policy = request.backend.get_container_policy(
361             request.user_uniq, v_account,
362             v_container)
363     except NotAllowedError:
364         raise Forbidden('Not allowed')
365     except ItemNotExists:
366         raise ItemNotFound('Container does not exist')
367
368     validate_modification_preconditions(request, meta)
369
370     response = HttpResponse(status=204)
371     put_container_headers(request, response, meta, policy)
372     return response
373
374
375 @api_method('PUT')
376 def container_create(request, v_account, v_container):
377     # Normal Response Codes: 201, 202
378     # Error Response Codes: internalServerError (500),
379     #                       itemNotFound (404),
380     #                       forbidden (403),
381     #                       badRequest (400)
382
383     meta, policy = get_container_headers(request)
384
385     try:
386         request.backend.put_container(
387             request.user_uniq, v_account, v_container, policy)
388         ret = 201
389     except NotAllowedError:
390         raise Forbidden('Not allowed')
391     except ValueError:
392         raise BadRequest('Invalid policy header')
393     except NameError:
394         ret = 202
395
396     if ret == 202 and policy:
397         try:
398             request.backend.update_container_policy(
399                 request.user_uniq, v_account,
400                 v_container, policy, replace=False)
401         except NotAllowedError:
402             raise Forbidden('Not allowed')
403         except ItemNotExists:
404             raise ItemNotFound('Container does not exist')
405         except ValueError:
406             raise BadRequest('Invalid policy header')
407     if meta:
408         try:
409             request.backend.update_container_meta(request.user_uniq, v_account,
410                                                   v_container, 'pithos', meta, replace=False)
411         except NotAllowedError:
412             raise Forbidden('Not allowed')
413         except ItemNotExists:
414             raise ItemNotFound('Container does not exist')
415
416     return HttpResponse(status=ret)
417
418
419 @api_method('POST', format_allowed=True)
420 def container_update(request, v_account, v_container):
421     # Normal Response Codes: 202
422     # Error Response Codes: internalServerError (500),
423     #                       itemNotFound (404),
424     #                       forbidden (403),
425     #                       badRequest (400)
426
427     meta, policy = get_container_headers(request)
428     replace = True
429     if 'update' in request.GET:
430         replace = False
431     if policy:
432         try:
433             request.backend.update_container_policy(
434                 request.user_uniq, v_account,
435                 v_container, policy, replace)
436         except NotAllowedError:
437             raise Forbidden('Not allowed')
438         except ItemNotExists:
439             raise ItemNotFound('Container does not exist')
440         except ValueError:
441             raise BadRequest('Invalid policy header')
442     if meta or replace:
443         try:
444             request.backend.update_container_meta(request.user_uniq, v_account,
445                                                   v_container, 'pithos', meta, replace)
446         except NotAllowedError:
447             raise Forbidden('Not allowed')
448         except ItemNotExists:
449             raise ItemNotFound('Container does not exist')
450
451     content_length = -1
452     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
453         content_length = get_int_parameter(
454             request.META.get('CONTENT_LENGTH', 0))
455     content_type = request.META.get('CONTENT_TYPE')
456     hashmap = []
457     if content_type and content_type == 'application/octet-stream' and content_length != 0:
458         for data in socket_read_iterator(request, content_length,
459                                          request.backend.block_size):
460             # TODO: Raise 408 (Request Timeout) if this takes too long.
461             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
462             hashmap.append(request.backend.put_block(data))
463
464     response = HttpResponse(status=202)
465     if hashmap:
466         response.content = simple_list_response(request, hashmap)
467     return response
468
469
470 @api_method('DELETE')
471 def container_delete(request, v_account, v_container):
472     # Normal Response Codes: 204
473     # Error Response Codes: internalServerError (500),
474     #                       conflict (409),
475     #                       itemNotFound (404),
476     #                       forbidden (403),
477     #                       badRequest (400)
478
479     until = get_int_parameter(request.GET.get('until'))
480
481     delimiter = request.GET.get('delimiter')
482
483     try:
484         request.backend.delete_container(
485             request.user_uniq, v_account, v_container,
486             until, delimiter=delimiter)
487     except NotAllowedError:
488         raise Forbidden('Not allowed')
489     except ItemNotExists:
490         raise ItemNotFound('Container does not exist')
491     except ContainerNotEmpty:
492         raise Conflict('Container is not empty')
493     return HttpResponse(status=204)
494
495
496 @api_method('GET', format_allowed=True)
497 def object_list(request, v_account, v_container):
498     # Normal Response Codes: 200, 204
499     # Error Response Codes: internalServerError (500),
500     #                       itemNotFound (404),
501     #                       forbidden (403),
502     #                       badRequest (400)
503
504     until = get_int_parameter(request.GET.get('until'))
505     try:
506         meta = request.backend.get_container_meta(request.user_uniq, v_account,
507                                                   v_container, 'pithos', until)
508         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
509                                                                   v_account, v_container, 'pithos', until)
510         policy = request.backend.get_container_policy(
511             request.user_uniq, v_account,
512             v_container)
513     except NotAllowedError:
514         raise Forbidden('Not allowed')
515     except ItemNotExists:
516         raise ItemNotFound('Container does not exist')
517
518     validate_modification_preconditions(request, meta)
519
520     response = HttpResponse()
521     put_container_headers(request, response, meta, policy)
522
523     path = request.GET.get('path')
524     prefix = request.GET.get('prefix')
525     delimiter = request.GET.get('delimiter')
526
527     # Path overrides prefix and delimiter.
528     virtual = True
529     if path:
530         prefix = path
531         delimiter = '/'
532         virtual = False
533
534     # Naming policy.
535     if prefix and delimiter and not prefix.endswith(delimiter):
536         prefix = prefix + delimiter
537     if not prefix:
538         prefix = ''
539     prefix = prefix.lstrip('/')
540
541     marker = request.GET.get('marker')
542     limit = get_int_parameter(request.GET.get('limit'))
543     if not limit:
544         limit = 10000
545
546     keys = request.GET.get('meta')
547     if keys:
548         keys = [smart_str(x.strip()) for x in keys.split(',')
549                 if x.strip() != '']
550         included, excluded, opers = parse_filters(keys)
551         keys = []
552         keys += [format_header_key('X-Object-Meta-' + x) for x in included]
553         keys += ['!' + format_header_key('X-Object-Meta-' + x)
554                  for x in excluded]
555         keys += ['%s%s%s' % (format_header_key(
556             'X-Object-Meta-' + k), o, v) for k, o, v in opers]
557     else:
558         keys = []
559
560     shared = False
561     if 'shared' in request.GET:
562         shared = True
563     public = False
564     if 'public' in request.GET:
565         public = True
566
567     if request.serialization == 'text':
568         try:
569             objects = request.backend.list_objects(
570                 request.user_uniq, v_account,
571                 v_container, prefix, delimiter, marker,
572                 limit, virtual, 'pithos', keys, shared,
573                 until, None, public)
574         except NotAllowedError:
575             raise Forbidden('Not allowed')
576         except ItemNotExists:
577             raise ItemNotFound('Container does not exist')
578
579         if len(objects) == 0:
580             # The cloudfiles python bindings expect 200 if json/xml.
581             response.status_code = 204
582             return response
583         response.status_code = 200
584         response.content = '\n'.join([x[0] for x in objects]) + '\n'
585         return response
586
587     try:
588         objects = request.backend.list_object_meta(
589             request.user_uniq, v_account,
590             v_container, prefix, delimiter, marker,
591             limit, virtual, 'pithos', keys, shared, until, None, public)
592         object_permissions = {}
593         object_public = {}
594         if until is None:
595             name_idx = len('/'.join((v_account, v_container, '')))
596             for x in request.backend.list_object_permissions(request.user_uniq,
597                                                              v_account, v_container, prefix):
598                 object = x[name_idx:]
599                 object_permissions[object] = request.backend.get_object_permissions(
600                     request.user_uniq, v_account, v_container, object)
601             for k, v in request.backend.list_object_public(request.user_uniq,
602                                                            v_account, v_container, prefix).iteritems():
603                 object_public[k[name_idx:]] = v
604     except NotAllowedError:
605         raise Forbidden('Not allowed')
606     except ItemNotExists:
607         raise ItemNotFound('Container does not exist')
608
609     object_meta = []
610     for meta in objects:
611         if len(meta) == 1:
612             # Virtual objects/directories.
613             object_meta.append(meta)
614         else:
615             rename_meta_key(
616                 meta, 'hash', 'x_object_hash')  # Will be replaced by checksum.
617             rename_meta_key(meta, 'checksum', 'hash')
618             rename_meta_key(meta, 'type', 'content_type')
619             rename_meta_key(meta, 'uuid', 'x_object_uuid')
620             if until is not None and 'modified' in meta:
621                 del(meta['modified'])
622             else:
623                 rename_meta_key(meta, 'modified', 'last_modified')
624             rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
625             rename_meta_key(meta, 'version', 'x_object_version')
626             rename_meta_key(
627                 meta, 'version_timestamp', 'x_object_version_timestamp')
628             permissions = object_permissions.get(meta['name'], None)
629             if permissions:
630                 update_sharing_meta(request, permissions, v_account,
631                                     v_container, meta['name'], meta)
632             public = object_public.get(meta['name'], None)
633             if public:
634                 update_public_meta(public, meta)
635             object_meta.append(printable_header_dict(meta))
636     if request.serialization == 'xml':
637         data = render_to_string(
638             'objects.xml', {'container': v_container, 'objects': object_meta})
639     elif request.serialization == 'json':
640         data = json.dumps(object_meta, default=json_encode_decimal)
641     response.status_code = 200
642     response.content = data
643     return response
644
645
646 @api_method('HEAD')
647 def object_meta(request, v_account, v_container, v_object):
648     # Normal Response Codes: 204
649     # Error Response Codes: internalServerError (500),
650     #                       itemNotFound (404),
651     #                       forbidden (403),
652     #                       badRequest (400)
653
654     version = request.GET.get('version')
655     try:
656         meta = request.backend.get_object_meta(request.user_uniq, v_account,
657                                                v_container, v_object, 'pithos', version)
658         if version is None:
659             permissions = request.backend.get_object_permissions(
660                 request.user_uniq,
661                 v_account, v_container, v_object)
662             public = request.backend.get_object_public(
663                 request.user_uniq, v_account,
664                 v_container, v_object)
665         else:
666             permissions = None
667             public = None
668     except NotAllowedError:
669         raise Forbidden('Not allowed')
670     except ItemNotExists:
671         raise ItemNotFound('Object does not exist')
672     except VersionNotExists:
673         raise ItemNotFound('Version does not exist')
674
675     update_manifest_meta(request, v_account, meta)
676     update_sharing_meta(
677         request, permissions, v_account, v_container, v_object, meta)
678     update_public_meta(public, meta)
679
680     # Evaluate conditions.
681     validate_modification_preconditions(request, meta)
682     try:
683         validate_matching_preconditions(request, meta)
684     except NotModified:
685         response = HttpResponse(status=304)
686         response['ETag'] = meta['checksum']
687         return response
688
689     response = HttpResponse(status=200)
690     put_object_headers(response, meta)
691     return response
692
693
694 @api_method('GET', format_allowed=True)
695 def object_read(request, v_account, v_container, v_object):
696     # Normal Response Codes: 200, 206
697     # Error Response Codes: internalServerError (500),
698     #                       rangeNotSatisfiable (416),
699     #                       preconditionFailed (412),
700     #                       itemNotFound (404),
701     #                       forbidden (403),
702     #                       badRequest (400),
703     #                       notModified (304)
704
705     version = request.GET.get('version')
706
707     # Reply with the version list. Do this first, as the object may be deleted.
708     if version == 'list':
709         if request.serialization == 'text':
710             raise BadRequest('No format specified for version list.')
711
712         try:
713             v = request.backend.list_versions(request.user_uniq, v_account,
714                                               v_container, v_object)
715         except NotAllowedError:
716             raise Forbidden('Not allowed')
717         d = {'versions': v}
718         if request.serialization == 'xml':
719             d['object'] = v_object
720             data = render_to_string('versions.xml', d)
721         elif request.serialization == 'json':
722             data = json.dumps(d, default=json_encode_decimal)
723
724         response = HttpResponse(data, status=200)
725         response['Content-Length'] = len(data)
726         return response
727
728     try:
729         meta = request.backend.get_object_meta(request.user_uniq, v_account,
730                                                v_container, v_object, 'pithos', version)
731         if version is None:
732             permissions = request.backend.get_object_permissions(
733                 request.user_uniq,
734                 v_account, v_container, v_object)
735             public = request.backend.get_object_public(
736                 request.user_uniq, v_account,
737                 v_container, v_object)
738         else:
739             permissions = None
740             public = None
741     except NotAllowedError:
742         raise Forbidden('Not allowed')
743     except ItemNotExists:
744         raise ItemNotFound('Object does not exist')
745     except VersionNotExists:
746         raise ItemNotFound('Version does not exist')
747
748     update_manifest_meta(request, v_account, meta)
749     update_sharing_meta(
750         request, permissions, v_account, v_container, v_object, meta)
751     update_public_meta(public, meta)
752
753     # Evaluate conditions.
754     validate_modification_preconditions(request, meta)
755     try:
756         validate_matching_preconditions(request, meta)
757     except NotModified:
758         response = HttpResponse(status=304)
759         response['ETag'] = meta['checksum']
760         return response
761
762     hashmap_reply = False
763     if 'hashmap' in request.GET and request.serialization != 'text':
764         hashmap_reply = True
765
766     sizes = []
767     hashmaps = []
768     if 'X-Object-Manifest' in meta and not hashmap_reply:
769         try:
770             src_container, src_name = split_container_object_string(
771                 '/' + meta['X-Object-Manifest'])
772             objects = request.backend.list_objects(
773                 request.user_uniq, v_account,
774                 src_container, prefix=src_name, virtual=False)
775         except NotAllowedError:
776             raise Forbidden('Not allowed')
777         except ValueError:
778             raise BadRequest('Invalid X-Object-Manifest header')
779         except ItemNotExists:
780             raise ItemNotFound('Container does not exist')
781
782         try:
783             for x in objects:
784                 s, h = request.backend.get_object_hashmap(request.user_uniq,
785                                                           v_account, src_container, x[0], x[1])
786                 sizes.append(s)
787                 hashmaps.append(h)
788         except NotAllowedError:
789             raise Forbidden('Not allowed')
790         except ItemNotExists:
791             raise ItemNotFound('Object does not exist')
792         except VersionNotExists:
793             raise ItemNotFound('Version does not exist')
794     else:
795         try:
796             s, h = request.backend.get_object_hashmap(
797                 request.user_uniq, v_account,
798                 v_container, v_object, version)
799             sizes.append(s)
800             hashmaps.append(h)
801         except NotAllowedError:
802             raise Forbidden('Not allowed')
803         except ItemNotExists:
804             raise ItemNotFound('Object does not exist')
805         except VersionNotExists:
806             raise ItemNotFound('Version does not exist')
807
808     # Reply with the hashmap.
809     if hashmap_reply:
810         size = sum(sizes)
811         hashmap = sum(hashmaps, [])
812         d = {
813             'block_size': request.backend.block_size,
814             'block_hash': request.backend.hash_algorithm,
815             'bytes': size,
816             'hashes': hashmap}
817         if request.serialization == 'xml':
818             d['object'] = v_object
819             data = render_to_string('hashes.xml', d)
820         elif request.serialization == 'json':
821             data = json.dumps(d)
822
823         response = HttpResponse(data, status=200)
824         put_object_headers(response, meta)
825         response['Content-Length'] = len(data)
826         return response
827
828     request.serialization = 'text'  # Unset.
829     return object_data_response(request, sizes, hashmaps, meta)
830
831
832 @api_method('PUT', format_allowed=True)
833 def object_write(request, v_account, v_container, v_object):
834     # Normal Response Codes: 201
835     # Error Response Codes: internalServerError (500),
836     #                       unprocessableEntity (422),
837     #                       lengthRequired (411),
838     #                       conflict (409),
839     #                       itemNotFound (404),
840     #                       forbidden (403),
841     #                       badRequest (400)
842
843     # Evaluate conditions.
844     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
845         try:
846             meta = request.backend.get_object_meta(
847                 request.user_uniq, v_account,
848                 v_container, v_object, 'pithos')
849         except NotAllowedError:
850             raise Forbidden('Not allowed')
851         except NameError:
852             meta = {}
853         validate_matching_preconditions(request, meta)
854
855     copy_from = request.META.get('HTTP_X_COPY_FROM')
856     move_from = request.META.get('HTTP_X_MOVE_FROM')
857     if copy_from or move_from:
858         delimiter = request.GET.get('delimiter')
859         content_length = get_content_length(request)  # Required by the API.
860
861         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
862         if not src_account:
863             src_account = request.user_uniq
864         if move_from:
865             try:
866                 src_container, src_name = split_container_object_string(
867                     move_from)
868             except ValueError:
869                 raise BadRequest('Invalid X-Move-From header')
870             version_id = copy_or_move_object(
871                 request, src_account, src_container, src_name,
872                 v_account, v_container, v_object, move=True, delimiter=delimiter)
873         else:
874             try:
875                 src_container, src_name = split_container_object_string(
876                     copy_from)
877             except ValueError:
878                 raise BadRequest('Invalid X-Copy-From header')
879             version_id = copy_or_move_object(
880                 request, src_account, src_container, src_name,
881                 v_account, v_container, v_object, move=False, delimiter=delimiter)
882         response = HttpResponse(status=201)
883         response['X-Object-Version'] = version_id
884         return response
885
886     content_type, meta, permissions, public = get_object_headers(request)
887     content_length = -1
888     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
889         content_length = get_content_length(request)
890     # Should be BadRequest, but API says otherwise.
891     if content_type is None:
892         raise LengthRequired('Missing Content-Type header')
893
894     if 'hashmap' in request.GET:
895         if request.serialization not in ('json', 'xml'):
896             raise BadRequest('Invalid hashmap format')
897
898         data = ''
899         for block in socket_read_iterator(request, content_length,
900                                           request.backend.block_size):
901             data = '%s%s' % (data, block)
902
903         if request.serialization == 'json':
904             d = json.loads(data)
905             if not hasattr(d, '__getitem__'):
906                 raise BadRequest('Invalid data formating')
907             try:
908                 hashmap = d['hashes']
909                 size = int(d['bytes'])
910             except:
911                 raise BadRequest('Invalid data formatting')
912         elif request.serialization == 'xml':
913             try:
914                 xml = minidom.parseString(data)
915                 obj = xml.getElementsByTagName('object')[0]
916                 size = int(obj.attributes['bytes'].value)
917
918                 hashes = xml.getElementsByTagName('hash')
919                 hashmap = []
920                 for hash in hashes:
921                     hashmap.append(hash.firstChild.data)
922             except:
923                 raise BadRequest('Invalid data formatting')
924
925         checksum = ''  # Do not set to None (will copy previous value).
926     else:
927         md5 = hashlib.md5()
928         size = 0
929         hashmap = []
930         for data in socket_read_iterator(request, content_length,
931                                          request.backend.block_size):
932             # TODO: Raise 408 (Request Timeout) if this takes too long.
933             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
934             size += len(data)
935             hashmap.append(request.backend.put_block(data))
936             md5.update(data)
937
938         checksum = md5.hexdigest().lower()
939         etag = request.META.get('HTTP_ETAG')
940         if etag and parse_etags(etag)[0].lower() != checksum:
941             raise UnprocessableEntity('Object ETag does not match')
942
943     try:
944         version_id = request.backend.update_object_hashmap(request.user_uniq,
945                                                            v_account, v_container, v_object, size, content_type,
946                                                            hashmap, checksum, 'pithos', meta, True, permissions)
947     except NotAllowedError:
948         raise Forbidden('Not allowed')
949     except IndexError, e:
950         raise Conflict(simple_list_response(request, e.data))
951     except ItemNotExists:
952         raise ItemNotFound('Container does not exist')
953     except ValueError:
954         raise BadRequest('Invalid sharing header')
955     except QuotaError:
956         raise RequestEntityTooLarge('Quota exceeded')
957     if not checksum and UPDATE_MD5:
958         # Update the MD5 after the hashmap, as there may be missing hashes.
959         checksum = hashmap_md5(request.backend, hashmap, size)
960         try:
961             request.backend.update_object_checksum(request.user_uniq,
962                                                    v_account, v_container, v_object, version_id, checksum)
963         except NotAllowedError:
964             raise Forbidden('Not allowed')
965     if public is not None:
966         try:
967             request.backend.update_object_public(request.user_uniq, v_account,
968                                                  v_container, v_object, public)
969         except NotAllowedError:
970             raise Forbidden('Not allowed')
971         except ItemNotExists:
972             raise ItemNotFound('Object does not exist')
973
974     response = HttpResponse(status=201)
975     if checksum:
976         response['ETag'] = checksum
977     response['X-Object-Version'] = version_id
978     return response
979
980
981 @api_method('POST')
982 def object_write_form(request, v_account, v_container, v_object):
983     # Normal Response Codes: 201
984     # Error Response Codes: internalServerError (500),
985     #                       itemNotFound (404),
986     #                       forbidden (403),
987     #                       badRequest (400)
988
989     request.upload_handlers = [SaveToBackendHandler(request)]
990     if 'X-Object-Data' not in request.FILES:
991         raise BadRequest('Missing X-Object-Data field')
992     file = request.FILES['X-Object-Data']
993
994     checksum = file.etag
995     try:
996         version_id = request.backend.update_object_hashmap(request.user_uniq,
997                                                            v_account, v_container, v_object, file.size, file.content_type,
998                                                            file.hashmap, checksum, 'pithos', {}, True)
999     except NotAllowedError:
1000         raise Forbidden('Not allowed')
1001     except ItemNotExists:
1002         raise ItemNotFound('Container does not exist')
1003     except QuotaError:
1004         raise RequestEntityTooLarge('Quota exceeded')
1005
1006     response = HttpResponse(status=201)
1007     response['ETag'] = checksum
1008     response['X-Object-Version'] = version_id
1009     response.content = checksum
1010     return response
1011
1012
1013 @api_method('COPY', format_allowed=True)
1014 def object_copy(request, v_account, v_container, v_object):
1015     # Normal Response Codes: 201
1016     # Error Response Codes: internalServerError (500),
1017     #                       itemNotFound (404),
1018     #                       forbidden (403),
1019     #                       badRequest (400)
1020
1021     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
1022     if not dest_account:
1023         dest_account = request.user_uniq
1024     dest_path = request.META.get('HTTP_DESTINATION')
1025     if not dest_path:
1026         raise BadRequest('Missing Destination header')
1027     try:
1028         dest_container, dest_name = split_container_object_string(dest_path)
1029     except ValueError:
1030         raise BadRequest('Invalid Destination header')
1031
1032     # Evaluate conditions.
1033     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1034         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
1035         try:
1036             meta = request.backend.get_object_meta(
1037                 request.user_uniq, v_account,
1038                 v_container, v_object, 'pithos', src_version)
1039         except NotAllowedError:
1040             raise Forbidden('Not allowed')
1041         except (ItemNotExists, VersionNotExists):
1042             raise ItemNotFound('Container or object does not exist')
1043         validate_matching_preconditions(request, meta)
1044
1045     delimiter = request.GET.get('delimiter')
1046
1047     version_id = copy_or_move_object(request, v_account, v_container, v_object,
1048                                      dest_account, dest_container, dest_name, move=False, delimiter=delimiter)
1049     response = HttpResponse(status=201)
1050     response['X-Object-Version'] = version_id
1051     return response
1052
1053
1054 @api_method('MOVE', format_allowed=True)
1055 def object_move(request, v_account, v_container, v_object):
1056     # Normal Response Codes: 201
1057     # Error Response Codes: internalServerError (500),
1058     #                       itemNotFound (404),
1059     #                       forbidden (403),
1060     #                       badRequest (400)
1061
1062     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
1063     if not dest_account:
1064         dest_account = request.user_uniq
1065     dest_path = request.META.get('HTTP_DESTINATION')
1066     if not dest_path:
1067         raise BadRequest('Missing Destination header')
1068     try:
1069         dest_container, dest_name = split_container_object_string(dest_path)
1070     except ValueError:
1071         raise BadRequest('Invalid Destination header')
1072
1073     # Evaluate conditions.
1074     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1075         try:
1076             meta = request.backend.get_object_meta(
1077                 request.user_uniq, v_account,
1078                 v_container, v_object, 'pithos')
1079         except NotAllowedError:
1080             raise Forbidden('Not allowed')
1081         except ItemNotExists:
1082             raise ItemNotFound('Container or object does not exist')
1083         validate_matching_preconditions(request, meta)
1084
1085     delimiter = request.GET.get('delimiter')
1086
1087     version_id = copy_or_move_object(request, v_account, v_container, v_object,
1088                                      dest_account, dest_container, dest_name, move=True, delimiter=delimiter)
1089     response = HttpResponse(status=201)
1090     response['X-Object-Version'] = version_id
1091     return response
1092
1093
1094 @api_method('POST', format_allowed=True)
1095 def object_update(request, v_account, v_container, v_object):
1096     # Normal Response Codes: 202, 204
1097     # Error Response Codes: internalServerError (500),
1098     #                       conflict (409),
1099     #                       itemNotFound (404),
1100     #                       forbidden (403),
1101     #                       badRequest (400)
1102
1103     content_type, meta, permissions, public = get_object_headers(request)
1104
1105     try:
1106         prev_meta = request.backend.get_object_meta(
1107             request.user_uniq, v_account,
1108             v_container, v_object, 'pithos')
1109     except NotAllowedError:
1110         raise Forbidden('Not allowed')
1111     except ItemNotExists:
1112         raise ItemNotFound('Object does not exist')
1113
1114     # Evaluate conditions.
1115     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1116         validate_matching_preconditions(request, prev_meta)
1117
1118     replace = True
1119     if 'update' in request.GET:
1120         replace = False
1121
1122     # A Content-Type or X-Source-Object header indicates data updates.
1123     src_object = request.META.get('HTTP_X_SOURCE_OBJECT')
1124     if (not content_type or content_type != 'application/octet-stream') and not src_object:
1125         response = HttpResponse(status=202)
1126
1127         # Do permissions first, as it may fail easier.
1128         if permissions is not None:
1129             try:
1130                 request.backend.update_object_permissions(request.user_uniq,
1131                                                           v_account, v_container, v_object, permissions)
1132             except NotAllowedError:
1133                 raise Forbidden('Not allowed')
1134             except ItemNotExists:
1135                 raise ItemNotFound('Object does not exist')
1136             except ValueError:
1137                 raise BadRequest('Invalid sharing header')
1138         if public is not None:
1139             try:
1140                 request.backend.update_object_public(
1141                     request.user_uniq, v_account,
1142                     v_container, v_object, public)
1143             except NotAllowedError:
1144                 raise Forbidden('Not allowed')
1145             except ItemNotExists:
1146                 raise ItemNotFound('Object does not exist')
1147         if meta or replace:
1148             try:
1149                 version_id = request.backend.update_object_meta(
1150                     request.user_uniq,
1151                     v_account, v_container, v_object, 'pithos', meta, replace)
1152             except NotAllowedError:
1153                 raise Forbidden('Not allowed')
1154             except ItemNotExists:
1155                 raise ItemNotFound('Object does not exist')
1156             response['X-Object-Version'] = version_id
1157
1158         return response
1159
1160     # Single range update. Range must be in Content-Range.
1161     # Based on: http://code.google.com/p/gears/wiki/ContentRangePostProposal
1162     # (with the addition that '*' is allowed for the range - will append).
1163     content_range = request.META.get('HTTP_CONTENT_RANGE')
1164     if not content_range:
1165         raise BadRequest('Missing Content-Range header')
1166     ranges = get_content_range(request)
1167     if not ranges:
1168         raise RangeNotSatisfiable('Invalid Content-Range header')
1169
1170     try:
1171         size, hashmap = request.backend.get_object_hashmap(request.user_uniq,
1172                                                            v_account, v_container, v_object)
1173     except NotAllowedError:
1174         raise Forbidden('Not allowed')
1175     except ItemNotExists:
1176         raise ItemNotFound('Object does not exist')
1177
1178     offset, length, total = ranges
1179     if offset is None:
1180         offset = size
1181     elif offset > size:
1182         raise RangeNotSatisfiable('Supplied offset is beyond object limits')
1183     if src_object:
1184         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
1185         if not src_account:
1186             src_account = request.user_uniq
1187         src_container, src_name = split_container_object_string(src_object)
1188         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
1189         try:
1190             src_size, src_hashmap = request.backend.get_object_hashmap(
1191                 request.user_uniq,
1192                 src_account, src_container, src_name, src_version)
1193         except NotAllowedError:
1194             raise Forbidden('Not allowed')
1195         except ItemNotExists:
1196             raise ItemNotFound('Source object does not exist')
1197
1198         if length is None:
1199             length = src_size
1200         elif length > src_size:
1201             raise BadRequest('Object length is smaller than range length')
1202     else:
1203         # Require either a Content-Length, or 'chunked' Transfer-Encoding.
1204         content_length = -1
1205         if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
1206             content_length = get_content_length(request)
1207
1208         if length is None:
1209             length = content_length
1210         else:
1211             if content_length == -1:
1212                 # TODO: Get up to length bytes in chunks.
1213                 length = content_length
1214             elif length != content_length:
1215                 raise BadRequest('Content length does not match range length')
1216     if total is not None and (total != size or offset >= size or (length > 0 and offset + length >= size)):
1217         raise RangeNotSatisfiable(
1218             'Supplied range will change provided object limits')
1219
1220     dest_bytes = request.META.get('HTTP_X_OBJECT_BYTES')
1221     if dest_bytes is not None:
1222         dest_bytes = get_int_parameter(dest_bytes)
1223         if dest_bytes is None:
1224             raise BadRequest('Invalid X-Object-Bytes header')
1225
1226     if src_object:
1227         if offset % request.backend.block_size == 0:
1228             # Update the hashes only.
1229             sbi = 0
1230             while length > 0:
1231                 bi = int(offset / request.backend.block_size)
1232                 bl = min(length, request.backend.block_size)
1233                 if bi < len(hashmap):
1234                     if bl == request.backend.block_size:
1235                         hashmap[bi] = src_hashmap[sbi]
1236                     else:
1237                         data = request.backend.get_block(src_hashmap[sbi])
1238                         hashmap[bi] = request.backend.update_block(hashmap[bi],
1239                                                                    data[:bl], 0)
1240                 else:
1241                     hashmap.append(src_hashmap[sbi])
1242                 offset += bl
1243                 length -= bl
1244                 sbi += 1
1245         else:
1246             data = ''
1247             sbi = 0
1248             while length > 0:
1249                 data += request.backend.get_block(src_hashmap[sbi])
1250                 if length < request.backend.block_size:
1251                     data = data[:length]
1252                 bytes = put_object_block(request, hashmap, data, offset)
1253                 offset += bytes
1254                 data = data[bytes:]
1255                 length -= bytes
1256                 sbi += 1
1257     else:
1258         data = ''
1259         for d in socket_read_iterator(request, length,
1260                                       request.backend.block_size):
1261             # TODO: Raise 408 (Request Timeout) if this takes too long.
1262             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
1263             data += d
1264             bytes = put_object_block(request, hashmap, data, offset)
1265             offset += bytes
1266             data = data[bytes:]
1267         if len(data) > 0:
1268             put_object_block(request, hashmap, data, offset)
1269
1270     if offset > size:
1271         size = offset
1272     if dest_bytes is not None and dest_bytes < size:
1273         size = dest_bytes
1274         hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)]
1275     checksum = hashmap_md5(
1276         request.backend, hashmap, size) if UPDATE_MD5 else ''
1277     try:
1278         version_id = request.backend.update_object_hashmap(request.user_uniq,
1279                                                            v_account, v_container, v_object, size, prev_meta[
1280                                                            'type'],
1281                                                            hashmap, checksum, 'pithos', meta, replace, permissions)
1282     except NotAllowedError:
1283         raise Forbidden('Not allowed')
1284     except ItemNotExists:
1285         raise ItemNotFound('Container does not exist')
1286     except ValueError:
1287         raise BadRequest('Invalid sharing header')
1288     except QuotaError:
1289         raise RequestEntityTooLarge('Quota exceeded')
1290     if public is not None:
1291         try:
1292             request.backend.update_object_public(request.user_uniq, v_account,
1293                                                  v_container, v_object, public)
1294         except NotAllowedError:
1295             raise Forbidden('Not allowed')
1296         except ItemNotExists:
1297             raise ItemNotFound('Object does not exist')
1298
1299     response = HttpResponse(status=204)
1300     response['ETag'] = checksum
1301     response['X-Object-Version'] = version_id
1302     return response
1303
1304
1305 @api_method('DELETE')
1306 def object_delete(request, v_account, v_container, v_object):
1307     # Normal Response Codes: 204
1308     # Error Response Codes: internalServerError (500),
1309     #                       itemNotFound (404),
1310     #                       forbidden (403),
1311     #                       badRequest (400)
1312
1313     until = get_int_parameter(request.GET.get('until'))
1314     delimiter = request.GET.get('delimiter')
1315
1316     try:
1317         request.backend.delete_object(
1318             request.user_uniq, v_account, v_container,
1319             v_object, until, delimiter=delimiter)
1320     except NotAllowedError:
1321         raise Forbidden('Not allowed')
1322     except ItemNotExists:
1323         raise ItemNotFound('Object does not exist')
1324     return HttpResponse(status=204)
1325
1326
1327 @api_method()
1328 def method_not_allowed(request):
1329     raise BadRequest('Method not allowed')