Merge branch 'master' of https://code.grnet.gr/git/pithos
[pithos] / snf-pithos-app / pithos / api / functions.py
1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
2
3 # Redistribution and use in source and binary forms, with or
4 # without modification, are permitted provided that the following
5 # conditions are met:
6
7 #   1. Redistributions of source code must retain the above
8 #      copyright notice, this list of conditions and the following
9 #      disclaimer.
10
11 #   2. Redistributions in binary form must reproduce the above
12 #      copyright notice, this list of conditions and the following
13 #      disclaimer in the documentation and/or other materials
14 #      provided with the distribution.
15
16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 # POSSIBILITY OF SUCH DAMAGE.
28
29 # The views and conclusions contained in the software and
30 # documentation are those of the authors and should not be
31 # interpreted as representing official policies, either expressed
32 # or implied, of GRNET S.A.
33
34 from xml.dom import minidom
35 from urllib import unquote
36
37 from django.conf import settings
38 from django.http import HttpResponse
39 from django.template.loader import render_to_string
40 from django.utils import simplejson as json
41 from django.utils.http import parse_etags
42 from django.utils.encoding import smart_str
43 from django.views.decorators.csrf import csrf_exempt
44
45 from synnefo.lib.astakos import get_user
46
47 from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
48     LengthRequired, PreconditionFailed, RequestEntityTooLarge, RangeNotSatisfiable, UnprocessableEntity)
49 from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
50     get_account_headers, put_account_headers, get_container_headers, put_container_headers, get_object_headers,
51     put_object_headers, update_manifest_meta, update_sharing_meta, update_public_meta,
52     validate_modification_preconditions, validate_matching_preconditions, split_container_object_string,
53     copy_or_move_object, get_int_parameter, get_content_length, get_content_range, socket_read_iterator,
54     SaveToBackendHandler, object_data_response, put_object_block, hashmap_md5, simple_list_response, api_method)
55 from pithos.api.settings import UPDATE_MD5
56
57 from pithos.backends.base import NotAllowedError, QuotaError, ContainerNotEmpty, ItemNotExists, VersionNotExists
58
59 from pithos.backends.filter import parse_filters
60
61 import logging
62 import hashlib
63
64
65 logger = logging.getLogger(__name__)
66
67
68 @csrf_exempt
69 def top_demux(request):
70     if request.method == 'GET':
71         try:
72                 request.GET['X-Auth-Token']
73         except KeyError:
74                 try:
75                         request.META['HTTP_X_AUTH_TOKEN']
76                 except KeyError:
77                         return authenticate(request)
78         return account_list(request)
79     else:
80         return method_not_allowed(request)
81
82 @csrf_exempt
83 def account_demux(request, v_account):
84     if request.method == 'HEAD':
85         return account_meta(request, v_account)
86     elif request.method == 'POST':
87         return account_update(request, v_account)
88     elif request.method == 'GET':
89         return container_list(request, v_account)
90     else:
91         return method_not_allowed(request)
92
93 @csrf_exempt
94 def container_demux(request, v_account, v_container):
95     if request.method == 'HEAD':
96         return container_meta(request, v_account, v_container)
97     elif request.method == 'PUT':
98         return container_create(request, v_account, v_container)
99     elif request.method == 'POST':
100         return container_update(request, v_account, v_container)
101     elif request.method == 'DELETE':
102         return container_delete(request, v_account, v_container)
103     elif request.method == 'GET':
104         return object_list(request, v_account, v_container)
105     else:
106         return method_not_allowed(request)
107
108 @csrf_exempt
109 def object_demux(request, v_account, v_container, v_object):
110     # Helper to avoid placing the token in the URL when loading objects from a browser.
111     if request.method == 'HEAD':
112         return object_meta(request, v_account, v_container, v_object)
113     elif request.method == 'GET':
114         return object_read(request, v_account, v_container, v_object)
115     elif request.method == 'PUT':
116         return object_write(request, v_account, v_container, v_object)
117     elif request.method == 'COPY':
118         return object_copy(request, v_account, v_container, v_object)
119     elif request.method == 'MOVE':
120         return object_move(request, v_account, v_container, v_object)
121     elif request.method == 'POST':
122         if request.META.get('CONTENT_TYPE', '').startswith('multipart/form-data'):
123             return object_write_form(request, v_account, v_container, v_object)
124         return object_update(request, v_account, v_container, v_object)
125     elif request.method == 'DELETE':
126         return object_delete(request, v_account, v_container, v_object)
127     else:
128         return method_not_allowed(request)
129
130 @api_method('GET', user_required=False)
131 def authenticate(request):
132     # Normal Response Codes: 204
133     # Error Response Codes: internalServerError (500),
134     #                       forbidden (403),
135     #                       badRequest (400)
136     
137     x_auth_user = request.META.get('HTTP_X_AUTH_USER')
138     x_auth_key = request.META.get('HTTP_X_AUTH_KEY')
139     if not x_auth_user or not x_auth_key:
140         raise BadRequest('Missing X-Auth-User or X-Auth-Key header')
141     response = HttpResponse(status=204)
142     
143     uri = request.build_absolute_uri()
144     if '?' in uri:
145         uri = uri[:uri.find('?')]
146     
147     response['X-Auth-Token'] = x_auth_key
148     response['X-Storage-Url'] = uri + ('' if uri.endswith('/') else '/') + x_auth_user
149     return response
150
151 @api_method('GET', format_allowed=True)
152 def account_list(request):
153     # Normal Response Codes: 200, 204
154     # Error Response Codes: internalServerError (500),
155     #                       badRequest (400)
156     response = HttpResponse()
157     
158     marker = request.GET.get('marker')
159     limit = get_int_parameter(request.GET.get('limit'))
160     if not limit:
161         limit = 10000
162     
163     accounts = request.backend.list_accounts(request.user_uniq, marker, limit)
164     
165     if request.serialization == 'text':
166         if len(accounts) == 0:
167             # The cloudfiles python bindings expect 200 if json/xml.
168             response.status_code = 204
169             return response
170         response.status_code = 200
171         response.content = '\n'.join(accounts) + '\n'
172         return response
173     
174     account_meta = []
175     for x in accounts:
176         if x == request.user_uniq:
177             continue
178         try:
179             meta = request.backend.get_account_meta(request.user_uniq, x, 'pithos', include_user_defined=False)
180             groups = request.backend.get_account_groups(request.user_uniq, x)
181         except NotAllowedError:
182             raise Forbidden('Not allowed')
183         else:
184             rename_meta_key(meta, 'modified', 'last_modified')
185             rename_meta_key(meta, 'until_timestamp', 'x_account_until_timestamp')
186             if groups:
187                 meta['X-Account-Group'] = printable_header_dict(dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
188             account_meta.append(printable_header_dict(meta))
189     if request.serialization == 'xml':
190         data = render_to_string('accounts.xml', {'accounts': account_meta})
191     elif request.serialization  == 'json':
192         data = json.dumps(account_meta)
193     response.status_code = 200
194     response.content = data
195     return response
196
197 @api_method('HEAD')
198 def account_meta(request, v_account):
199     # Normal Response Codes: 204
200     # Error Response Codes: internalServerError (500),
201     #                       forbidden (403),
202     #                       badRequest (400)
203     
204     until = get_int_parameter(request.GET.get('until'))
205     try:
206         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
207         groups = request.backend.get_account_groups(request.user_uniq, v_account)
208         policy = request.backend.get_account_policy(request.user_uniq, v_account)
209     except NotAllowedError:
210         raise Forbidden('Not allowed')
211     
212     validate_modification_preconditions(request, meta)
213     
214     response = HttpResponse(status=204)
215     put_account_headers(response, meta, groups, policy)
216     return response
217
218 @api_method('POST')
219 def account_update(request, v_account):
220     # Normal Response Codes: 202
221     # Error Response Codes: internalServerError (500),
222     #                       forbidden (403),
223     #                       badRequest (400)
224     
225     meta, groups = get_account_headers(request)
226     replace = True
227     if 'update' in request.GET:
228         replace = False
229     if groups:
230         try:
231             request.backend.update_account_groups(request.user_uniq, v_account,
232                                                     groups, replace)
233         except NotAllowedError:
234             raise Forbidden('Not allowed')
235         except ValueError:
236             raise BadRequest('Invalid groups header')
237     if meta or replace:
238         try:
239             request.backend.update_account_meta(request.user_uniq, v_account,
240                                                 'pithos', meta, replace)
241         except NotAllowedError:
242             raise Forbidden('Not allowed')
243     return HttpResponse(status=202)
244
245 @api_method('GET', format_allowed=True)
246 def container_list(request, v_account):
247     # Normal Response Codes: 200, 204
248     # Error Response Codes: internalServerError (500),
249     #                       itemNotFound (404),
250     #                       forbidden (403),
251     #                       badRequest (400)
252     
253     until = get_int_parameter(request.GET.get('until'))
254     try:
255         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
256         groups = request.backend.get_account_groups(request.user_uniq, v_account)
257         policy = request.backend.get_account_policy(request.user_uniq, v_account)
258     except NotAllowedError:
259         raise Forbidden('Not allowed')
260     
261     validate_modification_preconditions(request, meta)
262     
263     response = HttpResponse()
264     put_account_headers(response, meta, groups, policy)
265     
266     marker = request.GET.get('marker')
267     limit = get_int_parameter(request.GET.get('limit'))
268     if not limit:
269         limit = 10000
270     
271     shared = False
272     if 'shared' in request.GET:
273         shared = True
274     public = False
275     if 'public' in request.GET:
276         public = True
277     
278     try:
279         containers = request.backend.list_containers(request.user_uniq, v_account,
280                                                 marker, limit, shared, until, public)
281     except NotAllowedError:
282         raise Forbidden('Not allowed')
283     except NameError:
284         containers = []
285     
286     if request.serialization == 'text':
287         if len(containers) == 0:
288             # The cloudfiles python bindings expect 200 if json/xml.
289             response.status_code = 204
290             return response
291         response.status_code = 200
292         response.content = '\n'.join(containers) + '\n'
293         return response
294     
295     container_meta = []
296     for x in containers:
297         try:
298             meta = request.backend.get_container_meta(request.user_uniq, v_account,
299                                                         x, 'pithos', until, include_user_defined=False)
300             policy = request.backend.get_container_policy(request.user_uniq,
301                                                             v_account, x)
302         except NotAllowedError:
303             raise Forbidden('Not allowed')
304         except NameError:
305             pass
306         else:
307             rename_meta_key(meta, 'modified', 'last_modified')
308             rename_meta_key(meta, 'until_timestamp', 'x_container_until_timestamp')
309             if policy:
310                 meta['X-Container-Policy'] = printable_header_dict(dict([(k, v) for k, v in policy.iteritems()]))
311             container_meta.append(printable_header_dict(meta))
312     if request.serialization == 'xml':
313         data = render_to_string('containers.xml', {'account': v_account, 'containers': container_meta})
314     elif request.serialization  == 'json':
315         data = json.dumps(container_meta)
316     response.status_code = 200
317     response.content = data
318     return response
319
320 @api_method('HEAD')
321 def container_meta(request, v_account, v_container):
322     # Normal Response Codes: 204
323     # Error Response Codes: internalServerError (500),
324     #                       itemNotFound (404),
325     #                       forbidden (403),
326     #                       badRequest (400)
327     
328     until = get_int_parameter(request.GET.get('until'))
329     try:
330         meta = request.backend.get_container_meta(request.user_uniq, v_account,
331                                                     v_container, 'pithos', until)
332         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
333                                                 v_account, v_container, 'pithos', until)
334         policy = request.backend.get_container_policy(request.user_uniq, v_account,
335                                                         v_container)
336     except NotAllowedError:
337         raise Forbidden('Not allowed')
338     except ItemNotExists:
339         raise ItemNotFound('Container does not exist')
340     
341     validate_modification_preconditions(request, meta)
342     
343     response = HttpResponse(status=204)
344     put_container_headers(request, response, meta, policy)
345     return response
346
347 @api_method('PUT')
348 def container_create(request, v_account, v_container):
349     # Normal Response Codes: 201, 202
350     # Error Response Codes: internalServerError (500),
351     #                       itemNotFound (404),
352     #                       forbidden (403),
353     #                       badRequest (400)
354     
355     meta, policy = get_container_headers(request)
356     
357     try:
358         request.backend.put_container(request.user_uniq, v_account, v_container, policy)
359         ret = 201
360     except NotAllowedError:
361         raise Forbidden('Not allowed')
362     except ValueError:
363         raise BadRequest('Invalid policy header')
364     except NameError:
365         ret = 202
366     
367     if ret == 202 and policy:
368         try:
369             request.backend.update_container_policy(request.user_uniq, v_account,
370                                             v_container, policy, replace=False)
371         except NotAllowedError:
372             raise Forbidden('Not allowed')
373         except ItemNotExists:
374             raise ItemNotFound('Container does not exist')
375         except ValueError:
376             raise BadRequest('Invalid policy header')
377     if meta:
378         try:
379             request.backend.update_container_meta(request.user_uniq, v_account,
380                                             v_container, 'pithos', meta, replace=False)
381         except NotAllowedError:
382             raise Forbidden('Not allowed')
383         except ItemNotExists:
384             raise ItemNotFound('Container does not exist')
385     
386     return HttpResponse(status=ret)
387
388 @api_method('POST', format_allowed=True)
389 def container_update(request, v_account, v_container):
390     # Normal Response Codes: 202
391     # Error Response Codes: internalServerError (500),
392     #                       itemNotFound (404),
393     #                       forbidden (403),
394     #                       badRequest (400)
395     
396     meta, policy = get_container_headers(request)
397     replace = True
398     if 'update' in request.GET:
399         replace = False
400     if policy:
401         try:
402             request.backend.update_container_policy(request.user_uniq, v_account,
403                                                 v_container, policy, replace)
404         except NotAllowedError:
405             raise Forbidden('Not allowed')
406         except ItemNotExists:
407             raise ItemNotFound('Container does not exist')
408         except ValueError:
409             raise BadRequest('Invalid policy header')
410     if meta or replace:
411         try:
412             request.backend.update_container_meta(request.user_uniq, v_account,
413                                                     v_container, 'pithos', meta, replace)
414         except NotAllowedError:
415             raise Forbidden('Not allowed')
416         except ItemNotExists:
417             raise ItemNotFound('Container does not exist')
418     
419     content_length = -1
420     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
421         content_length = get_int_parameter(request.META.get('CONTENT_LENGTH', 0))
422     content_type = request.META.get('CONTENT_TYPE')
423     hashmap = []
424     if content_type and content_type == 'application/octet-stream' and content_length != 0:
425         for data in socket_read_iterator(request, content_length,
426                                             request.backend.block_size):
427             # TODO: Raise 408 (Request Timeout) if this takes too long.
428             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
429             hashmap.append(request.backend.put_block(data))
430     
431     response = HttpResponse(status=202)
432     if hashmap:
433         response.content = simple_list_response(request, hashmap)
434     return response
435
436 @api_method('DELETE')
437 def container_delete(request, v_account, v_container):
438     # Normal Response Codes: 204
439     # Error Response Codes: internalServerError (500),
440     #                       conflict (409),
441     #                       itemNotFound (404),
442     #                       forbidden (403),
443     #                       badRequest (400)
444     
445     until = get_int_parameter(request.GET.get('until'))
446     
447     delimiter = request.GET.get('delimiter')
448     
449     try:
450         request.backend.delete_container(request.user_uniq, v_account, v_container,
451                                             until, delimiter=delimiter)
452     except NotAllowedError:
453         raise Forbidden('Not allowed')
454     except ItemNotExists:
455         raise ItemNotFound('Container does not exist')
456     except ContainerNotEmpty:
457         raise Conflict('Container is not empty')
458     return HttpResponse(status=204)
459
460 @api_method('GET', format_allowed=True)
461 def object_list(request, v_account, v_container):
462     # Normal Response Codes: 200, 204
463     # Error Response Codes: internalServerError (500),
464     #                       itemNotFound (404),
465     #                       forbidden (403),
466     #                       badRequest (400)
467     
468     until = get_int_parameter(request.GET.get('until'))
469     try:
470         meta = request.backend.get_container_meta(request.user_uniq, v_account,
471                                                     v_container, 'pithos', until)
472         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
473                                                 v_account, v_container, 'pithos', until)
474         policy = request.backend.get_container_policy(request.user_uniq, v_account,
475                                                         v_container)
476     except NotAllowedError:
477         raise Forbidden('Not allowed')
478     except ItemNotExists:
479         raise ItemNotFound('Container does not exist')
480     
481     validate_modification_preconditions(request, meta)
482     
483     response = HttpResponse()
484     put_container_headers(request, response, meta, policy)
485     
486     path = request.GET.get('path')
487     prefix = request.GET.get('prefix')
488     delimiter = request.GET.get('delimiter')
489     
490     # Path overrides prefix and delimiter.
491     virtual = True
492     if path:
493         prefix = path
494         delimiter = '/'
495         virtual = False
496     
497     # Naming policy.
498     if prefix and delimiter and not prefix.endswith(delimiter):
499         prefix = prefix + delimiter
500     if not prefix:
501         prefix = ''
502     prefix = prefix.lstrip('/')
503     
504     marker = request.GET.get('marker')
505     limit = get_int_parameter(request.GET.get('limit'))
506     if not limit:
507         limit = 10000
508     
509     keys = request.GET.get('meta')
510     if keys:
511         keys = [smart_str(x.strip()) for x in keys.split(',') if x.strip() != '']
512         included, excluded, opers = parse_filters(keys)
513         keys = []
514         keys += [format_header_key('X-Object-Meta-' + x) for x in included]
515         keys += ['!'+format_header_key('X-Object-Meta-' + x) for x in excluded]
516         keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
517     else:
518         keys = []
519     
520     shared = False
521     if 'shared' in request.GET:
522         shared = True
523     public = False
524     if 'public' in request.GET:
525         public = True
526     
527     if request.serialization == 'text':
528         try:
529             objects = request.backend.list_objects(request.user_uniq, v_account,
530                                         v_container, prefix, delimiter, marker,
531                                         limit, virtual, 'pithos', keys, shared,
532                                         until, None, public)
533         except NotAllowedError:
534             raise Forbidden('Not allowed')
535         except ItemNotExists:
536             raise ItemNotFound('Container does not exist')
537         
538         if len(objects) == 0:
539             # The cloudfiles python bindings expect 200 if json/xml.
540             response.status_code = 204
541             return response
542         response.status_code = 200
543         response.content = '\n'.join([x[0] for x in objects]) + '\n'
544         return response
545
546     try:
547         objects = request.backend.list_object_meta(request.user_uniq, v_account,
548                                     v_container, prefix, delimiter, marker,
549                                     limit, virtual, 'pithos', keys, shared, until, None, public)
550         object_permissions = {}
551         object_public = {}
552         if until is None:
553             name_idx = len('/'.join((v_account, v_container, '')))
554             for x in request.backend.list_object_permissions(request.user_uniq,
555                                     v_account, v_container, prefix):
556                 object = x[name_idx:]
557                 object_permissions[object] = request.backend.get_object_permissions(
558                                     request.user_uniq, v_account, v_container, object)
559             for k, v in request.backend.list_object_public(request.user_uniq,
560                                     v_account, v_container, prefix).iteritems():
561                 object_public[k[name_idx:]] = v
562     except NotAllowedError:
563         raise Forbidden('Not allowed')
564     except ItemNotExists:
565         raise ItemNotFound('Container does not exist')
566     
567     object_meta = []
568     for meta in objects:
569         if len(meta) == 1:
570             # Virtual objects/directories.
571             object_meta.append(meta)
572         else:
573             rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
574             rename_meta_key(meta, 'checksum', 'hash')
575             rename_meta_key(meta, 'type', 'content_type')
576             rename_meta_key(meta, 'uuid', 'x_object_uuid')
577             if until is not None and 'modified' in meta:
578                 del(meta['modified'])
579             else:
580                 rename_meta_key(meta, 'modified', 'last_modified')
581             rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
582             rename_meta_key(meta, 'version', 'x_object_version')
583             rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
584             permissions = object_permissions.get(meta['name'], None)
585             if permissions:
586                 update_sharing_meta(request, permissions, v_account, v_container, meta['name'], meta)
587             public = object_public.get(meta['name'], None)
588             if public:
589                 update_public_meta(public, meta)
590             object_meta.append(printable_header_dict(meta))
591     if request.serialization == 'xml':
592         data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
593     elif request.serialization  == 'json':
594         data = json.dumps(object_meta, default=json_encode_decimal)
595     response.status_code = 200
596     response.content = data
597     return response
598
599 @api_method('HEAD')
600 def object_meta(request, v_account, v_container, v_object):
601     # Normal Response Codes: 204
602     # Error Response Codes: internalServerError (500),
603     #                       itemNotFound (404),
604     #                       forbidden (403),
605     #                       badRequest (400)
606     
607     version = request.GET.get('version')
608     try:
609         meta = request.backend.get_object_meta(request.user_uniq, v_account,
610                                                 v_container, v_object, 'pithos', version)
611         if version is None:
612             permissions = request.backend.get_object_permissions(request.user_uniq,
613                                             v_account, v_container, v_object)
614             public = request.backend.get_object_public(request.user_uniq, v_account,
615                                                         v_container, v_object)
616         else:
617             permissions = None
618             public = None
619     except NotAllowedError:
620         raise Forbidden('Not allowed')
621     except ItemNotExists:
622         raise ItemNotFound('Object does not exist')
623     except VersionNotExists:
624         raise ItemNotFound('Version does not exist')
625     
626     update_manifest_meta(request, v_account, meta)
627     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
628     update_public_meta(public, meta)
629     
630     # Evaluate conditions.
631     validate_modification_preconditions(request, meta)
632     try:
633         validate_matching_preconditions(request, meta)
634     except NotModified:
635         response = HttpResponse(status=304)
636         response['ETag'] = meta['checksum']
637         return response
638     
639     response = HttpResponse(status=200)
640     put_object_headers(response, meta)
641     return response
642
643 @api_method('GET', format_allowed=True)
644 def object_read(request, v_account, v_container, v_object):
645     # Normal Response Codes: 200, 206
646     # Error Response Codes: internalServerError (500),
647     #                       rangeNotSatisfiable (416),
648     #                       preconditionFailed (412),
649     #                       itemNotFound (404),
650     #                       forbidden (403),
651     #                       badRequest (400),
652     #                       notModified (304)
653     
654     version = request.GET.get('version')
655     
656     # Reply with the version list. Do this first, as the object may be deleted.
657     if version == 'list':
658         if request.serialization == 'text':
659             raise BadRequest('No format specified for version list.')
660         
661         try:
662             v = request.backend.list_versions(request.user_uniq, v_account,
663                                                 v_container, v_object)
664         except NotAllowedError:
665             raise Forbidden('Not allowed')
666         d = {'versions': v}
667         if request.serialization == 'xml':
668             d['object'] = v_object
669             data = render_to_string('versions.xml', d)
670         elif request.serialization  == 'json':
671             data = json.dumps(d, default=json_encode_decimal)
672         
673         response = HttpResponse(data, status=200)
674         response['Content-Length'] = len(data)
675         return response
676     
677     try:
678         meta = request.backend.get_object_meta(request.user_uniq, v_account,
679                                                 v_container, v_object, 'pithos', version)
680         if version is None:
681             permissions = request.backend.get_object_permissions(request.user_uniq,
682                                             v_account, v_container, v_object)
683             public = request.backend.get_object_public(request.user_uniq, v_account,
684                                                         v_container, v_object)
685         else:
686             permissions = None
687             public = None
688     except NotAllowedError:
689         raise Forbidden('Not allowed')
690     except ItemNotExists:
691         raise ItemNotFound('Object does not exist')
692     except VersionNotExists:
693         raise ItemNotFound('Version does not exist')
694     
695     update_manifest_meta(request, v_account, meta)
696     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
697     update_public_meta(public, meta)
698     
699     # Evaluate conditions.
700     validate_modification_preconditions(request, meta)
701     try:
702         validate_matching_preconditions(request, meta)
703     except NotModified:
704         response = HttpResponse(status=304)
705         response['ETag'] = meta['checksum']
706         return response
707     
708     hashmap_reply = False
709     if 'hashmap' in request.GET and request.serialization != 'text':
710         hashmap_reply = True
711     
712     sizes = []
713     hashmaps = []
714     if 'X-Object-Manifest' in meta and not hashmap_reply:
715         try:
716             src_container, src_name = split_container_object_string('/' + meta['X-Object-Manifest'])
717             objects = request.backend.list_objects(request.user_uniq, v_account,
718                                 src_container, prefix=src_name, virtual=False)
719         except NotAllowedError:
720             raise Forbidden('Not allowed')
721         except ValueError:
722             raise BadRequest('Invalid X-Object-Manifest header')
723         except ItemNotExists:
724             raise ItemNotFound('Container does not exist')
725         
726         try:
727             for x in objects:
728                 s, h = request.backend.get_object_hashmap(request.user_uniq,
729                                         v_account, src_container, x[0], x[1])
730                 sizes.append(s)
731                 hashmaps.append(h)
732         except NotAllowedError:
733             raise Forbidden('Not allowed')
734         except ItemNotExists:
735             raise ItemNotFound('Object does not exist')
736         except VersionNotExists:
737             raise ItemNotFound('Version does not exist')
738     else:
739         try:
740             s, h = request.backend.get_object_hashmap(request.user_uniq, v_account,
741                                                 v_container, v_object, version)
742             sizes.append(s)
743             hashmaps.append(h)
744         except NotAllowedError:
745             raise Forbidden('Not allowed')
746         except ItemNotExists:
747             raise ItemNotFound('Object does not exist')
748         except VersionNotExists:
749             raise ItemNotFound('Version does not exist')
750     
751     # Reply with the hashmap.
752     if hashmap_reply:
753         size = sum(sizes)
754         hashmap = sum(hashmaps, [])
755         d = {
756             'block_size': request.backend.block_size,
757             'block_hash': request.backend.hash_algorithm,
758             'bytes': size,
759             'hashes': hashmap}
760         if request.serialization == 'xml':
761             d['object'] = v_object
762             data = render_to_string('hashes.xml', d)
763         elif request.serialization  == 'json':
764             data = json.dumps(d)
765         
766         response = HttpResponse(data, status=200)
767         put_object_headers(response, meta)
768         response['Content-Length'] = len(data)
769         return response
770     
771     request.serialization = 'text' # Unset.
772     return object_data_response(request, sizes, hashmaps, meta)
773
774 @api_method('PUT', format_allowed=True)
775 def object_write(request, v_account, v_container, v_object):
776     # Normal Response Codes: 201
777     # Error Response Codes: internalServerError (500),
778     #                       unprocessableEntity (422),
779     #                       lengthRequired (411),
780     #                       conflict (409),
781     #                       itemNotFound (404),
782     #                       forbidden (403),
783     #                       badRequest (400)
784     
785     # Evaluate conditions.
786     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
787         try:
788             meta = request.backend.get_object_meta(request.user_uniq, v_account,
789                                                         v_container, v_object, 'pithos')
790         except NotAllowedError:
791             raise Forbidden('Not allowed')
792         except NameError:
793             meta = {}
794         validate_matching_preconditions(request, meta)
795     
796     copy_from = request.META.get('HTTP_X_COPY_FROM')
797     move_from = request.META.get('HTTP_X_MOVE_FROM')
798     if copy_from or move_from:
799         delimiter = request.GET.get('delimiter')
800         content_length = get_content_length(request) # Required by the API.
801         
802         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
803         if not src_account:
804             src_account = request.user_uniq
805         if move_from:
806             try:
807                 src_container, src_name = split_container_object_string(move_from)
808             except ValueError:
809                 raise BadRequest('Invalid X-Move-From header')
810             version_id = copy_or_move_object(request, src_account, src_container, src_name,
811                                                 v_account, v_container, v_object, move=True, delimiter=delimiter)
812         else:
813             try:
814                 src_container, src_name = split_container_object_string(copy_from)
815             except ValueError:
816                 raise BadRequest('Invalid X-Copy-From header')
817             version_id = copy_or_move_object(request, src_account, src_container, src_name,
818                                                 v_account, v_container, v_object, move=False, delimiter=delimiter)
819         response = HttpResponse(status=201)
820         response['X-Object-Version'] = version_id
821         return response
822     
823     content_type, meta, permissions, public = get_object_headers(request)
824     content_length = -1
825     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
826         content_length = get_content_length(request)
827     # Should be BadRequest, but API says otherwise.
828     if content_type is None:
829         raise LengthRequired('Missing Content-Type header')
830     
831     if 'hashmap' in request.GET:
832         if request.serialization not in ('json', 'xml'):
833             raise BadRequest('Invalid hashmap format')
834         
835         data = ''
836         for block in socket_read_iterator(request, content_length,
837                                             request.backend.block_size):
838             data = '%s%s' % (data, block)
839         
840         if request.serialization == 'json':
841             d = json.loads(data)
842             if not hasattr(d, '__getitem__'):
843                 raise BadRequest('Invalid data formating')
844             try:
845                 hashmap = d['hashes']
846                 size = int(d['bytes'])
847             except:
848                 raise BadRequest('Invalid data formatting')
849         elif request.serialization == 'xml':
850             try:
851                 xml = minidom.parseString(data)
852                 obj = xml.getElementsByTagName('object')[0]
853                 size = int(obj.attributes['bytes'].value)
854                 
855                 hashes = xml.getElementsByTagName('hash')
856                 hashmap = []
857                 for hash in hashes:
858                     hashmap.append(hash.firstChild.data)
859             except:
860                 raise BadRequest('Invalid data formatting')
861         
862         checksum = '' # Do not set to None (will copy previous value).
863     else:
864         md5 = hashlib.md5()
865         size = 0
866         hashmap = []
867         for data in socket_read_iterator(request, content_length,
868                                             request.backend.block_size):
869             # TODO: Raise 408 (Request Timeout) if this takes too long.
870             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
871             size += len(data)
872             hashmap.append(request.backend.put_block(data))
873             md5.update(data)
874         
875         checksum = md5.hexdigest().lower()
876         etag = request.META.get('HTTP_ETAG')
877         if etag and parse_etags(etag)[0].lower() != checksum:
878             raise UnprocessableEntity('Object ETag does not match')
879     
880     try:
881         version_id = request.backend.update_object_hashmap(request.user_uniq,
882                         v_account, v_container, v_object, size, content_type,
883                         hashmap, checksum, 'pithos', meta, True, permissions)
884     except NotAllowedError:
885         raise Forbidden('Not allowed')
886     except IndexError, e:
887         raise Conflict(simple_list_response(request, e.data))
888     except ItemNotExists:
889         raise ItemNotFound('Container does not exist')
890     except ValueError:
891         raise BadRequest('Invalid sharing header')
892     except QuotaError:
893         raise RequestEntityTooLarge('Quota exceeded')
894     if not checksum and UPDATE_MD5:
895         # Update the MD5 after the hashmap, as there may be missing hashes.
896         checksum = hashmap_md5(request.backend, hashmap, size)
897         try:
898             request.backend.update_object_checksum(request.user_uniq,
899               v_account, v_container, v_object, version_id, checksum)
900         except NotAllowedError:
901             raise Forbidden('Not allowed')
902     if public is not None:
903         try:
904             request.backend.update_object_public(request.user_uniq, v_account,
905                                                 v_container, v_object, public)
906         except NotAllowedError:
907             raise Forbidden('Not allowed')
908         except ItemNotExists:
909             raise ItemNotFound('Object does not exist')
910     
911     response = HttpResponse(status=201)
912     if checksum:
913         response['ETag'] = checksum
914     response['X-Object-Version'] = version_id
915     return response
916
917 @api_method('POST')
918 def object_write_form(request, v_account, v_container, v_object):
919     # Normal Response Codes: 201
920     # Error Response Codes: internalServerError (500),
921     #                       itemNotFound (404),
922     #                       forbidden (403),
923     #                       badRequest (400)
924     
925     request.upload_handlers = [SaveToBackendHandler(request)]
926     if not request.FILES.has_key('X-Object-Data'):
927         raise BadRequest('Missing X-Object-Data field')
928     file = request.FILES['X-Object-Data']
929     
930     checksum = file.etag
931     try:
932         version_id = request.backend.update_object_hashmap(request.user_uniq,
933                         v_account, v_container, v_object, file.size, file.content_type,
934                         file.hashmap, checksum, 'pithos', {}, True)
935     except NotAllowedError:
936         raise Forbidden('Not allowed')
937     except ItemNotExists:
938         raise ItemNotFound('Container does not exist')
939     except QuotaError:
940         raise RequestEntityTooLarge('Quota exceeded')
941     
942     response = HttpResponse(status=201)
943     response['ETag'] = checksum
944     response['X-Object-Version'] = version_id
945     response.content = checksum
946     return response
947
948 @api_method('COPY', format_allowed=True)
949 def object_copy(request, v_account, v_container, v_object):
950     # Normal Response Codes: 201
951     # Error Response Codes: internalServerError (500),
952     #                       itemNotFound (404),
953     #                       forbidden (403),
954     #                       badRequest (400)
955     
956     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
957     if not dest_account:
958         dest_account = request.user_uniq
959     dest_path = request.META.get('HTTP_DESTINATION')
960     if not dest_path:
961         raise BadRequest('Missing Destination header')
962     try:
963         dest_container, dest_name = split_container_object_string(dest_path)
964     except ValueError:
965         raise BadRequest('Invalid Destination header')
966     
967     # Evaluate conditions.
968     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
969         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
970         try:
971             meta = request.backend.get_object_meta(request.user_uniq, v_account,
972                                             v_container, v_object, 'pithos', src_version)
973         except NotAllowedError:
974             raise Forbidden('Not allowed')
975         except (ItemNotExists, VersionNotExists):
976             raise ItemNotFound('Container or object does not exist')
977         validate_matching_preconditions(request, meta)
978     
979     delimiter = request.GET.get('delimiter')
980     
981     version_id = copy_or_move_object(request, v_account, v_container, v_object,
982                                         dest_account, dest_container, dest_name, move=False, delimiter=delimiter)
983     response = HttpResponse(status=201)
984     response['X-Object-Version'] = version_id
985     return response
986
987 @api_method('MOVE', format_allowed=True)
988 def object_move(request, v_account, v_container, v_object):
989     # Normal Response Codes: 201
990     # Error Response Codes: internalServerError (500),
991     #                       itemNotFound (404),
992     #                       forbidden (403),
993     #                       badRequest (400)
994     
995     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
996     if not dest_account:
997         dest_account = request.user_uniq
998     dest_path = request.META.get('HTTP_DESTINATION')
999     if not dest_path:
1000         raise BadRequest('Missing Destination header')
1001     try:
1002         dest_container, dest_name = split_container_object_string(dest_path)
1003     except ValueError:
1004         raise BadRequest('Invalid Destination header')
1005     
1006     # Evaluate conditions.
1007     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1008         try:
1009             meta = request.backend.get_object_meta(request.user_uniq, v_account,
1010                                                     v_container, v_object, 'pithos')
1011         except NotAllowedError:
1012             raise Forbidden('Not allowed')
1013         except ItemNotExists:
1014             raise ItemNotFound('Container or object does not exist')
1015         validate_matching_preconditions(request, meta)
1016     
1017     delimiter = request.GET.get('delimiter')
1018     
1019     version_id = copy_or_move_object(request, v_account, v_container, v_object,
1020                                         dest_account, dest_container, dest_name, move=True, delimiter=delimiter)
1021     response = HttpResponse(status=201)
1022     response['X-Object-Version'] = version_id
1023     return response
1024
1025 @api_method('POST', format_allowed=True)
1026 def object_update(request, v_account, v_container, v_object):
1027     # Normal Response Codes: 202, 204
1028     # Error Response Codes: internalServerError (500),
1029     #                       conflict (409),
1030     #                       itemNotFound (404),
1031     #                       forbidden (403),
1032     #                       badRequest (400)
1033     
1034     content_type, meta, permissions, public = get_object_headers(request)
1035     
1036     try:
1037         prev_meta = request.backend.get_object_meta(request.user_uniq, v_account,
1038                                                     v_container, v_object, 'pithos')
1039     except NotAllowedError:
1040         raise Forbidden('Not allowed')
1041     except ItemNotExists:
1042         raise ItemNotFound('Object does not exist')
1043     
1044     # Evaluate conditions.
1045     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1046         validate_matching_preconditions(request, prev_meta)
1047     
1048     replace = True
1049     if 'update' in request.GET:
1050         replace = False
1051     
1052     # A Content-Type or X-Source-Object header indicates data updates.
1053     src_object = request.META.get('HTTP_X_SOURCE_OBJECT')
1054     if (not content_type or content_type != 'application/octet-stream') and not src_object:
1055         response = HttpResponse(status=202)
1056         
1057         # Do permissions first, as it may fail easier.
1058         if permissions is not None:
1059             try:
1060                 request.backend.update_object_permissions(request.user_uniq,
1061                                 v_account, v_container, v_object, permissions)
1062             except NotAllowedError:
1063                 raise Forbidden('Not allowed')
1064             except ItemNotExists:
1065                 raise ItemNotFound('Object does not exist')
1066             except ValueError:
1067                 raise BadRequest('Invalid sharing header')
1068         if public is not None:
1069             try:
1070                 request.backend.update_object_public(request.user_uniq, v_account,
1071                                                 v_container, v_object, public)
1072             except NotAllowedError:
1073                 raise Forbidden('Not allowed')
1074             except ItemNotExists:
1075                 raise ItemNotFound('Object does not exist')
1076         if meta or replace:
1077             try:
1078                 version_id = request.backend.update_object_meta(request.user_uniq,
1079                                 v_account, v_container, v_object, 'pithos', meta, replace)
1080             except NotAllowedError:
1081                 raise Forbidden('Not allowed')
1082             except ItemNotExists:
1083                 raise ItemNotFound('Object does not exist')        
1084             response['X-Object-Version'] = version_id
1085         
1086         return response
1087     
1088     # Single range update. Range must be in Content-Range.
1089     # Based on: http://code.google.com/p/gears/wiki/ContentRangePostProposal
1090     # (with the addition that '*' is allowed for the range - will append).
1091     content_range = request.META.get('HTTP_CONTENT_RANGE')
1092     if not content_range:
1093         raise BadRequest('Missing Content-Range header')
1094     ranges = get_content_range(request)
1095     if not ranges:
1096         raise RangeNotSatisfiable('Invalid Content-Range header')
1097     
1098     try:
1099         size, hashmap = request.backend.get_object_hashmap(request.user_uniq,
1100                                             v_account, v_container, v_object)
1101     except NotAllowedError:
1102         raise Forbidden('Not allowed')
1103     except ItemNotExists:
1104         raise ItemNotFound('Object does not exist')
1105     
1106     offset, length, total = ranges
1107     if offset is None:
1108         offset = size
1109     elif offset > size:
1110         raise RangeNotSatisfiable('Supplied offset is beyond object limits')
1111     if src_object:
1112         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
1113         if not src_account:
1114             src_account = request.user_uniq
1115         src_container, src_name = split_container_object_string(src_object)
1116         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
1117         try:
1118             src_size, src_hashmap = request.backend.get_object_hashmap(request.user_uniq,
1119                                         src_account, src_container, src_name, src_version)
1120         except NotAllowedError:
1121             raise Forbidden('Not allowed')
1122         except ItemNotExists:
1123             raise ItemNotFound('Source object does not exist')
1124         
1125         if length is None:
1126             length = src_size
1127         elif length > src_size:
1128             raise BadRequest('Object length is smaller than range length')
1129     else:
1130         # Require either a Content-Length, or 'chunked' Transfer-Encoding.
1131         content_length = -1
1132         if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
1133             content_length = get_content_length(request)
1134         
1135         if length is None:
1136             length = content_length
1137         else:
1138             if content_length == -1:
1139                 # TODO: Get up to length bytes in chunks.
1140                 length = content_length
1141             elif length != content_length:
1142                 raise BadRequest('Content length does not match range length')
1143     if total is not None and (total != size or offset >= size or (length > 0 and offset + length >= size)):
1144         raise RangeNotSatisfiable('Supplied range will change provided object limits')
1145     
1146     dest_bytes = request.META.get('HTTP_X_OBJECT_BYTES')
1147     if dest_bytes is not None:
1148         dest_bytes = get_int_parameter(dest_bytes)
1149         if dest_bytes is None:
1150             raise BadRequest('Invalid X-Object-Bytes header')
1151     
1152     if src_object:
1153         if offset % request.backend.block_size == 0:
1154             # Update the hashes only.
1155             sbi = 0
1156             while length > 0:
1157                 bi = int(offset / request.backend.block_size)
1158                 bl = min(length, request.backend.block_size)
1159                 if bi < len(hashmap):
1160                     if bl == request.backend.block_size:
1161                         hashmap[bi] = src_hashmap[sbi]
1162                     else:
1163                         data = request.backend.get_block(src_hashmap[sbi])
1164                         hashmap[bi] = request.backend.update_block(hashmap[bi],
1165                                                                 data[:bl], 0)
1166                 else:
1167                     hashmap.append(src_hashmap[sbi])
1168                 offset += bl
1169                 length -= bl
1170                 sbi += 1
1171         else:
1172             data = ''
1173             sbi = 0
1174             while length > 0:
1175                 data += request.backend.get_block(src_hashmap[sbi])
1176                 if length < request.backend.block_size:
1177                     data = data[:length]
1178                 bytes = put_object_block(request, hashmap, data, offset)
1179                 offset += bytes
1180                 data = data[bytes:]
1181                 length -= bytes
1182                 sbi += 1
1183     else:
1184         data = ''
1185         for d in socket_read_iterator(request, length,
1186                                         request.backend.block_size):
1187             # TODO: Raise 408 (Request Timeout) if this takes too long.
1188             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
1189             data += d
1190             bytes = put_object_block(request, hashmap, data, offset)
1191             offset += bytes
1192             data = data[bytes:]
1193         if len(data) > 0:
1194             put_object_block(request, hashmap, data, offset)
1195     
1196     if offset > size:
1197         size = offset
1198     if dest_bytes is not None and dest_bytes < size:
1199         size = dest_bytes
1200         hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)]
1201     checksum = hashmap_md5(request.backend, hashmap, size) if UPDATE_MD5 else ''
1202     try:
1203         version_id = request.backend.update_object_hashmap(request.user_uniq,
1204                         v_account, v_container, v_object, size, prev_meta['type'],
1205                         hashmap, checksum, 'pithos', meta, replace, permissions)
1206     except NotAllowedError:
1207         raise Forbidden('Not allowed')
1208     except ItemNotExists:
1209         raise ItemNotFound('Container does not exist')
1210     except ValueError:
1211         raise BadRequest('Invalid sharing header')
1212     except QuotaError:
1213         raise RequestEntityTooLarge('Quota exceeded')
1214     if public is not None:
1215         try:
1216             request.backend.update_object_public(request.user_uniq, v_account,
1217                                                 v_container, v_object, public)
1218         except NotAllowedError:
1219             raise Forbidden('Not allowed')
1220         except ItemNotExists:
1221             raise ItemNotFound('Object does not exist')
1222     
1223     response = HttpResponse(status=204)
1224     response['ETag'] = checksum
1225     response['X-Object-Version'] = version_id
1226     return response
1227
1228 @api_method('DELETE')
1229 def object_delete(request, v_account, v_container, v_object):
1230     # Normal Response Codes: 204
1231     # Error Response Codes: internalServerError (500),
1232     #                       itemNotFound (404),
1233     #                       forbidden (403),
1234     #                       badRequest (400)
1235     
1236     until = get_int_parameter(request.GET.get('until'))
1237     delimiter = request.GET.get('delimiter')
1238     
1239     try:
1240         request.backend.delete_object(request.user_uniq, v_account, v_container,
1241                                         v_object, until, delimiter=delimiter)
1242     except NotAllowedError:
1243         raise Forbidden('Not allowed')
1244     except ItemNotExists:
1245         raise ItemNotFound('Object does not exist')
1246     return HttpResponse(status=204)
1247
1248 @api_method()
1249 def method_not_allowed(request):
1250     raise BadRequest('Method not allowed')