code.grnet.gr Git - pithos/blob - snf-pithos-app/pithos/api/functions.py

   1 # Copyright 2011-2012 GRNET S.A. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or
   4 # without modification, are permitted provided that the following
   5 # conditions are met:
   6 #
   7 #   1. Redistributions of source code must retain the above
   8 #      copyright notice, this list of conditions and the following
   9 #      disclaimer.
  10 #
  11 #   2. Redistributions in binary form must reproduce the above
  12 #      copyright notice, this list of conditions and the following
  13 #      disclaimer in the documentation and/or other materials
  14 #      provided with the distribution.
  15 #
  16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  27 # POSSIBILITY OF SUCH DAMAGE.
  28 #
  29 # The views and conclusions contained in the software and
  30 # documentation are those of the authors and should not be
  31 # interpreted as representing official policies, either expressed
  32 # or implied, of GRNET S.A.
  33
  34 from xml.dom import minidom
  35 from urllib import unquote
  36
  37 from django.conf import settings
  38 from django.http import HttpResponse
  39 from django.template.loader import render_to_string
  40 from django.utils import simplejson as json
  41 from django.utils.http import parse_etags
  42 from django.utils.encoding import smart_str
  43 from django.views.decorators.csrf import csrf_exempt
  44
  45 from synnefo.lib.astakos import get_user
  46
  47 from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound, Conflict,
  48     LengthRequired, PreconditionFailed, RequestEntityTooLarge, RangeNotSatisfiable, UnprocessableEntity)
  49 from pithos.api.util import (json_encode_decimal, rename_meta_key, format_header_key, printable_header_dict,
  50     get_account_headers, put_account_headers, get_container_headers, put_container_headers, get_object_headers,
  51     put_object_headers, update_manifest_meta, update_sharing_meta, update_public_meta,
  52     validate_modification_preconditions, validate_matching_preconditions, split_container_object_string,
  53     copy_or_move_object, get_int_parameter, get_content_length, get_content_range, socket_read_iterator,
  54     SaveToBackendHandler, object_data_response, put_object_block, hashmap_md5, simple_list_response, api_method)
  55 from pithos.api.settings import UPDATE_MD5
  56
  57 from pithos.backends.base import NotAllowedError, QuotaError
  58 from pithos.backends.filter import parse_filters
  59
  60 import logging
  61 import hashlib
  62
  63
  64 logger = logging.getLogger(__name__)
  65
  66
  67 @csrf_exempt
  68 def top_demux(request):
  69     if request.method == 'GET':
  70         return account_list(request)
  71     else:
  72         return method_not_allowed(request)
  73
  74 @csrf_exempt
  75 def account_demux(request, v_account):
  76     if request.method == 'HEAD':
  77         return account_meta(request, v_account)
  78     elif request.method == 'POST':
  79         return account_update(request, v_account)
  80     elif request.method == 'GET':
  81         return container_list(request, v_account)
  82     else:
  83         return method_not_allowed(request)
  84
  85 @csrf_exempt
  86 def container_demux(request, v_account, v_container):
  87     if request.method == 'HEAD':
  88         return container_meta(request, v_account, v_container)
  89     elif request.method == 'PUT':
  90         return container_create(request, v_account, v_container)
  91     elif request.method == 'POST':
  92         return container_update(request, v_account, v_container)
  93     elif request.method == 'DELETE':
  94         return container_delete(request, v_account, v_container)
  95     elif request.method == 'GET':
  96         return object_list(request, v_account, v_container)
  97     else:
  98         return method_not_allowed(request)
  99
 100 @csrf_exempt
 101 def object_demux(request, v_account, v_container, v_object):
 102     # Helper to avoid placing the token in the URL when loading objects from a browser.
 103     if request.method == 'HEAD':
 104         return object_meta(request, v_account, v_container, v_object)
 105     elif request.method == 'GET':
 106         return object_read(request, v_account, v_container, v_object)
 107     elif request.method == 'PUT':
 108         return object_write(request, v_account, v_container, v_object)
 109     elif request.method == 'COPY':
 110         return object_copy(request, v_account, v_container, v_object)
 111     elif request.method == 'MOVE':
 112         return object_move(request, v_account, v_container, v_object)
 113     elif request.method == 'POST':
 114         if request.META.get('CONTENT_TYPE', '').startswith('multipart/form-data'):
 115             return object_write_form(request, v_account, v_container, v_object)
 116         return object_update(request, v_account, v_container, v_object)
 117     elif request.method == 'DELETE':
 118         return object_delete(request, v_account, v_container, v_object)
 119     else:
 120         return method_not_allowed(request)
 121
 122 @api_method('GET', user_required=False)
 123 def authenticate(request):
 124     # Normal Response Codes: 204
 125     # Error Response Codes: internalServerError (500),
 126     #                       forbidden (403),
 127     #                       badRequest (400)
 128
 129     x_auth_user = request.META.get('HTTP_X_AUTH_USER')
 130     x_auth_key = request.META.get('HTTP_X_AUTH_KEY')
 131     if not x_auth_user or not x_auth_key:
 132         raise BadRequest('Missing X-Auth-User or X-Auth-Key header')
 133     response = HttpResponse(status=204)
 134
 135     uri = request.build_absolute_uri()
 136     if '?' in uri:
 137         uri = uri[:uri.find('?')]
 138
 139     response['X-Auth-Token'] = x_auth_key
 140     response['X-Storage-Url'] = uri + ('' if uri.endswith('/') else '/') + x_auth_user
 141     return response
 142
 143 @api_method('GET', format_allowed=True)
 144 def account_list(request):
 145     # Normal Response Codes: 200, 204
 146     # Error Response Codes: internalServerError (500),
 147     #                       badRequest (400)
 148     if getattr(request, 'user', None) is None:
 149         return authenticate(request)
 150
 151     response = HttpResponse()
 152
 153     marker = request.GET.get('marker')
 154     limit = get_int_parameter(request.GET.get('limit'))
 155     if not limit:
 156         limit = 10000
 157
 158     accounts = request.backend.list_accounts(request.user_uniq, marker, limit)
 159
 160     if request.serialization == 'text':
 161         if len(accounts) == 0:
 162             # The cloudfiles python bindings expect 200 if json/xml.
 163             response.status_code = 204
 164             return response
 165         response.status_code = 200
 166         response.content = '\n'.join(accounts) + '\n'
 167         return response
 168
 169     account_meta = []
 170     for x in accounts:
 171         if x == request.user_uniq:
 172             continue
 173         try:
 174             meta = request.backend.get_account_meta(request.user_uniq, x, 'pithos', include_user_defined=False)
 175             groups = request.backend.get_account_groups(request.user_uniq, x)
 176         except NotAllowedError:
 177             raise Forbidden('Not allowed')
 178         else:
 179             rename_meta_key(meta, 'modified', 'last_modified')
 180             rename_meta_key(meta, 'until_timestamp', 'x_account_until_timestamp')
 181             if groups:
 182                 meta['X-Account-Group'] = printable_header_dict(dict([(k, ','.join(v)) for k, v in groups.iteritems()]))
 183             account_meta.append(printable_header_dict(meta))
 184     if request.serialization == 'xml':
 185         data = render_to_string('accounts.xml', {'accounts': account_meta})
 186     elif request.serialization  == 'json':
 187         data = json.dumps(account_meta)
 188     response.status_code = 200
 189     response.content = data
 190     return response
 191
 192 @api_method('HEAD')
 193 def account_meta(request, v_account):
 194     # Normal Response Codes: 204
 195     # Error Response Codes: internalServerError (500),
 196     #                       forbidden (403),
 197     #                       badRequest (400)
 198
 199     until = get_int_parameter(request.GET.get('until'))
 200     try:
 201         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
 202         groups = request.backend.get_account_groups(request.user_uniq, v_account)
 203         policy = request.backend.get_account_policy(request.user_uniq, v_account)
 204     except NotAllowedError:
 205         raise Forbidden('Not allowed')
 206
 207     validate_modification_preconditions(request, meta)
 208
 209     response = HttpResponse(status=204)
 210     put_account_headers(response, meta, groups, policy)
 211     return response
 212
 213 @api_method('POST')
 214 def account_update(request, v_account):
 215     # Normal Response Codes: 202
 216     # Error Response Codes: internalServerError (500),
 217     #                       forbidden (403),
 218     #                       badRequest (400)
 219
 220     meta, groups = get_account_headers(request)
 221     replace = True
 222     if 'update' in request.GET:
 223         replace = False
 224     if groups:
 225         try:
 226             request.backend.update_account_groups(request.user_uniq, v_account,
 227                                                     groups, replace)
 228         except NotAllowedError:
 229             raise Forbidden('Not allowed')
 230         except ValueError:
 231             raise BadRequest('Invalid groups header')
 232     if meta or replace:
 233         try:
 234             request.backend.update_account_meta(request.user_uniq, v_account,
 235                                                 'pithos', meta, replace)
 236         except NotAllowedError:
 237             raise Forbidden('Not allowed')
 238     return HttpResponse(status=202)
 239
 240 @api_method('GET', format_allowed=True)
 241 def container_list(request, v_account):
 242     # Normal Response Codes: 200, 204
 243     # Error Response Codes: internalServerError (500),
 244     #                       itemNotFound (404),
 245     #                       forbidden (403),
 246     #                       badRequest (400)
 247
 248     until = get_int_parameter(request.GET.get('until'))
 249     try:
 250         meta = request.backend.get_account_meta(request.user_uniq, v_account, 'pithos', until)
 251         groups = request.backend.get_account_groups(request.user_uniq, v_account)
 252         policy = request.backend.get_account_policy(request.user_uniq, v_account)
 253     except NotAllowedError:
 254         raise Forbidden('Not allowed')
 255
 256     validate_modification_preconditions(request, meta)
 257
 258     response = HttpResponse()
 259     put_account_headers(response, meta, groups, policy)
 260
 261     marker = request.GET.get('marker')
 262     limit = get_int_parameter(request.GET.get('limit'))
 263     if not limit:
 264         limit = 10000
 265
 266     shared = False
 267     if 'shared' in request.GET:
 268         shared = True
 269     public = False
 270     if 'public' in request.GET:
 271         public = True
 272
 273     try:
 274         containers = request.backend.list_containers(request.user_uniq, v_account,
 275                                                 marker, limit, shared, until, public)
 276     except NotAllowedError:
 277         raise Forbidden('Not allowed')
 278     except NameError:
 279         containers = []
 280
 281     if request.serialization == 'text':
 282         if len(containers) == 0:
 283             # The cloudfiles python bindings expect 200 if json/xml.
 284             response.status_code = 204
 285             return response
 286         response.status_code = 200
 287         response.content = '\n'.join(containers) + '\n'
 288         return response
 289
 290     container_meta = []
 291     for x in containers:
 292         try:
 293             meta = request.backend.get_container_meta(request.user_uniq, v_account,
 294                                                         x, 'pithos', until, include_user_defined=False)
 295             policy = request.backend.get_container_policy(request.user_uniq,
 296                                                             v_account, x)
 297         except NotAllowedError:
 298             raise Forbidden('Not allowed')
 299         except NameError:
 300             pass
 301         else:
 302             rename_meta_key(meta, 'modified', 'last_modified')
 303             rename_meta_key(meta, 'until_timestamp', 'x_container_until_timestamp')
 304             if policy:
 305                 meta['X-Container-Policy'] = printable_header_dict(dict([(k, v) for k, v in policy.iteritems()]))
 306             container_meta.append(printable_header_dict(meta))
 307     if request.serialization == 'xml':
 308         data = render_to_string('containers.xml', {'account': v_account, 'containers': container_meta})
 309     elif request.serialization  == 'json':
 310         data = json.dumps(container_meta)
 311     response.status_code = 200
 312     response.content = data
 313     return response
 314
 315 @api_method('HEAD')
 316 def container_meta(request, v_account, v_container):
 317     # Normal Response Codes: 204
 318     # Error Response Codes: internalServerError (500),
 319     #                       itemNotFound (404),
 320     #                       forbidden (403),
 321     #                       badRequest (400)
 322
 323     until = get_int_parameter(request.GET.get('until'))
 324     try:
 325         meta = request.backend.get_container_meta(request.user_uniq, v_account,
 326                                                     v_container, 'pithos', until)
 327         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
 328                                                 v_account, v_container, 'pithos', until)
 329         policy = request.backend.get_container_policy(request.user_uniq, v_account,
 330                                                         v_container)
 331     except NotAllowedError:
 332         raise Forbidden('Not allowed')
 333     except NameError:
 334         raise ItemNotFound('Container does not exist')
 335
 336     validate_modification_preconditions(request, meta)
 337
 338     response = HttpResponse(status=204)
 339     put_container_headers(request, response, meta, policy)
 340     return response
 341
 342 @api_method('PUT')
 343 def container_create(request, v_account, v_container):
 344     # Normal Response Codes: 201, 202
 345     # Error Response Codes: internalServerError (500),
 346     #                       itemNotFound (404),
 347     #                       forbidden (403),
 348     #                       badRequest (400)
 349
 350     meta, policy = get_container_headers(request)
 351
 352     try:
 353         request.backend.put_container(request.user_uniq, v_account, v_container, policy)
 354         ret = 201
 355     except NotAllowedError:
 356         raise Forbidden('Not allowed')
 357     except ValueError:
 358         raise BadRequest('Invalid policy header')
 359     except NameError:
 360         ret = 202
 361
 362     if ret == 202 and policy:
 363         try:
 364             request.backend.update_container_policy(request.user_uniq, v_account,
 365                                             v_container, policy, replace=False)
 366         except NotAllowedError:
 367             raise Forbidden('Not allowed')
 368         except NameError:
 369             raise ItemNotFound('Container does not exist')
 370         except ValueError:
 371             raise BadRequest('Invalid policy header')
 372     if meta:
 373         try:
 374             request.backend.update_container_meta(request.user_uniq, v_account,
 375                                             v_container, 'pithos', meta, replace=False)
 376         except NotAllowedError:
 377             raise Forbidden('Not allowed')
 378         except NameError:
 379             raise ItemNotFound('Container does not exist')
 380
 381     return HttpResponse(status=ret)
 382
 383 @api_method('POST', format_allowed=True)
 384 def container_update(request, v_account, v_container):
 385     # Normal Response Codes: 202
 386     # Error Response Codes: internalServerError (500),
 387     #                       itemNotFound (404),
 388     #                       forbidden (403),
 389     #                       badRequest (400)
 390
 391     meta, policy = get_container_headers(request)
 392     replace = True
 393     if 'update' in request.GET:
 394         replace = False
 395     if policy:
 396         try:
 397             request.backend.update_container_policy(request.user_uniq, v_account,
 398                                                 v_container, policy, replace)
 399         except NotAllowedError:
 400             raise Forbidden('Not allowed')
 401         except NameError:
 402             raise ItemNotFound('Container does not exist')
 403         except ValueError:
 404             raise BadRequest('Invalid policy header')
 405     if meta or replace:
 406         try:
 407             request.backend.update_container_meta(request.user_uniq, v_account,
 408                                                     v_container, 'pithos', meta, replace)
 409         except NotAllowedError:
 410             raise Forbidden('Not allowed')
 411         except NameError:
 412             raise ItemNotFound('Container does not exist')
 413
 414     content_length = -1
 415     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
 416         content_length = get_int_parameter(request.META.get('CONTENT_LENGTH', 0))
 417     content_type = request.META.get('CONTENT_TYPE')
 418     hashmap = []
 419     if content_type and content_type == 'application/octet-stream' and content_length != 0:
 420         for data in socket_read_iterator(request, content_length,
 421                                             request.backend.block_size):
 422             # TODO: Raise 408 (Request Timeout) if this takes too long.
 423             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
 424             hashmap.append(request.backend.put_block(data))
 425
 426     response = HttpResponse(status=202)
 427     if hashmap:
 428         response.content = simple_list_response(request, hashmap)
 429     return response
 430
 431 @api_method('DELETE')
 432 def container_delete(request, v_account, v_container):
 433     # Normal Response Codes: 204
 434     # Error Response Codes: internalServerError (500),
 435     #                       conflict (409),
 436     #                       itemNotFound (404),
 437     #                       forbidden (403),
 438     #                       badRequest (400)
 439
 440     until = get_int_parameter(request.GET.get('until'))
 441     try:
 442         request.backend.delete_container(request.user_uniq, v_account, v_container,
 443                                             until)
 444     except NotAllowedError:
 445         raise Forbidden('Not allowed')
 446     except NameError:
 447         raise ItemNotFound('Container does not exist')
 448     except IndexError:
 449         raise Conflict('Container is not empty')
 450     return HttpResponse(status=204)
 451
 452 @api_method('GET', format_allowed=True)
 453 def object_list(request, v_account, v_container):
 454     # Normal Response Codes: 200, 204
 455     # Error Response Codes: internalServerError (500),
 456     #                       itemNotFound (404),
 457     #                       forbidden (403),
 458     #                       badRequest (400)
 459
 460     until = get_int_parameter(request.GET.get('until'))
 461     try:
 462         meta = request.backend.get_container_meta(request.user_uniq, v_account,
 463                                                     v_container, 'pithos', until)
 464         meta['object_meta'] = request.backend.list_container_meta(request.user_uniq,
 465                                                 v_account, v_container, 'pithos', until)
 466         policy = request.backend.get_container_policy(request.user_uniq, v_account,
 467                                                         v_container)
 468     except NotAllowedError:
 469         raise Forbidden('Not allowed')
 470     except NameError:
 471         raise ItemNotFound('Container does not exist')
 472
 473     validate_modification_preconditions(request, meta)
 474
 475     response = HttpResponse()
 476     put_container_headers(request, response, meta, policy)
 477
 478     path = request.GET.get('path')
 479     prefix = request.GET.get('prefix')
 480     delimiter = request.GET.get('delimiter')
 481
 482     # Path overrides prefix and delimiter.
 483     virtual = True
 484     if path:
 485         prefix = path
 486         delimiter = '/'
 487         virtual = False
 488
 489     # Naming policy.
 490     if prefix and delimiter and not prefix.endswith(delimiter):
 491         prefix = prefix + delimiter
 492     if not prefix:
 493         prefix = ''
 494     prefix = prefix.lstrip('/')
 495
 496     marker = request.GET.get('marker')
 497     limit = get_int_parameter(request.GET.get('limit'))
 498     if not limit:
 499         limit = 10000
 500
 501     keys = request.GET.get('meta')
 502     if keys:
 503         keys = [smart_str(x.strip()) for x in keys.split(',') if x.strip() != '']
 504         included, excluded, opers = parse_filters(keys)
 505         keys = []
 506         keys += [format_header_key('X-Object-Meta-' + x) for x in included]
 507         keys += ['!'+format_header_key('X-Object-Meta-' + x) for x in excluded]
 508         keys += ['%s%s%s' % (format_header_key('X-Object-Meta-' + k), o, v) for k, o, v in opers]
 509     else:
 510         keys = []
 511
 512     shared = False
 513     if 'shared' in request.GET:
 514         shared = True
 515     public = False
 516     if 'public' in request.GET:
 517         public = True
 518
 519     if request.serialization == 'text':
 520         try:
 521             objects = request.backend.list_objects(request.user_uniq, v_account,
 522                                         v_container, prefix, delimiter, marker,
 523                                         limit, virtual, 'pithos', keys, shared,
 524                                         until, None, public)
 525         except NotAllowedError:
 526             raise Forbidden('Not allowed')
 527         except NameError:
 528             raise ItemNotFound('Container does not exist')
 529
 530         if len(objects) == 0:
 531             # The cloudfiles python bindings expect 200 if json/xml.
 532             response.status_code = 204
 533             return response
 534         response.status_code = 200
 535         response.content = '\n'.join([x[0] for x in objects]) + '\n'
 536         return response
 537
 538     try:
 539         objects = request.backend.list_object_meta(request.user_uniq, v_account,
 540                                     v_container, prefix, delimiter, marker,
 541                                     limit, virtual, 'pithos', keys, shared, until, None, public)
 542         object_permissions = {}
 543         object_public = {}
 544         if until is None:
 545             name_idx = len('/'.join((v_account, v_container, '')))
 546             for x in request.backend.list_object_permissions(request.user_uniq,
 547                                     v_account, v_container, prefix):
 548                 object = x[name_idx:]
 549                 object_permissions[object] = request.backend.get_object_permissions(
 550                                     request.user_uniq, v_account, v_container, object)
 551             for k, v in request.backend.list_object_public(request.user_uniq,
 552                                     v_account, v_container, prefix).iteritems():
 553                 object_public[k[name_idx:]] = v
 554     except NotAllowedError:
 555         raise Forbidden('Not allowed')
 556     except NameError:
 557         raise ItemNotFound('Container does not exist')
 558
 559     object_meta = []
 560     for meta in objects:
 561         if len(meta) == 1:
 562             # Virtual objects/directories.
 563             object_meta.append(meta)
 564         else:
 565             rename_meta_key(meta, 'hash', 'x_object_hash') # Will be replaced by checksum.
 566             rename_meta_key(meta, 'checksum', 'hash')
 567             rename_meta_key(meta, 'type', 'content_type')
 568             rename_meta_key(meta, 'uuid', 'x_object_uuid')
 569             if until is not None and 'modified' in meta:
 570                 del(meta['modified'])
 571             else:
 572                 rename_meta_key(meta, 'modified', 'last_modified')
 573             rename_meta_key(meta, 'modified_by', 'x_object_modified_by')
 574             rename_meta_key(meta, 'version', 'x_object_version')
 575             rename_meta_key(meta, 'version_timestamp', 'x_object_version_timestamp')
 576             permissions = object_permissions.get(meta['name'], None)
 577             if permissions:
 578                 update_sharing_meta(request, permissions, v_account, v_container, meta['name'], meta)
 579             public = object_public.get(meta['name'], None)
 580             if public:
 581                 update_public_meta(public, meta)
 582             object_meta.append(printable_header_dict(meta))
 583     if request.serialization == 'xml':
 584         data = render_to_string('objects.xml', {'container': v_container, 'objects': object_meta})
 585     elif request.serialization  == 'json':
 586         data = json.dumps(object_meta, default=json_encode_decimal)
 587     response.status_code = 200
 588     response.content = data
 589     return response
 590
 591 @api_method('HEAD')
 592 def object_meta(request, v_account, v_container, v_object):
 593     # Normal Response Codes: 204
 594     # Error Response Codes: internalServerError (500),
 595     #                       itemNotFound (404),
 596     #                       forbidden (403),
 597     #                       badRequest (400)
 598
 599     version = request.GET.get('version')
 600     try:
 601         meta = request.backend.get_object_meta(request.user_uniq, v_account,
 602                                                 v_container, v_object, 'pithos', version)
 603         if version is None:
 604             permissions = request.backend.get_object_permissions(request.user_uniq,
 605                                             v_account, v_container, v_object)
 606             public = request.backend.get_object_public(request.user_uniq, v_account,
 607                                                         v_container, v_object)
 608         else:
 609             permissions = None
 610             public = None
 611     except NotAllowedError:
 612         raise Forbidden('Not allowed')
 613     except NameError:
 614         raise ItemNotFound('Object does not exist')
 615     except IndexError:
 616         raise ItemNotFound('Version does not exist')
 617
 618     update_manifest_meta(request, v_account, meta)
 619     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
 620     update_public_meta(public, meta)
 621
 622     # Evaluate conditions.
 623     validate_modification_preconditions(request, meta)
 624     try:
 625         validate_matching_preconditions(request, meta)
 626     except NotModified:
 627         response = HttpResponse(status=304)
 628         response['ETag'] = meta['checksum']
 629         return response
 630
 631     response = HttpResponse(status=200)
 632     put_object_headers(response, meta)
 633     return response
 634
 635 @api_method('GET', format_allowed=True)
 636 def object_read(request, v_account, v_container, v_object):
 637     # Normal Response Codes: 200, 206
 638     # Error Response Codes: internalServerError (500),
 639     #                       rangeNotSatisfiable (416),
 640     #                       preconditionFailed (412),
 641     #                       itemNotFound (404),
 642     #                       forbidden (403),
 643     #                       badRequest (400),
 644     #                       notModified (304)
 645
 646     version = request.GET.get('version')
 647
 648     # Reply with the version list. Do this first, as the object may be deleted.
 649     if version == 'list':
 650         if request.serialization == 'text':
 651             raise BadRequest('No format specified for version list.')
 652
 653         try:
 654             v = request.backend.list_versions(request.user_uniq, v_account,
 655                                                 v_container, v_object)
 656         except NotAllowedError:
 657             raise Forbidden('Not allowed')
 658         d = {'versions': v}
 659         if request.serialization == 'xml':
 660             d['object'] = v_object
 661             data = render_to_string('versions.xml', d)
 662         elif request.serialization  == 'json':
 663             data = json.dumps(d, default=json_encode_decimal)
 664
 665         response = HttpResponse(data, status=200)
 666         response['Content-Length'] = len(data)
 667         return response
 668
 669     try:
 670         meta = request.backend.get_object_meta(request.user_uniq, v_account,
 671                                                 v_container, v_object, 'pithos', version)
 672         if version is None:
 673             permissions = request.backend.get_object_permissions(request.user_uniq,
 674                                             v_account, v_container, v_object)
 675             public = request.backend.get_object_public(request.user_uniq, v_account,
 676                                                         v_container, v_object)
 677         else:
 678             permissions = None
 679             public = None
 680     except NotAllowedError:
 681         raise Forbidden('Not allowed')
 682     except NameError:
 683         raise ItemNotFound('Object does not exist')
 684     except IndexError:
 685         raise ItemNotFound('Version does not exist')
 686
 687     update_manifest_meta(request, v_account, meta)
 688     update_sharing_meta(request, permissions, v_account, v_container, v_object, meta)
 689     update_public_meta(public, meta)
 690
 691     # Evaluate conditions.
 692     validate_modification_preconditions(request, meta)
 693     try:
 694         validate_matching_preconditions(request, meta)
 695     except NotModified:
 696         response = HttpResponse(status=304)
 697         response['ETag'] = meta['checksum']
 698         return response
 699
 700     hashmap_reply = False
 701     if 'hashmap' in request.GET and request.serialization != 'text':
 702         hashmap_reply = True
 703
 704     sizes = []
 705     hashmaps = []
 706     if 'X-Object-Manifest' in meta and not hashmap_reply:
 707         try:
 708             src_container, src_name = split_container_object_string('/' + meta['X-Object-Manifest'])
 709             objects = request.backend.list_objects(request.user_uniq, v_account,
 710                                 src_container, prefix=src_name, virtual=False)
 711         except NotAllowedError:
 712             raise Forbidden('Not allowed')
 713         except ValueError:
 714             raise BadRequest('Invalid X-Object-Manifest header')
 715         except NameError:
 716             raise ItemNotFound('Container does not exist')
 717
 718         try:
 719             for x in objects:
 720                 s, h = request.backend.get_object_hashmap(request.user_uniq,
 721                                         v_account, src_container, x[0], x[1])
 722                 sizes.append(s)
 723                 hashmaps.append(h)
 724         except NotAllowedError:
 725             raise Forbidden('Not allowed')
 726         except NameError:
 727             raise ItemNotFound('Object does not exist')
 728         except IndexError:
 729             raise ItemNotFound('Version does not exist')
 730     else:
 731         try:
 732             s, h = request.backend.get_object_hashmap(request.user_uniq, v_account,
 733                                                 v_container, v_object, version)
 734             sizes.append(s)
 735             hashmaps.append(h)
 736         except NotAllowedError:
 737             raise Forbidden('Not allowed')
 738         except NameError:
 739             raise ItemNotFound('Object does not exist')
 740         except IndexError:
 741             raise ItemNotFound('Version does not exist')
 742
 743     # Reply with the hashmap.
 744     if hashmap_reply:
 745         size = sum(sizes)
 746         hashmap = sum(hashmaps, [])
 747         d = {
 748             'block_size': request.backend.block_size,
 749             'block_hash': request.backend.hash_algorithm,
 750             'bytes': size,
 751             'hashes': hashmap}
 752         if request.serialization == 'xml':
 753             d['object'] = v_object
 754             data = render_to_string('hashes.xml', d)
 755         elif request.serialization  == 'json':
 756             data = json.dumps(d)
 757
 758         response = HttpResponse(data, status=200)
 759         put_object_headers(response, meta)
 760         response['Content-Length'] = len(data)
 761         return response
 762
 763     request.serialization = 'text' # Unset.
 764     return object_data_response(request, sizes, hashmaps, meta)
 765
 766 @api_method('PUT', format_allowed=True)
 767 def object_write(request, v_account, v_container, v_object):
 768     # Normal Response Codes: 201
 769     # Error Response Codes: internalServerError (500),
 770     #                       unprocessableEntity (422),
 771     #                       lengthRequired (411),
 772     #                       conflict (409),
 773     #                       itemNotFound (404),
 774     #                       forbidden (403),
 775     #                       badRequest (400)
 776
 777     # Evaluate conditions.
 778     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
 779         try:
 780             meta = request.backend.get_object_meta(request.user_uniq, v_account,
 781                                                         v_container, v_object, 'pithos')
 782         except NotAllowedError:
 783             raise Forbidden('Not allowed')
 784         except NameError:
 785             meta = {}
 786         validate_matching_preconditions(request, meta)
 787
 788     copy_from = request.META.get('HTTP_X_COPY_FROM')
 789     move_from = request.META.get('HTTP_X_MOVE_FROM')
 790     if copy_from or move_from:
 791         content_length = get_content_length(request) # Required by the API.
 792
 793         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
 794         if not src_account:
 795             src_account = request.user_uniq
 796         if move_from:
 797             try:
 798                 src_container, src_name = split_container_object_string(move_from)
 799             except ValueError:
 800                 raise BadRequest('Invalid X-Move-From header')
 801             version_id = copy_or_move_object(request, src_account, src_container, src_name,
 802                                                 v_account, v_container, v_object, move=True)
 803         else:
 804             try:
 805                 src_container, src_name = split_container_object_string(copy_from)
 806             except ValueError:
 807                 raise BadRequest('Invalid X-Copy-From header')
 808             version_id = copy_or_move_object(request, src_account, src_container, src_name,
 809                                                 v_account, v_container, v_object, move=False)
 810         response = HttpResponse(status=201)
 811         response['X-Object-Version'] = version_id
 812         return response
 813
 814     content_type, meta, permissions, public = get_object_headers(request)
 815     content_length = -1
 816     if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
 817         content_length = get_content_length(request)
 818     # Should be BadRequest, but API says otherwise.
 819     if content_type is None:
 820         raise LengthRequired('Missing Content-Type header')
 821
 822     if 'hashmap' in request.GET:
 823         if request.serialization not in ('json', 'xml'):
 824             raise BadRequest('Invalid hashmap format')
 825
 826         data = ''
 827         for block in socket_read_iterator(request, content_length,
 828                                             request.backend.block_size):
 829             data = '%s%s' % (data, block)
 830
 831         if request.serialization == 'json':
 832             d = json.loads(data)
 833             if not hasattr(d, '__getitem__'):
 834                 raise BadRequest('Invalid data formating')
 835             try:
 836                 hashmap = d['hashes']
 837                 size = int(d['bytes'])
 838             except:
 839                 raise BadRequest('Invalid data formatting')
 840         elif request.serialization == 'xml':
 841             try:
 842                 xml = minidom.parseString(data)
 843                 obj = xml.getElementsByTagName('object')[0]
 844                 size = int(obj.attributes['bytes'].value)
 845
 846                 hashes = xml.getElementsByTagName('hash')
 847                 hashmap = []
 848                 for hash in hashes:
 849                     hashmap.append(hash.firstChild.data)
 850             except:
 851                 raise BadRequest('Invalid data formatting')
 852
 853         checksum = '' # Do not set to None (will copy previous value).
 854     else:
 855         md5 = hashlib.md5()
 856         size = 0
 857         hashmap = []
 858         for data in socket_read_iterator(request, content_length,
 859                                             request.backend.block_size):
 860             # TODO: Raise 408 (Request Timeout) if this takes too long.
 861             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
 862             size += len(data)
 863             hashmap.append(request.backend.put_block(data))
 864             md5.update(data)
 865
 866         checksum = md5.hexdigest().lower()
 867         etag = request.META.get('HTTP_ETAG')
 868         if etag and parse_etags(etag)[0].lower() != checksum:
 869             raise UnprocessableEntity('Object ETag does not match')
 870
 871     try:
 872         version_id = request.backend.update_object_hashmap(request.user_uniq,
 873                         v_account, v_container, v_object, size, content_type,
 874                         hashmap, checksum, 'pithos', meta, True, permissions)
 875     except NotAllowedError:
 876         raise Forbidden('Not allowed')
 877     except IndexError, e:
 878         raise Conflict(simple_list_response(request, e.data))
 879     except NameError:
 880         raise ItemNotFound('Container does not exist')
 881     except ValueError:
 882         raise BadRequest('Invalid sharing header')
 883     except QuotaError:
 884         raise RequestEntityTooLarge('Quota exceeded')
 885     if not checksum and UPDATE_MD5:
 886         # Update the MD5 after the hashmap, as there may be missing hashes.
 887         checksum = hashmap_md5(request.backend, hashmap, size)
 888         try:
 889             request.backend.update_object_checksum(request.user_uniq,
 890               v_account, v_container, v_object, version_id, checksum)
 891         except NotAllowedError:
 892             raise Forbidden('Not allowed')
 893     if public is not None:
 894         try:
 895             request.backend.update_object_public(request.user_uniq, v_account,
 896                                                 v_container, v_object, public)
 897         except NotAllowedError:
 898             raise Forbidden('Not allowed')
 899         except NameError:
 900             raise ItemNotFound('Object does not exist')
 901
 902     response = HttpResponse(status=201)
 903     if checksum:
 904         response['ETag'] = checksum
 905     response['X-Object-Version'] = version_id
 906     return response
 907
 908 @api_method('POST')
 909 def object_write_form(request, v_account, v_container, v_object):
 910     # Normal Response Codes: 201
 911     # Error Response Codes: internalServerError (500),
 912     #                       itemNotFound (404),
 913     #                       forbidden (403),
 914     #                       badRequest (400)
 915
 916     request.upload_handlers = [SaveToBackendHandler(request)]
 917     if not request.FILES.has_key('X-Object-Data'):
 918         raise BadRequest('Missing X-Object-Data field')
 919     file = request.FILES['X-Object-Data']
 920
 921     checksum = file.etag
 922     try:
 923         version_id = request.backend.update_object_hashmap(request.user_uniq,
 924                         v_account, v_container, v_object, file.size, file.content_type,
 925                         file.hashmap, checksum, 'pithos', {}, True)
 926     except NotAllowedError:
 927         raise Forbidden('Not allowed')
 928     except NameError:
 929         raise ItemNotFound('Container does not exist')
 930     except QuotaError:
 931         raise RequestEntityTooLarge('Quota exceeded')
 932
 933     response = HttpResponse(status=201)
 934     response['ETag'] = checksum
 935     response['X-Object-Version'] = version_id
 936     response.content = checksum
 937     return response
 938
 939 @api_method('COPY', format_allowed=True)
 940 def object_copy(request, v_account, v_container, v_object):
 941     # Normal Response Codes: 201
 942     # Error Response Codes: internalServerError (500),
 943     #                       itemNotFound (404),
 944     #                       forbidden (403),
 945     #                       badRequest (400)
 946
 947     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
 948     if not dest_account:
 949         dest_account = request.user_uniq
 950     dest_path = request.META.get('HTTP_DESTINATION')
 951     if not dest_path:
 952         raise BadRequest('Missing Destination header')
 953     try:
 954         dest_container, dest_name = split_container_object_string(dest_path)
 955     except ValueError:
 956         raise BadRequest('Invalid Destination header')
 957
 958     # Evaluate conditions.
 959     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
 960         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
 961         try:
 962             meta = request.backend.get_object_meta(request.user_uniq, v_account,
 963                                             v_container, v_object, 'pithos', src_version)
 964         except NotAllowedError:
 965             raise Forbidden('Not allowed')
 966         except (NameError, IndexError):
 967             raise ItemNotFound('Container or object does not exist')
 968         validate_matching_preconditions(request, meta)
 969
 970     version_id = copy_or_move_object(request, v_account, v_container, v_object,
 971                                         dest_account, dest_container, dest_name, move=False)
 972     response = HttpResponse(status=201)
 973     response['X-Object-Version'] = version_id
 974     return response
 975
 976 @api_method('MOVE', format_allowed=True)
 977 def object_move(request, v_account, v_container, v_object):
 978     # Normal Response Codes: 201
 979     # Error Response Codes: internalServerError (500),
 980     #                       itemNotFound (404),
 981     #                       forbidden (403),
 982     #                       badRequest (400)
 983
 984     dest_account = request.META.get('HTTP_DESTINATION_ACCOUNT')
 985     if not dest_account:
 986         dest_account = request.user_uniq
 987     dest_path = request.META.get('HTTP_DESTINATION')
 988     if not dest_path:
 989         raise BadRequest('Missing Destination header')
 990     try:
 991         dest_container, dest_name = split_container_object_string(dest_path)
 992     except ValueError:
 993         raise BadRequest('Invalid Destination header')
 994
 995     # Evaluate conditions.
 996     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
 997         try:
 998             meta = request.backend.get_object_meta(request.user_uniq, v_account,
 999                                                     v_container, v_object, 'pithos')
1000         except NotAllowedError:
1001             raise Forbidden('Not allowed')
1002         except NameError:
1003             raise ItemNotFound('Container or object does not exist')
1004         validate_matching_preconditions(request, meta)
1005
1006     version_id = copy_or_move_object(request, v_account, v_container, v_object,
1007                                         dest_account, dest_container, dest_name, move=True)
1008     response = HttpResponse(status=201)
1009     response['X-Object-Version'] = version_id
1010     return response
1011
1012 @api_method('POST', format_allowed=True)
1013 def object_update(request, v_account, v_container, v_object):
1014     # Normal Response Codes: 202, 204
1015     # Error Response Codes: internalServerError (500),
1016     #                       conflict (409),
1017     #                       itemNotFound (404),
1018     #                       forbidden (403),
1019     #                       badRequest (400)
1020
1021     content_type, meta, permissions, public = get_object_headers(request)
1022
1023     try:
1024         prev_meta = request.backend.get_object_meta(request.user_uniq, v_account,
1025                                                     v_container, v_object, 'pithos')
1026     except NotAllowedError:
1027         raise Forbidden('Not allowed')
1028     except NameError:
1029         raise ItemNotFound('Object does not exist')
1030
1031     # Evaluate conditions.
1032     if request.META.get('HTTP_IF_MATCH') or request.META.get('HTTP_IF_NONE_MATCH'):
1033         validate_matching_preconditions(request, prev_meta)
1034
1035     replace = True
1036     if 'update' in request.GET:
1037         replace = False
1038
1039     # A Content-Type or X-Source-Object header indicates data updates.
1040     src_object = request.META.get('HTTP_X_SOURCE_OBJECT')
1041     if (not content_type or content_type != 'application/octet-stream') and not src_object:
1042         response = HttpResponse(status=202)
1043
1044         # Do permissions first, as it may fail easier.
1045         if permissions is not None:
1046             try:
1047                 request.backend.update_object_permissions(request.user_uniq,
1048                                 v_account, v_container, v_object, permissions)
1049             except NotAllowedError:
1050                 raise Forbidden('Not allowed')
1051             except NameError:
1052                 raise ItemNotFound('Object does not exist')
1053             except ValueError:
1054                 raise BadRequest('Invalid sharing header')
1055         if public is not None:
1056             try:
1057                 request.backend.update_object_public(request.user_uniq, v_account,
1058                                                 v_container, v_object, public)
1059             except NotAllowedError:
1060                 raise Forbidden('Not allowed')
1061             except NameError:
1062                 raise ItemNotFound('Object does not exist')
1063         if meta or replace:
1064             try:
1065                 version_id = request.backend.update_object_meta(request.user_uniq,
1066                                 v_account, v_container, v_object, 'pithos', meta, replace)
1067             except NotAllowedError:
1068                 raise Forbidden('Not allowed')
1069             except NameError:
1070                 raise ItemNotFound('Object does not exist')
1071             response['X-Object-Version'] = version_id
1072
1073         return response
1074
1075     # Single range update. Range must be in Content-Range.
1076     # Based on: http://code.google.com/p/gears/wiki/ContentRangePostProposal
1077     # (with the addition that '*' is allowed for the range - will append).
1078     content_range = request.META.get('HTTP_CONTENT_RANGE')
1079     if not content_range:
1080         raise BadRequest('Missing Content-Range header')
1081     ranges = get_content_range(request)
1082     if not ranges:
1083         raise RangeNotSatisfiable('Invalid Content-Range header')
1084
1085     try:
1086         size, hashmap = request.backend.get_object_hashmap(request.user_uniq,
1087                                             v_account, v_container, v_object)
1088     except NotAllowedError:
1089         raise Forbidden('Not allowed')
1090     except NameError:
1091         raise ItemNotFound('Object does not exist')
1092
1093     offset, length, total = ranges
1094     if offset is None:
1095         offset = size
1096     elif offset > size:
1097         raise RangeNotSatisfiable('Supplied offset is beyond object limits')
1098     if src_object:
1099         src_account = request.META.get('HTTP_X_SOURCE_ACCOUNT')
1100         if not src_account:
1101             src_account = request.user_uniq
1102         src_container, src_name = split_container_object_string(src_object)
1103         src_version = request.META.get('HTTP_X_SOURCE_VERSION')
1104         try:
1105             src_size, src_hashmap = request.backend.get_object_hashmap(request.user_uniq,
1106                                         src_account, src_container, src_name, src_version)
1107         except NotAllowedError:
1108             raise Forbidden('Not allowed')
1109         except NameError:
1110             raise ItemNotFound('Source object does not exist')
1111
1112         if length is None:
1113             length = src_size
1114         elif length > src_size:
1115             raise BadRequest('Object length is smaller than range length')
1116     else:
1117         # Require either a Content-Length, or 'chunked' Transfer-Encoding.
1118         content_length = -1
1119         if request.META.get('HTTP_TRANSFER_ENCODING') != 'chunked':
1120             content_length = get_content_length(request)
1121
1122         if length is None:
1123             length = content_length
1124         else:
1125             if content_length == -1:
1126                 # TODO: Get up to length bytes in chunks.
1127                 length = content_length
1128             elif length != content_length:
1129                 raise BadRequest('Content length does not match range length')
1130     if total is not None and (total != size or offset >= size or (length > 0 and offset + length >= size)):
1131         raise RangeNotSatisfiable('Supplied range will change provided object limits')
1132
1133     dest_bytes = request.META.get('HTTP_X_OBJECT_BYTES')
1134     if dest_bytes is not None:
1135         dest_bytes = get_int_parameter(dest_bytes)
1136         if dest_bytes is None:
1137             raise BadRequest('Invalid X-Object-Bytes header')
1138
1139     if src_object:
1140         if offset % request.backend.block_size == 0:
1141             # Update the hashes only.
1142             sbi = 0
1143             while length > 0:
1144                 bi = int(offset / request.backend.block_size)
1145                 bl = min(length, request.backend.block_size)
1146                 if bi < len(hashmap):
1147                     if bl == request.backend.block_size:
1148                         hashmap[bi] = src_hashmap[sbi]
1149                     else:
1150                         data = request.backend.get_block(src_hashmap[sbi])
1151                         hashmap[bi] = request.backend.update_block(hashmap[bi],
1152                                                                 data[:bl], 0)
1153                 else:
1154                     hashmap.append(src_hashmap[sbi])
1155                 offset += bl
1156                 length -= bl
1157                 sbi += 1
1158         else:
1159             data = ''
1160             sbi = 0
1161             while length > 0:
1162                 data += request.backend.get_block(src_hashmap[sbi])
1163                 if length < request.backend.block_size:
1164                     data = data[:length]
1165                 bytes = put_object_block(request, hashmap, data, offset)
1166                 offset += bytes
1167                 data = data[bytes:]
1168                 length -= bytes
1169                 sbi += 1
1170     else:
1171         data = ''
1172         for d in socket_read_iterator(request, length,
1173                                         request.backend.block_size):
1174             # TODO: Raise 408 (Request Timeout) if this takes too long.
1175             # TODO: Raise 499 (Client Disconnect) if a length is defined and we stop before getting this much data.
1176             data += d
1177             bytes = put_object_block(request, hashmap, data, offset)
1178             offset += bytes
1179             data = data[bytes:]
1180         if len(data) > 0:
1181             put_object_block(request, hashmap, data, offset)
1182
1183     if offset > size:
1184         size = offset
1185     if dest_bytes is not None and dest_bytes < size:
1186         size = dest_bytes
1187         hashmap = hashmap[:(int((size - 1) / request.backend.block_size) + 1)]
1188     checksum = hashmap_md5(request.backend, hashmap, size) if UPDATE_MD5 else ''
1189     try:
1190         version_id = request.backend.update_object_hashmap(request.user_uniq,
1191                         v_account, v_container, v_object, size, prev_meta['type'],
1192                         hashmap, checksum, 'pithos', meta, replace, permissions)
1193     except NotAllowedError:
1194         raise Forbidden('Not allowed')
1195     except NameError:
1196         raise ItemNotFound('Container does not exist')
1197     except ValueError:
1198         raise BadRequest('Invalid sharing header')
1199     except QuotaError:
1200         raise RequestEntityTooLarge('Quota exceeded')
1201     if public is not None:
1202         try:
1203             request.backend.update_object_public(request.user_uniq, v_account,
1204                                                 v_container, v_object, public)
1205         except NotAllowedError:
1206             raise Forbidden('Not allowed')
1207         except NameError:
1208             raise ItemNotFound('Object does not exist')
1209
1210     response = HttpResponse(status=204)
1211     response['ETag'] = checksum
1212     response['X-Object-Version'] = version_id
1213     return response
1214
1215 @api_method('DELETE')
1216 def object_delete(request, v_account, v_container, v_object):
1217     # Normal Response Codes: 204
1218     # Error Response Codes: internalServerError (500),
1219     #                       itemNotFound (404),
1220     #                       forbidden (403),
1221     #                       badRequest (400)
1222
1223     until = get_int_parameter(request.GET.get('until'))
1224     try:
1225         request.backend.delete_object(request.user_uniq, v_account, v_container,
1226                                         v_object, until)
1227     except NotAllowedError:
1228         raise Forbidden('Not allowed')
1229     except NameError:
1230         raise ItemNotFound('Object does not exist')
1231     return HttpResponse(status=204)
1232
1233 @api_method()
1234 def method_not_allowed(request):
1235     raise BadRequest('Method not allowed')