Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ daf4fb05

History | View | Annotate | Download (34.1 kB)

1 5635f9ef Antony Chazapis
# Copyright 2011 GRNET S.A. All rights reserved.
2 5635f9ef Antony Chazapis
# 
3 5635f9ef Antony Chazapis
# Redistribution and use in source and binary forms, with or
4 5635f9ef Antony Chazapis
# without modification, are permitted provided that the following
5 5635f9ef Antony Chazapis
# conditions are met:
6 5635f9ef Antony Chazapis
# 
7 5635f9ef Antony Chazapis
#   1. Redistributions of source code must retain the above
8 5635f9ef Antony Chazapis
#      copyright notice, this list of conditions and the following
9 5635f9ef Antony Chazapis
#      disclaimer.
10 5635f9ef Antony Chazapis
# 
11 5635f9ef Antony Chazapis
#   2. Redistributions in binary form must reproduce the above
12 5635f9ef Antony Chazapis
#      copyright notice, this list of conditions and the following
13 5635f9ef Antony Chazapis
#      disclaimer in the documentation and/or other materials
14 5635f9ef Antony Chazapis
#      provided with the distribution.
15 5635f9ef Antony Chazapis
# 
16 5635f9ef Antony Chazapis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 5635f9ef Antony Chazapis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 5635f9ef Antony Chazapis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 5635f9ef Antony Chazapis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 5635f9ef Antony Chazapis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 5635f9ef Antony Chazapis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 5635f9ef Antony Chazapis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 5635f9ef Antony Chazapis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 5635f9ef Antony Chazapis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 5635f9ef Antony Chazapis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 5635f9ef Antony Chazapis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 5635f9ef Antony Chazapis
# POSSIBILITY OF SUCH DAMAGE.
28 5635f9ef Antony Chazapis
# 
29 5635f9ef Antony Chazapis
# The views and conclusions contained in the software and
30 5635f9ef Antony Chazapis
# documentation are those of the authors and should not be
31 5635f9ef Antony Chazapis
# interpreted as representing official policies, either expressed
32 5635f9ef Antony Chazapis
# or implied, of GRNET S.A.
33 5635f9ef Antony Chazapis
34 b956618e Antony Chazapis
from functools import wraps
35 b956618e Antony Chazapis
from time import time
36 b956618e Antony Chazapis
from traceback import format_exc
37 b956618e Antony Chazapis
from wsgiref.handlers import format_date_time
38 0d4ea090 Antony Chazapis
from binascii import hexlify, unhexlify
39 e11087c2 Antony Chazapis
from datetime import datetime, tzinfo, timedelta
40 9fefc052 Antony Chazapis
from urllib import quote, unquote
41 b956618e Antony Chazapis
42 b956618e Antony Chazapis
from django.conf import settings
43 b956618e Antony Chazapis
from django.http import HttpResponse
44 1993fea9 Antony Chazapis
from django.utils import simplejson as json
45 22dab079 Antony Chazapis
from django.utils.http import http_date, parse_etags
46 fb064032 Antony Chazapis
from django.utils.encoding import smart_unicode, smart_str
47 817890f2 Antony Chazapis
from django.core.files.uploadhandler import FileUploadHandler
48 817890f2 Antony Chazapis
from django.core.files.uploadedfile import UploadedFile
49 b956618e Antony Chazapis
50 5a96180b Antony Chazapis
from pithos.lib.compat import parse_http_date_safe, parse_http_date
51 7301127c Antony Chazapis
from pithos.lib.hashmap import HashMap
52 5a96180b Antony Chazapis
53 297513ba Antony Chazapis
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound,
54 5df6c6d1 Antony Chazapis
                                Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge,
55 5df6c6d1 Antony Chazapis
                                RangeNotSatisfiable, ServiceUnavailable)
56 bb4eafc6 Antony Chazapis
from pithos.api.short_url import encode_url
57 39593b2b Giorgos Verigakis
from pithos.backends import connect_backend
58 5df6c6d1 Antony Chazapis
from pithos.backends.base import NotAllowedError, QuotaError
59 b956618e Antony Chazapis
60 b956618e Antony Chazapis
import logging
61 22dab079 Antony Chazapis
import re
62 cbfb6636 Sofia Papagiannaki
import hashlib
63 7bef5750 Antony Chazapis
import uuid
64 c48acbfd Sofia Papagiannaki
import decimal
65 8c793655 Antony Chazapis
66 b956618e Antony Chazapis
67 b956618e Antony Chazapis
logger = logging.getLogger(__name__)
68 b956618e Antony Chazapis
69 b956618e Antony Chazapis
70 e11087c2 Antony Chazapis
class UTC(tzinfo):
71 e11087c2 Antony Chazapis
   def utcoffset(self, dt):
72 e11087c2 Antony Chazapis
       return timedelta(0)
73 e11087c2 Antony Chazapis
74 e11087c2 Antony Chazapis
   def tzname(self, dt):
75 e11087c2 Antony Chazapis
       return 'UTC'
76 e11087c2 Antony Chazapis
77 e11087c2 Antony Chazapis
   def dst(self, dt):
78 e11087c2 Antony Chazapis
       return timedelta(0)
79 e11087c2 Antony Chazapis
80 c48acbfd Sofia Papagiannaki
def json_encode_decimal(obj):
81 c48acbfd Sofia Papagiannaki
    if isinstance(obj, decimal.Decimal):
82 c48acbfd Sofia Papagiannaki
        return str(obj)
83 c48acbfd Sofia Papagiannaki
    raise TypeError(repr(obj) + " is not JSON serializable")
84 c48acbfd Sofia Papagiannaki
85 e11087c2 Antony Chazapis
def isoformat(d):
86 e11087c2 Antony Chazapis
   """Return an ISO8601 date string that includes a timezone."""
87 e11087c2 Antony Chazapis
88 e11087c2 Antony Chazapis
   return d.replace(tzinfo=UTC()).isoformat()
89 e11087c2 Antony Chazapis
90 804e8fe7 Antony Chazapis
def rename_meta_key(d, old, new):
91 804e8fe7 Antony Chazapis
    if old not in d:
92 804e8fe7 Antony Chazapis
        return
93 804e8fe7 Antony Chazapis
    d[new] = d[old]
94 804e8fe7 Antony Chazapis
    del(d[old])
95 804e8fe7 Antony Chazapis
96 02c0c3fa Antony Chazapis
def printable_header_dict(d):
97 b956618e Antony Chazapis
    """Format a meta dictionary for printing out json/xml.
98 b956618e Antony Chazapis
    
99 804e8fe7 Antony Chazapis
    Convert all keys to lower case and replace dashes with underscores.
100 804e8fe7 Antony Chazapis
    Format 'last_modified' timestamp.
101 b956618e Antony Chazapis
    """
102 83dd59c5 Antony Chazapis
    
103 690747fe Antony Chazapis
    if 'last_modified' in d:
104 690747fe Antony Chazapis
        d['last_modified'] = isoformat(datetime.fromtimestamp(d['last_modified']))
105 b956618e Antony Chazapis
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
106 b956618e Antony Chazapis
107 02c0c3fa Antony Chazapis
def format_header_key(k):
108 58a6c894 Antony Chazapis
    """Convert underscores to dashes and capitalize intra-dash strings."""
109 b956618e Antony Chazapis
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
110 b956618e Antony Chazapis
111 02c0c3fa Antony Chazapis
def get_header_prefix(request, prefix):
112 02c0c3fa Antony Chazapis
    """Get all prefix-* request headers in a dict. Reformat keys with format_header_key()."""
113 83dd59c5 Antony Chazapis
    
114 b956618e Antony Chazapis
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
115 3ab38c43 Antony Chazapis
    # TODO: Document or remove '~' replacing.
116 3ab38c43 Antony Chazapis
    return dict([(format_header_key(k[5:]), v.replace('~', '')) for k, v in request.META.iteritems() if k.startswith(prefix) and len(k) > len(prefix)])
117 02c0c3fa Antony Chazapis
118 02c0c3fa Antony Chazapis
def get_account_headers(request):
119 02c0c3fa Antony Chazapis
    meta = get_header_prefix(request, 'X-Account-Meta-')
120 02c0c3fa Antony Chazapis
    groups = {}
121 02c0c3fa Antony Chazapis
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
122 02c0c3fa Antony Chazapis
        n = k[16:].lower()
123 02c0c3fa Antony Chazapis
        if '-' in n or '_' in n:
124 02c0c3fa Antony Chazapis
            raise BadRequest('Bad characters in group name')
125 02c0c3fa Antony Chazapis
        groups[n] = v.replace(' ', '').split(',')
126 40d6b76d Antony Chazapis
        while '' in groups[n]:
127 02c0c3fa Antony Chazapis
            groups[n].remove('')
128 02c0c3fa Antony Chazapis
    return meta, groups
129 02c0c3fa Antony Chazapis
130 647a5f48 Antony Chazapis
def put_account_headers(response, meta, groups, policy):
131 f6c97079 Antony Chazapis
    if 'count' in meta:
132 f6c97079 Antony Chazapis
        response['X-Account-Container-Count'] = meta['count']
133 f6c97079 Antony Chazapis
    if 'bytes' in meta:
134 f6c97079 Antony Chazapis
        response['X-Account-Bytes-Used'] = meta['bytes']
135 d065f612 Antony Chazapis
    response['Last-Modified'] = http_date(int(meta['modified']))
136 b956618e Antony Chazapis
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
137 e7b51248 Sofia Papagiannaki
        response[smart_str(k, strings_only=True)] = smart_str(meta[k], strings_only=True)
138 83dd59c5 Antony Chazapis
    if 'until_timestamp' in meta:
139 83dd59c5 Antony Chazapis
        response['X-Account-Until-Timestamp'] = http_date(int(meta['until_timestamp']))
140 02c0c3fa Antony Chazapis
    for k, v in groups.iteritems():
141 e7b51248 Sofia Papagiannaki
        k = smart_str(k, strings_only=True)
142 e7b51248 Sofia Papagiannaki
        k = format_header_key('X-Account-Group-' + k)
143 e7b51248 Sofia Papagiannaki
        v = smart_str(','.join(v), strings_only=True)
144 e7b51248 Sofia Papagiannaki
        response[k] = v
145 647a5f48 Antony Chazapis
    for k, v in policy.iteritems():
146 647a5f48 Antony Chazapis
        response[smart_str(format_header_key('X-Account-Policy-' + k), strings_only=True)] = smart_str(v, strings_only=True)
147 647a5f48 Antony Chazapis
148 02c0c3fa Antony Chazapis
def get_container_headers(request):
149 02c0c3fa Antony Chazapis
    meta = get_header_prefix(request, 'X-Container-Meta-')
150 3ab38c43 Antony Chazapis
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in get_header_prefix(request, 'X-Container-Policy-').iteritems()])
151 3ab38c43 Antony Chazapis
    return meta, policy
152 b956618e Antony Chazapis
153 39593b2b Giorgos Verigakis
def put_container_headers(request, response, meta, policy):
154 f6c97079 Antony Chazapis
    if 'count' in meta:
155 f6c97079 Antony Chazapis
        response['X-Container-Object-Count'] = meta['count']
156 f6c97079 Antony Chazapis
    if 'bytes' in meta:
157 f6c97079 Antony Chazapis
        response['X-Container-Bytes-Used'] = meta['bytes']
158 58a6c894 Antony Chazapis
    response['Last-Modified'] = http_date(int(meta['modified']))
159 b956618e Antony Chazapis
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
160 e7b51248 Sofia Papagiannaki
        response[smart_str(k, strings_only=True)] = smart_str(meta[k], strings_only=True)
161 e7b51248 Sofia Papagiannaki
    l = [smart_str(x, strings_only=True) for x in meta['object_meta'] if x.startswith('X-Object-Meta-')]
162 e7b51248 Sofia Papagiannaki
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
163 39593b2b Giorgos Verigakis
    response['X-Container-Block-Size'] = request.backend.block_size
164 39593b2b Giorgos Verigakis
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
165 83dd59c5 Antony Chazapis
    if 'until_timestamp' in meta:
166 83dd59c5 Antony Chazapis
        response['X-Container-Until-Timestamp'] = http_date(int(meta['until_timestamp']))
167 3ab38c43 Antony Chazapis
    for k, v in policy.iteritems():
168 e7b51248 Sofia Papagiannaki
        response[smart_str(format_header_key('X-Container-Policy-' + k), strings_only=True)] = smart_str(v, strings_only=True)
169 b956618e Antony Chazapis
170 02c0c3fa Antony Chazapis
def get_object_headers(request):
171 02c0c3fa Antony Chazapis
    meta = get_header_prefix(request, 'X-Object-Meta-')
172 b956618e Antony Chazapis
    if request.META.get('CONTENT_TYPE'):
173 b956618e Antony Chazapis
        meta['Content-Type'] = request.META['CONTENT_TYPE']
174 b956618e Antony Chazapis
    if request.META.get('HTTP_CONTENT_ENCODING'):
175 b956618e Antony Chazapis
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
176 22dab079 Antony Chazapis
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
177 22dab079 Antony Chazapis
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
178 b956618e Antony Chazapis
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
179 b956618e Antony Chazapis
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
180 3ab38c43 Antony Chazapis
    return meta, get_sharing(request), get_public(request)
181 b956618e Antony Chazapis
182 3ab38c43 Antony Chazapis
def put_object_headers(response, meta, restricted=False):
183 702bfe60 Antony Chazapis
    response['ETag'] = meta['ETag'] if 'ETag' in meta else meta['hash']
184 b956618e Antony Chazapis
    response['Content-Length'] = meta['bytes']
185 b956618e Antony Chazapis
    response['Content-Type'] = meta.get('Content-Type', 'application/octet-stream')
186 b956618e Antony Chazapis
    response['Last-Modified'] = http_date(int(meta['modified']))
187 3ab38c43 Antony Chazapis
    if not restricted:
188 4a1c29ea Antony Chazapis
        response['X-Object-Hash'] = meta['hash']
189 37bee317 Antony Chazapis
        response['X-Object-UUID'] = meta['uuid']
190 72c3ba3f Antony Chazapis
        response['X-Object-Modified-By'] = smart_str(meta['modified_by'], strings_only=True)
191 7bef5750 Antony Chazapis
        response['X-Object-Version'] = meta['version']
192 104626e3 Antony Chazapis
        response['X-Object-Version-Timestamp'] = http_date(int(meta['version_timestamp']))
193 7bef5750 Antony Chazapis
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
194 e7b51248 Sofia Papagiannaki
            response[smart_str(k, strings_only=True)] = smart_str(meta[k], strings_only=True)
195 067cf1fc Antony Chazapis
        for k in ('Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
196 067cf1fc Antony Chazapis
                  'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
197 067cf1fc Antony Chazapis
                  'X-Object-Public'):
198 7bef5750 Antony Chazapis
            if k in meta:
199 e7b51248 Sofia Papagiannaki
                response[k] = smart_str(meta[k], strings_only=True)
200 7bef5750 Antony Chazapis
    else:
201 c9af0703 Antony Chazapis
        for k in ('Content-Encoding', 'Content-Disposition'):
202 7bef5750 Antony Chazapis
            if k in meta:
203 e8d003e8 Antony Chazapis
                response[k] = smart_str(meta[k], strings_only=True)
204 b956618e Antony Chazapis
205 8cb45c13 Antony Chazapis
def update_manifest_meta(request, v_account, meta):
206 8cb45c13 Antony Chazapis
    """Update metadata if the object has an X-Object-Manifest."""
207 8cb45c13 Antony Chazapis
    
208 8cb45c13 Antony Chazapis
    if 'X-Object-Manifest' in meta:
209 4a1c29ea Antony Chazapis
        etag = ''
210 8cb45c13 Antony Chazapis
        bytes = 0
211 8cb45c13 Antony Chazapis
        try:
212 6d817842 Antony Chazapis
            src_container, src_name = split_container_object_string('/' + meta['X-Object-Manifest'])
213 61efb530 Antony Chazapis
            objects = request.backend.list_objects(request.user_uniq, v_account,
214 39593b2b Giorgos Verigakis
                                src_container, prefix=src_name, virtual=False)
215 8cb45c13 Antony Chazapis
            for x in objects:
216 61efb530 Antony Chazapis
                src_meta = request.backend.get_object_meta(request.user_uniq,
217 91817b22 chazapis
                                        v_account, src_container, x[0], 'pithos', x[1])
218 91817b22 chazapis
                if 'ETag' in src_meta:
219 91817b22 chazapis
                    etag += src_meta['ETag']
220 8cb45c13 Antony Chazapis
                bytes += src_meta['bytes']
221 8cb45c13 Antony Chazapis
        except:
222 8cb45c13 Antony Chazapis
            # Ignore errors.
223 8cb45c13 Antony Chazapis
            return
224 8cb45c13 Antony Chazapis
        meta['bytes'] = bytes
225 8cb45c13 Antony Chazapis
        md5 = hashlib.md5()
226 4a1c29ea Antony Chazapis
        md5.update(etag)
227 4a1c29ea Antony Chazapis
        meta['ETag'] = md5.hexdigest().lower()
228 8cb45c13 Antony Chazapis
229 067cf1fc Antony Chazapis
def update_sharing_meta(request, permissions, v_account, v_container, v_object, meta):
230 cca6c617 Antony Chazapis
    if permissions is None:
231 cca6c617 Antony Chazapis
        return
232 067cf1fc Antony Chazapis
    allowed, perm_path, perms = permissions
233 cca6c617 Antony Chazapis
    if len(perms) == 0:
234 cca6c617 Antony Chazapis
        return
235 3436eeb0 Antony Chazapis
    ret = []
236 cca6c617 Antony Chazapis
    r = ','.join(perms.get('read', []))
237 3436eeb0 Antony Chazapis
    if r:
238 3436eeb0 Antony Chazapis
        ret.append('read=' + r)
239 cca6c617 Antony Chazapis
    w = ','.join(perms.get('write', []))
240 3436eeb0 Antony Chazapis
    if w:
241 3436eeb0 Antony Chazapis
        ret.append('write=' + w)
242 cca6c617 Antony Chazapis
    meta['X-Object-Sharing'] = '; '.join(ret)
243 cca6c617 Antony Chazapis
    if '/'.join((v_account, v_container, v_object)) != perm_path:
244 cca6c617 Antony Chazapis
        meta['X-Object-Shared-By'] = perm_path
245 61efb530 Antony Chazapis
    if request.user_uniq != v_account:
246 067cf1fc Antony Chazapis
        meta['X-Object-Allowed-To'] = allowed
247 3436eeb0 Antony Chazapis
248 e0f916bb Antony Chazapis
def update_public_meta(public, meta):
249 e0f916bb Antony Chazapis
    if not public:
250 e0f916bb Antony Chazapis
        return
251 bb4eafc6 Antony Chazapis
    meta['X-Object-Public'] = '/public/' + encode_url(public)
252 e0f916bb Antony Chazapis
253 b956618e Antony Chazapis
def validate_modification_preconditions(request, meta):
254 58a6c894 Antony Chazapis
    """Check that the modified timestamp conforms with the preconditions set."""
255 83dd59c5 Antony Chazapis
    
256 22dab079 Antony Chazapis
    if 'modified' not in meta:
257 22dab079 Antony Chazapis
        return # TODO: Always return?
258 22dab079 Antony Chazapis
    
259 b956618e Antony Chazapis
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
260 b956618e Antony Chazapis
    if if_modified_since is not None:
261 b956618e Antony Chazapis
        if_modified_since = parse_http_date_safe(if_modified_since)
262 22dab079 Antony Chazapis
    if if_modified_since is not None and int(meta['modified']) <= if_modified_since:
263 3e5d2f38 Antony Chazapis
        raise NotModified('Resource has not been modified')
264 b956618e Antony Chazapis
    
265 b956618e Antony Chazapis
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
266 b956618e Antony Chazapis
    if if_unmodified_since is not None:
267 b956618e Antony Chazapis
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
268 22dab079 Antony Chazapis
    if if_unmodified_since is not None and int(meta['modified']) > if_unmodified_since:
269 3e5d2f38 Antony Chazapis
        raise PreconditionFailed('Resource has been modified')
270 b956618e Antony Chazapis
271 22dab079 Antony Chazapis
def validate_matching_preconditions(request, meta):
272 58a6c894 Antony Chazapis
    """Check that the ETag conforms with the preconditions set."""
273 83dd59c5 Antony Chazapis
    
274 4a1c29ea Antony Chazapis
    etag = meta.get('ETag', None)
275 22dab079 Antony Chazapis
    
276 22dab079 Antony Chazapis
    if_match = request.META.get('HTTP_IF_MATCH')
277 a8326bef Antony Chazapis
    if if_match is not None:
278 4a1c29ea Antony Chazapis
        if etag is None:
279 a8326bef Antony Chazapis
            raise PreconditionFailed('Resource does not exist')
280 4a1c29ea Antony Chazapis
        if if_match != '*' and etag not in [x.lower() for x in parse_etags(if_match)]:
281 a8326bef Antony Chazapis
            raise PreconditionFailed('Resource ETag does not match')
282 22dab079 Antony Chazapis
    
283 22dab079 Antony Chazapis
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
284 22dab079 Antony Chazapis
    if if_none_match is not None:
285 a8326bef Antony Chazapis
        # TODO: If this passes, must ignore If-Modified-Since header.
286 4a1c29ea Antony Chazapis
        if etag is not None:
287 4a1c29ea Antony Chazapis
            if if_none_match == '*' or etag in [x.lower() for x in parse_etags(if_none_match)]:
288 a8326bef Antony Chazapis
                # TODO: Continue if an If-Modified-Since header is present.
289 a8326bef Antony Chazapis
                if request.method in ('HEAD', 'GET'):
290 a8326bef Antony Chazapis
                    raise NotModified('Resource ETag matches')
291 a8326bef Antony Chazapis
                raise PreconditionFailed('Resource exists or ETag matches')
292 22dab079 Antony Chazapis
293 83dd59c5 Antony Chazapis
def split_container_object_string(s):
294 6d817842 Antony Chazapis
    if not len(s) > 0 or s[0] != '/':
295 6d817842 Antony Chazapis
        raise ValueError
296 6d817842 Antony Chazapis
    s = s[1:]
297 8cb45c13 Antony Chazapis
    pos = s.find('/')
298 22d7b01e Antony Chazapis
    if pos == -1 or pos == len(s) - 1:
299 83dd59c5 Antony Chazapis
        raise ValueError
300 8cb45c13 Antony Chazapis
    return s[:pos], s[(pos + 1):]
301 83dd59c5 Antony Chazapis
302 79bb41b7 Antony Chazapis
def copy_or_move_object(request, src_account, src_container, src_name, dest_account, dest_container, dest_name, move=False):
303 58a6c894 Antony Chazapis
    """Copy or move an object."""
304 22dab079 Antony Chazapis
    
305 3ab38c43 Antony Chazapis
    meta, permissions, public = get_object_headers(request)
306 79bb41b7 Antony Chazapis
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
307 b956618e Antony Chazapis
    try:
308 b956618e Antony Chazapis
        if move:
309 61efb530 Antony Chazapis
            version_id = request.backend.move_object(request.user_uniq, src_account, src_container, src_name,
310 79bb41b7 Antony Chazapis
                                                        dest_account, dest_container, dest_name,
311 808cea65 Antony Chazapis
                                                        'pithos', meta, False, permissions)
312 b956618e Antony Chazapis
        else:
313 61efb530 Antony Chazapis
            version_id = request.backend.copy_object(request.user_uniq, src_account, src_container, src_name,
314 79bb41b7 Antony Chazapis
                                                        dest_account, dest_container, dest_name,
315 808cea65 Antony Chazapis
                                                        'pithos', meta, False, permissions, src_version)
316 cca6c617 Antony Chazapis
    except NotAllowedError:
317 297513ba Antony Chazapis
        raise Forbidden('Not allowed')
318 a8326bef Antony Chazapis
    except (NameError, IndexError):
319 b956618e Antony Chazapis
        raise ItemNotFound('Container or object does not exist')
320 3436eeb0 Antony Chazapis
    except ValueError:
321 3436eeb0 Antony Chazapis
        raise BadRequest('Invalid sharing header')
322 1993fea9 Antony Chazapis
    except AttributeError, e:
323 a33dfe54 Antony Chazapis
        raise Conflict('\n'.join(e.data) + '\n')
324 5df6c6d1 Antony Chazapis
    except QuotaError:
325 5df6c6d1 Antony Chazapis
        raise RequestEntityTooLarge('Quota exceeded')
326 e0f916bb Antony Chazapis
    if public is not None:
327 e0f916bb Antony Chazapis
        try:
328 61efb530 Antony Chazapis
            request.backend.update_object_public(request.user_uniq, dest_account, dest_container, dest_name, public)
329 e0f916bb Antony Chazapis
        except NotAllowedError:
330 297513ba Antony Chazapis
            raise Forbidden('Not allowed')
331 e0f916bb Antony Chazapis
        except NameError:
332 e0f916bb Antony Chazapis
            raise ItemNotFound('Object does not exist')
333 7dd293a0 Antony Chazapis
    return version_id
334 b956618e Antony Chazapis
335 1495b972 Antony Chazapis
def get_int_parameter(p):
336 83dd59c5 Antony Chazapis
    if p is not None:
337 58a6c894 Antony Chazapis
        try:
338 83dd59c5 Antony Chazapis
            p = int(p)
339 58a6c894 Antony Chazapis
        except ValueError:
340 58a6c894 Antony Chazapis
            return None
341 83dd59c5 Antony Chazapis
        if p < 0:
342 58a6c894 Antony Chazapis
            return None
343 83dd59c5 Antony Chazapis
    return p
344 58a6c894 Antony Chazapis
345 22dab079 Antony Chazapis
def get_content_length(request):
346 1495b972 Antony Chazapis
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
347 1495b972 Antony Chazapis
    if content_length is None:
348 1495b972 Antony Chazapis
        raise LengthRequired('Missing or invalid Content-Length header')
349 22dab079 Antony Chazapis
    return content_length
350 22dab079 Antony Chazapis
351 22dab079 Antony Chazapis
def get_range(request, size):
352 58a6c894 Antony Chazapis
    """Parse a Range header from the request.
353 b956618e Antony Chazapis
    
354 22dab079 Antony Chazapis
    Either returns None, when the header is not existent or should be ignored,
355 22dab079 Antony Chazapis
    or a list of (offset, length) tuples - should be further checked.
356 b956618e Antony Chazapis
    """
357 83dd59c5 Antony Chazapis
    
358 22dab079 Antony Chazapis
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
359 22dab079 Antony Chazapis
    if not ranges.startswith('bytes='):
360 b956618e Antony Chazapis
        return None
361 b956618e Antony Chazapis
    
362 22dab079 Antony Chazapis
    ret = []
363 22dab079 Antony Chazapis
    for r in (x.strip() for x in ranges[6:].split(',')):
364 22dab079 Antony Chazapis
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
365 22dab079 Antony Chazapis
        m = p.match(r)
366 22dab079 Antony Chazapis
        if not m:
367 22dab079 Antony Chazapis
            return None
368 22dab079 Antony Chazapis
        offset = m.group('offset')
369 22dab079 Antony Chazapis
        upto = m.group('upto')
370 22dab079 Antony Chazapis
        if offset == '' and upto == '':
371 b956618e Antony Chazapis
            return None
372 b956618e Antony Chazapis
        
373 22dab079 Antony Chazapis
        if offset != '':
374 22dab079 Antony Chazapis
            offset = int(offset)
375 22dab079 Antony Chazapis
            if upto != '':
376 b956618e Antony Chazapis
                upto = int(upto)
377 22dab079 Antony Chazapis
                if offset > upto:
378 22dab079 Antony Chazapis
                    return None
379 22dab079 Antony Chazapis
                ret.append((offset, upto - offset + 1))
380 22dab079 Antony Chazapis
            else:
381 22dab079 Antony Chazapis
                ret.append((offset, size - offset))
382 b956618e Antony Chazapis
        else:
383 22dab079 Antony Chazapis
            length = int(upto)
384 22dab079 Antony Chazapis
            ret.append((size - length, length))
385 22dab079 Antony Chazapis
    
386 22dab079 Antony Chazapis
    return ret
387 22dab079 Antony Chazapis
388 22dab079 Antony Chazapis
def get_content_range(request):
389 58a6c894 Antony Chazapis
    """Parse a Content-Range header from the request.
390 22dab079 Antony Chazapis
    
391 22dab079 Antony Chazapis
    Either returns None, when the header is not existent or should be ignored,
392 22dab079 Antony Chazapis
    or an (offset, length, total) tuple - check as length, total may be None.
393 22dab079 Antony Chazapis
    Returns (None, None, None) if the provided range is '*/*'.
394 22dab079 Antony Chazapis
    """
395 22dab079 Antony Chazapis
    
396 22dab079 Antony Chazapis
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
397 22dab079 Antony Chazapis
    if not ranges:
398 22dab079 Antony Chazapis
        return None
399 22dab079 Antony Chazapis
    
400 22dab079 Antony Chazapis
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
401 22dab079 Antony Chazapis
    m = p.match(ranges)
402 22dab079 Antony Chazapis
    if not m:
403 22dab079 Antony Chazapis
        if ranges == 'bytes */*':
404 22dab079 Antony Chazapis
            return (None, None, None)
405 22dab079 Antony Chazapis
        return None
406 22dab079 Antony Chazapis
    offset = int(m.group('offset'))
407 22dab079 Antony Chazapis
    upto = m.group('upto')
408 22dab079 Antony Chazapis
    total = m.group('total')
409 22dab079 Antony Chazapis
    if upto != '':
410 22dab079 Antony Chazapis
        upto = int(upto)
411 b956618e Antony Chazapis
    else:
412 22dab079 Antony Chazapis
        upto = None
413 22dab079 Antony Chazapis
    if total != '*':
414 22dab079 Antony Chazapis
        total = int(total)
415 22dab079 Antony Chazapis
    else:
416 22dab079 Antony Chazapis
        total = None
417 70e526a0 Antony Chazapis
    if (upto is not None and offset > upto) or \
418 70e526a0 Antony Chazapis
        (total is not None and offset >= total) or \
419 70e526a0 Antony Chazapis
        (total is not None and upto is not None and upto >= total):
420 22dab079 Antony Chazapis
        return None
421 22dab079 Antony Chazapis
    
422 70e526a0 Antony Chazapis
    if upto is None:
423 22dab079 Antony Chazapis
        length = None
424 22dab079 Antony Chazapis
    else:
425 22dab079 Antony Chazapis
        length = upto - offset + 1
426 22dab079 Antony Chazapis
    return (offset, length, total)
427 b956618e Antony Chazapis
428 3436eeb0 Antony Chazapis
def get_sharing(request):
429 3436eeb0 Antony Chazapis
    """Parse an X-Object-Sharing header from the request.
430 3436eeb0 Antony Chazapis
    
431 3436eeb0 Antony Chazapis
    Raises BadRequest on error.
432 3436eeb0 Antony Chazapis
    """
433 3436eeb0 Antony Chazapis
    
434 3436eeb0 Antony Chazapis
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
435 cca6c617 Antony Chazapis
    if permissions is None:
436 3436eeb0 Antony Chazapis
        return None
437 3436eeb0 Antony Chazapis
    
438 0a7f1671 Antony Chazapis
    # TODO: Document or remove '~' replacing.
439 0a7f1671 Antony Chazapis
    permissions = permissions.replace('~', '')
440 0a7f1671 Antony Chazapis
    
441 3436eeb0 Antony Chazapis
    ret = {}
442 cca6c617 Antony Chazapis
    permissions = permissions.replace(' ', '')
443 cca6c617 Antony Chazapis
    if permissions == '':
444 cca6c617 Antony Chazapis
        return ret
445 cca6c617 Antony Chazapis
    for perm in (x for x in permissions.split(';')):
446 cca6c617 Antony Chazapis
        if perm.startswith('read='):
447 b0a2d1a6 Antony Chazapis
            ret['read'] = list(set([v.replace(' ','').lower() for v in perm[5:].split(',')]))
448 02c0c3fa Antony Chazapis
            if '' in ret['read']:
449 02c0c3fa Antony Chazapis
                ret['read'].remove('')
450 e8886082 Antony Chazapis
            if '*' in ret['read']:
451 e8886082 Antony Chazapis
                ret['read'] = ['*']
452 3436eeb0 Antony Chazapis
            if len(ret['read']) == 0:
453 3436eeb0 Antony Chazapis
                raise BadRequest('Bad X-Object-Sharing header value')
454 3436eeb0 Antony Chazapis
        elif perm.startswith('write='):
455 b0a2d1a6 Antony Chazapis
            ret['write'] = list(set([v.replace(' ','').lower() for v in perm[6:].split(',')]))
456 02c0c3fa Antony Chazapis
            if '' in ret['write']:
457 02c0c3fa Antony Chazapis
                ret['write'].remove('')
458 e8886082 Antony Chazapis
            if '*' in ret['write']:
459 e8886082 Antony Chazapis
                ret['write'] = ['*']
460 3436eeb0 Antony Chazapis
            if len(ret['write']) == 0:
461 3436eeb0 Antony Chazapis
                raise BadRequest('Bad X-Object-Sharing header value')
462 3436eeb0 Antony Chazapis
        else:
463 3436eeb0 Antony Chazapis
            raise BadRequest('Bad X-Object-Sharing header value')
464 21a8a6ff Antony Chazapis
    
465 21a8a6ff Antony Chazapis
    # Keep duplicates only in write list.
466 21a8a6ff Antony Chazapis
    dups = [x for x in ret.get('read', []) if x in ret.get('write', []) and x != '*']
467 21a8a6ff Antony Chazapis
    if dups:
468 21a8a6ff Antony Chazapis
        for x in dups:
469 21a8a6ff Antony Chazapis
            ret['read'].remove(x)
470 21a8a6ff Antony Chazapis
        if len(ret['read']) == 0:
471 21a8a6ff Antony Chazapis
            del(ret['read'])
472 21a8a6ff Antony Chazapis
    
473 3436eeb0 Antony Chazapis
    return ret
474 3436eeb0 Antony Chazapis
475 3ab38c43 Antony Chazapis
def get_public(request):
476 3ab38c43 Antony Chazapis
    """Parse an X-Object-Public header from the request.
477 3ab38c43 Antony Chazapis
    
478 3ab38c43 Antony Chazapis
    Raises BadRequest on error.
479 3ab38c43 Antony Chazapis
    """
480 3ab38c43 Antony Chazapis
    
481 3ab38c43 Antony Chazapis
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
482 3ab38c43 Antony Chazapis
    if public is None:
483 3ab38c43 Antony Chazapis
        return None
484 3ab38c43 Antony Chazapis
    
485 3ab38c43 Antony Chazapis
    public = public.replace(' ', '').lower()
486 3ab38c43 Antony Chazapis
    if public == 'true':
487 3ab38c43 Antony Chazapis
        return True
488 3ab38c43 Antony Chazapis
    elif public == 'false' or public == '':
489 3ab38c43 Antony Chazapis
        return False
490 3ab38c43 Antony Chazapis
    raise BadRequest('Bad X-Object-Public header value')
491 3ab38c43 Antony Chazapis
492 b956618e Antony Chazapis
def raw_input_socket(request):
493 58a6c894 Antony Chazapis
    """Return the socket for reading the rest of the request."""
494 83dd59c5 Antony Chazapis
    
495 b956618e Antony Chazapis
    server_software = request.META.get('SERVER_SOFTWARE')
496 fc1b2a75 Antony Chazapis
    if server_software and server_software.startswith('mod_python'):
497 b956618e Antony Chazapis
        return request._req
498 fc1b2a75 Antony Chazapis
    if 'wsgi.input' in request.environ:
499 fc1b2a75 Antony Chazapis
        return request.environ['wsgi.input']
500 b956618e Antony Chazapis
    raise ServiceUnavailable('Unknown server software')
501 b956618e Antony Chazapis
502 07fea84b Antony Chazapis
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024) # 5GB
503 b956618e Antony Chazapis
504 c032f34d Antony Chazapis
def socket_read_iterator(request, length=0, blocksize=4096):
505 58a6c894 Antony Chazapis
    """Return a maximum of blocksize data read from the socket in each iteration.
506 b956618e Antony Chazapis
    
507 22dab079 Antony Chazapis
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
508 b956618e Antony Chazapis
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
509 b956618e Antony Chazapis
    """
510 83dd59c5 Antony Chazapis
    
511 c032f34d Antony Chazapis
    sock = raw_input_socket(request)
512 b956618e Antony Chazapis
    if length < 0: # Chunked transfers
513 c032f34d Antony Chazapis
        # Small version (server does the dechunking).
514 032dc768 Antony Chazapis
        if request.environ.get('mod_wsgi.input_chunked', None) or request.META['SERVER_SOFTWARE'].startswith('gunicorn'):
515 c032f34d Antony Chazapis
            while length < MAX_UPLOAD_SIZE:
516 c032f34d Antony Chazapis
                data = sock.read(blocksize)
517 c032f34d Antony Chazapis
                if data == '':
518 c032f34d Antony Chazapis
                    return
519 c032f34d Antony Chazapis
                yield data
520 c032f34d Antony Chazapis
            raise BadRequest('Maximum size is reached')
521 c032f34d Antony Chazapis
        
522 c032f34d Antony Chazapis
        # Long version (do the dechunking).
523 22dab079 Antony Chazapis
        data = ''
524 b956618e Antony Chazapis
        while length < MAX_UPLOAD_SIZE:
525 22dab079 Antony Chazapis
            # Get chunk size.
526 22dab079 Antony Chazapis
            if hasattr(sock, 'readline'):
527 22dab079 Antony Chazapis
                chunk_length = sock.readline()
528 22dab079 Antony Chazapis
            else:
529 22dab079 Antony Chazapis
                chunk_length = ''
530 22dab079 Antony Chazapis
                while chunk_length[-1:] != '\n':
531 22dab079 Antony Chazapis
                    chunk_length += sock.read(1)
532 22dab079 Antony Chazapis
                chunk_length.strip()
533 b956618e Antony Chazapis
            pos = chunk_length.find(';')
534 b956618e Antony Chazapis
            if pos >= 0:
535 b956618e Antony Chazapis
                chunk_length = chunk_length[:pos]
536 b956618e Antony Chazapis
            try:
537 b956618e Antony Chazapis
                chunk_length = int(chunk_length, 16)
538 b956618e Antony Chazapis
            except Exception, e:
539 b956618e Antony Chazapis
                raise BadRequest('Bad chunk size') # TODO: Change to something more appropriate.
540 22dab079 Antony Chazapis
            # Check if done.
541 b956618e Antony Chazapis
            if chunk_length == 0:
542 22dab079 Antony Chazapis
                if len(data) > 0:
543 22dab079 Antony Chazapis
                    yield data
544 b956618e Antony Chazapis
                return
545 22dab079 Antony Chazapis
            # Get the actual data.
546 b956618e Antony Chazapis
            while chunk_length > 0:
547 22dab079 Antony Chazapis
                chunk = sock.read(min(chunk_length, blocksize))
548 22dab079 Antony Chazapis
                chunk_length -= len(chunk)
549 623a0cf4 Sofia Papagiannaki
                if length > 0:
550 623a0cf4 Sofia Papagiannaki
                    length += len(chunk)
551 22dab079 Antony Chazapis
                data += chunk
552 22dab079 Antony Chazapis
                if len(data) >= blocksize:
553 22dab079 Antony Chazapis
                    ret = data[:blocksize]
554 22dab079 Antony Chazapis
                    data = data[blocksize:]
555 22dab079 Antony Chazapis
                    yield ret
556 22dab079 Antony Chazapis
            sock.read(2) # CRLF
557 32a437b1 Sofia Papagiannaki
        raise BadRequest('Maximum size is reached')
558 b956618e Antony Chazapis
    else:
559 b956618e Antony Chazapis
        if length > MAX_UPLOAD_SIZE:
560 32a437b1 Sofia Papagiannaki
            raise BadRequest('Maximum size is reached')
561 b956618e Antony Chazapis
        while length > 0:
562 b956618e Antony Chazapis
            data = sock.read(min(length, blocksize))
563 7b25e082 Antony Chazapis
            if not data:
564 7b25e082 Antony Chazapis
                raise BadRequest()
565 b956618e Antony Chazapis
            length -= len(data)
566 b956618e Antony Chazapis
            yield data
567 b956618e Antony Chazapis
568 817890f2 Antony Chazapis
class SaveToBackendHandler(FileUploadHandler):
569 817890f2 Antony Chazapis
    """Handle a file from an HTML form the django way."""
570 817890f2 Antony Chazapis
    
571 817890f2 Antony Chazapis
    def __init__(self, request=None):
572 817890f2 Antony Chazapis
        super(SaveToBackendHandler, self).__init__(request)
573 817890f2 Antony Chazapis
        self.backend = request.backend
574 817890f2 Antony Chazapis
    
575 817890f2 Antony Chazapis
    def put_data(self, length):
576 817890f2 Antony Chazapis
        if len(self.data) >= length:
577 817890f2 Antony Chazapis
            block = self.data[:length]
578 817890f2 Antony Chazapis
            self.file.hashmap.append(self.backend.put_block(block))
579 817890f2 Antony Chazapis
            self.md5.update(block)
580 817890f2 Antony Chazapis
            self.data = self.data[length:]
581 817890f2 Antony Chazapis
    
582 817890f2 Antony Chazapis
    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
583 817890f2 Antony Chazapis
        self.md5 = hashlib.md5()        
584 817890f2 Antony Chazapis
        self.data = ''
585 817890f2 Antony Chazapis
        self.file = UploadedFile(name=file_name, content_type=content_type, charset=charset)
586 817890f2 Antony Chazapis
        self.file.size = 0
587 817890f2 Antony Chazapis
        self.file.hashmap = []
588 817890f2 Antony Chazapis
    
589 817890f2 Antony Chazapis
    def receive_data_chunk(self, raw_data, start):
590 817890f2 Antony Chazapis
        self.data += raw_data
591 817890f2 Antony Chazapis
        self.file.size += len(raw_data)
592 817890f2 Antony Chazapis
        self.put_data(self.request.backend.block_size)
593 817890f2 Antony Chazapis
        return None
594 817890f2 Antony Chazapis
    
595 817890f2 Antony Chazapis
    def file_complete(self, file_size):
596 817890f2 Antony Chazapis
        l = len(self.data)
597 817890f2 Antony Chazapis
        if l > 0:
598 817890f2 Antony Chazapis
            self.put_data(l)
599 817890f2 Antony Chazapis
        self.file.etag = self.md5.hexdigest().lower()
600 817890f2 Antony Chazapis
        return self.file
601 817890f2 Antony Chazapis
602 22dab079 Antony Chazapis
class ObjectWrapper(object):
603 58a6c894 Antony Chazapis
    """Return the object's data block-per-block in each iteration.
604 22dab079 Antony Chazapis
    
605 22dab079 Antony Chazapis
    Read from the object using the offset and length provided in each entry of the range list.
606 22dab079 Antony Chazapis
    """
607 22dab079 Antony Chazapis
    
608 39593b2b Giorgos Verigakis
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
609 39593b2b Giorgos Verigakis
        self.backend = backend
610 22dab079 Antony Chazapis
        self.ranges = ranges
611 8cb45c13 Antony Chazapis
        self.sizes = sizes
612 8cb45c13 Antony Chazapis
        self.hashmaps = hashmaps
613 22dab079 Antony Chazapis
        self.boundary = boundary
614 8cb45c13 Antony Chazapis
        self.size = sum(self.sizes)
615 22dab079 Antony Chazapis
        
616 8cb45c13 Antony Chazapis
        self.file_index = 0
617 8cb45c13 Antony Chazapis
        self.block_index = 0
618 8cb45c13 Antony Chazapis
        self.block_hash = -1
619 22dab079 Antony Chazapis
        self.block = ''
620 22dab079 Antony Chazapis
        
621 22dab079 Antony Chazapis
        self.range_index = -1
622 22dab079 Antony Chazapis
        self.offset, self.length = self.ranges[0]
623 22dab079 Antony Chazapis
    
624 22dab079 Antony Chazapis
    def __iter__(self):
625 22dab079 Antony Chazapis
        return self
626 22dab079 Antony Chazapis
    
627 22dab079 Antony Chazapis
    def part_iterator(self):
628 22dab079 Antony Chazapis
        if self.length > 0:
629 8cb45c13 Antony Chazapis
            # Get the file for the current offset.
630 8cb45c13 Antony Chazapis
            file_size = self.sizes[self.file_index]
631 8cb45c13 Antony Chazapis
            while self.offset >= file_size:
632 8cb45c13 Antony Chazapis
                self.offset -= file_size
633 8cb45c13 Antony Chazapis
                self.file_index += 1
634 8cb45c13 Antony Chazapis
                file_size = self.sizes[self.file_index]
635 8cb45c13 Antony Chazapis
            
636 8cb45c13 Antony Chazapis
            # Get the block for the current position.
637 39593b2b Giorgos Verigakis
            self.block_index = int(self.offset / self.backend.block_size)
638 8cb45c13 Antony Chazapis
            if self.block_hash != self.hashmaps[self.file_index][self.block_index]:
639 8cb45c13 Antony Chazapis
                self.block_hash = self.hashmaps[self.file_index][self.block_index]
640 22dab079 Antony Chazapis
                try:
641 39593b2b Giorgos Verigakis
                    self.block = self.backend.get_block(self.block_hash)
642 22dab079 Antony Chazapis
                except NameError:
643 22dab079 Antony Chazapis
                    raise ItemNotFound('Block does not exist')
644 8cb45c13 Antony Chazapis
            
645 22dab079 Antony Chazapis
            # Get the data from the block.
646 39593b2b Giorgos Verigakis
            bo = self.offset % self.backend.block_size
647 8cb45c13 Antony Chazapis
            bl = min(self.length, len(self.block) - bo)
648 22dab079 Antony Chazapis
            data = self.block[bo:bo + bl]
649 22dab079 Antony Chazapis
            self.offset += bl
650 22dab079 Antony Chazapis
            self.length -= bl
651 22dab079 Antony Chazapis
            return data
652 22dab079 Antony Chazapis
        else:
653 22dab079 Antony Chazapis
            raise StopIteration
654 22dab079 Antony Chazapis
    
655 22dab079 Antony Chazapis
    def next(self):
656 22dab079 Antony Chazapis
        if len(self.ranges) == 1:
657 22dab079 Antony Chazapis
            return self.part_iterator()
658 22dab079 Antony Chazapis
        if self.range_index == len(self.ranges):
659 22dab079 Antony Chazapis
            raise StopIteration
660 22dab079 Antony Chazapis
        try:
661 22dab079 Antony Chazapis
            if self.range_index == -1:
662 22dab079 Antony Chazapis
                raise StopIteration
663 22dab079 Antony Chazapis
            return self.part_iterator()
664 22dab079 Antony Chazapis
        except StopIteration:
665 22dab079 Antony Chazapis
            self.range_index += 1
666 22dab079 Antony Chazapis
            out = []
667 22dab079 Antony Chazapis
            if self.range_index < len(self.ranges):
668 22dab079 Antony Chazapis
                # Part header.
669 22dab079 Antony Chazapis
                self.offset, self.length = self.ranges[self.range_index]
670 8cb45c13 Antony Chazapis
                self.file_index = 0
671 22dab079 Antony Chazapis
                if self.range_index > 0:
672 22dab079 Antony Chazapis
                    out.append('')
673 22dab079 Antony Chazapis
                out.append('--' + self.boundary)
674 22dab079 Antony Chazapis
                out.append('Content-Range: bytes %d-%d/%d' % (self.offset, self.offset + self.length - 1, self.size))
675 22dab079 Antony Chazapis
                out.append('Content-Transfer-Encoding: binary')
676 22dab079 Antony Chazapis
                out.append('')
677 22dab079 Antony Chazapis
                out.append('')
678 22dab079 Antony Chazapis
                return '\r\n'.join(out)
679 22dab079 Antony Chazapis
            else:
680 22dab079 Antony Chazapis
                # Footer.
681 22dab079 Antony Chazapis
                out.append('')
682 22dab079 Antony Chazapis
                out.append('--' + self.boundary + '--')
683 22dab079 Antony Chazapis
                out.append('')
684 22dab079 Antony Chazapis
                return '\r\n'.join(out)
685 22dab079 Antony Chazapis
686 8cb45c13 Antony Chazapis
def object_data_response(request, sizes, hashmaps, meta, public=False):
687 7bef5750 Antony Chazapis
    """Get the HttpResponse object for replying with the object's data."""
688 7bef5750 Antony Chazapis
    
689 7bef5750 Antony Chazapis
    # Range handling.
690 8cb45c13 Antony Chazapis
    size = sum(sizes)
691 7bef5750 Antony Chazapis
    ranges = get_range(request, size)
692 7bef5750 Antony Chazapis
    if ranges is None:
693 7bef5750 Antony Chazapis
        ranges = [(0, size)]
694 7bef5750 Antony Chazapis
        ret = 200
695 7bef5750 Antony Chazapis
    else:
696 7bef5750 Antony Chazapis
        check = [True for offset, length in ranges if
697 7bef5750 Antony Chazapis
                    length <= 0 or length > size or
698 7bef5750 Antony Chazapis
                    offset < 0 or offset >= size or
699 7bef5750 Antony Chazapis
                    offset + length > size]
700 7bef5750 Antony Chazapis
        if len(check) > 0:
701 6d1e6dce Sofia Papagiannaki
            raise RangeNotSatisfiable('Requested range exceeds object limits')
702 7bef5750 Antony Chazapis
        ret = 206
703 15d465b8 Antony Chazapis
        if_range = request.META.get('HTTP_IF_RANGE')
704 15d465b8 Antony Chazapis
        if if_range:
705 6d1e6dce Sofia Papagiannaki
            try:
706 15d465b8 Antony Chazapis
                # Modification time has passed instead.
707 6d1e6dce Sofia Papagiannaki
                last_modified = parse_http_date(if_range)
708 6d1e6dce Sofia Papagiannaki
                if last_modified != meta['modified']:
709 6d1e6dce Sofia Papagiannaki
                    ranges = [(0, size)]
710 6d1e6dce Sofia Papagiannaki
                    ret = 200
711 6d1e6dce Sofia Papagiannaki
            except ValueError:
712 4a1c29ea Antony Chazapis
                if if_range != meta['ETag']:
713 6d1e6dce Sofia Papagiannaki
                    ranges = [(0, size)]
714 6d1e6dce Sofia Papagiannaki
                    ret = 200
715 7bef5750 Antony Chazapis
    
716 7bef5750 Antony Chazapis
    if ret == 206 and len(ranges) > 1:
717 7bef5750 Antony Chazapis
        boundary = uuid.uuid4().hex
718 7bef5750 Antony Chazapis
    else:
719 7bef5750 Antony Chazapis
        boundary = ''
720 39593b2b Giorgos Verigakis
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
721 7bef5750 Antony Chazapis
    response = HttpResponse(wrapper, status=ret)
722 02c0c3fa Antony Chazapis
    put_object_headers(response, meta, public)
723 7bef5750 Antony Chazapis
    if ret == 206:
724 7bef5750 Antony Chazapis
        if len(ranges) == 1:
725 7bef5750 Antony Chazapis
            offset, length = ranges[0]
726 7bef5750 Antony Chazapis
            response['Content-Length'] = length # Update with the correct length.
727 7bef5750 Antony Chazapis
            response['Content-Range'] = 'bytes %d-%d/%d' % (offset, offset + length - 1, size)
728 7bef5750 Antony Chazapis
        else:
729 7bef5750 Antony Chazapis
            del(response['Content-Length'])
730 7bef5750 Antony Chazapis
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (boundary,)
731 7bef5750 Antony Chazapis
    return response
732 7bef5750 Antony Chazapis
733 39593b2b Giorgos Verigakis
def put_object_block(request, hashmap, data, offset):
734 cb146cf9 Antony Chazapis
    """Put one block of data at the given offset."""
735 cb146cf9 Antony Chazapis
    
736 39593b2b Giorgos Verigakis
    bi = int(offset / request.backend.block_size)
737 39593b2b Giorgos Verigakis
    bo = offset % request.backend.block_size
738 39593b2b Giorgos Verigakis
    bl = min(len(data), request.backend.block_size - bo)
739 cb146cf9 Antony Chazapis
    if bi < len(hashmap):
740 39593b2b Giorgos Verigakis
        hashmap[bi] = request.backend.update_block(hashmap[bi], data[:bl], bo)
741 cb146cf9 Antony Chazapis
    else:
742 39593b2b Giorgos Verigakis
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
743 cb146cf9 Antony Chazapis
    return bl # Return ammount of data written.
744 cb146cf9 Antony Chazapis
745 cddcf432 chazapis
#def hashmap_hash(request, hashmap):
746 cddcf432 chazapis
#    """Produce the root hash, treating the hashmap as a Merkle-like tree."""
747 cddcf432 chazapis
#    
748 cddcf432 chazapis
#    map = HashMap(request.backend.block_size, request.backend.hash_algorithm)
749 cddcf432 chazapis
#    map.extend([unhexlify(x) for x in hashmap])
750 cddcf432 chazapis
#    return hexlify(map.hash())
751 cddcf432 chazapis
752 cddcf432 chazapis
def hashmap_md5(request, hashmap, size):
753 cddcf432 chazapis
    """Produce the MD5 sum from the data in the hashmap."""
754 cddcf432 chazapis
    
755 cddcf432 chazapis
    # TODO: Search backend for the MD5 of another object with the same hashmap and size...
756 cddcf432 chazapis
    md5 = hashlib.md5()
757 cddcf432 chazapis
    bs = request.backend.block_size
758 cddcf432 chazapis
    for bi, hash in enumerate(hashmap):
759 cddcf432 chazapis
        data = request.backend.get_block(hash)
760 cddcf432 chazapis
        if bi == len(hashmap) - 1:
761 cddcf432 chazapis
            bs = size % bs
762 cddcf432 chazapis
        pad = bs - min(len(data), bs)
763 cddcf432 chazapis
        md5.update(data + ('\x00' * pad))
764 cddcf432 chazapis
    return md5.hexdigest().lower()
765 49133392 Antony Chazapis
766 228de81b Antony Chazapis
def get_backend():
767 228de81b Antony Chazapis
    backend = connect_backend(db_module=settings.BACKEND_DB_MODULE,
768 228de81b Antony Chazapis
                              db_connection=settings.BACKEND_DB_CONNECTION,
769 228de81b Antony Chazapis
                              block_module=settings.BACKEND_BLOCK_MODULE,
770 228de81b Antony Chazapis
                              block_path=settings.BACKEND_BLOCK_PATH)
771 228de81b Antony Chazapis
    backend.default_policy['quota'] = settings.BACKEND_QUOTA
772 228de81b Antony Chazapis
    backend.default_policy['versioning'] = settings.BACKEND_VERSIONING
773 228de81b Antony Chazapis
    return backend
774 228de81b Antony Chazapis
775 9fefc052 Antony Chazapis
def update_request_headers(request):
776 9fefc052 Antony Chazapis
    # Handle URL-encoded keys and values.
777 88283e9e Antony Chazapis
    # Handle URL-encoded keys and values.
778 88283e9e Antony Chazapis
    meta = dict([(k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
779 88283e9e Antony Chazapis
    if len(meta) > 90:
780 88283e9e Antony Chazapis
        raise BadRequest('Too many headers.')
781 88283e9e Antony Chazapis
    for k, v in meta.iteritems():
782 88283e9e Antony Chazapis
        if len(k) > 128:
783 88283e9e Antony Chazapis
            raise BadRequest('Header name too large.')
784 88283e9e Antony Chazapis
        if len(v) > 256:
785 88283e9e Antony Chazapis
            raise BadRequest('Header value too large.')
786 88283e9e Antony Chazapis
        try:
787 88283e9e Antony Chazapis
            k.decode('ascii')
788 88283e9e Antony Chazapis
            v.decode('ascii')
789 88283e9e Antony Chazapis
        except UnicodeDecodeError:
790 88283e9e Antony Chazapis
            raise BadRequest('Bad character in headers.')
791 88283e9e Antony Chazapis
        if '%' in k or '%' in v:
792 88283e9e Antony Chazapis
            del(request.META[k])
793 88283e9e Antony Chazapis
            request.META[unquote(k)] = smart_unicode(unquote(v), strings_only=True)
794 9fefc052 Antony Chazapis
795 b956618e Antony Chazapis
def update_response_headers(request, response):
796 b956618e Antony Chazapis
    if request.serialization == 'xml':
797 b956618e Antony Chazapis
        response['Content-Type'] = 'application/xml; charset=UTF-8'
798 b956618e Antony Chazapis
    elif request.serialization == 'json':
799 b956618e Antony Chazapis
        response['Content-Type'] = 'application/json; charset=UTF-8'
800 22dab079 Antony Chazapis
    elif not response['Content-Type']:
801 b956618e Antony Chazapis
        response['Content-Type'] = 'text/plain; charset=UTF-8'
802 b980169d Antony Chazapis
    
803 9fefc052 Antony Chazapis
    if (not response.has_header('Content-Length') and
804 9fefc052 Antony Chazapis
        not (response.has_header('Content-Type') and
805 9fefc052 Antony Chazapis
             response['Content-Type'].startswith('multipart/byteranges'))):
806 b980169d Antony Chazapis
        response['Content-Length'] = len(response.content)
807 b980169d Antony Chazapis
    
808 9fefc052 Antony Chazapis
    # URL-encode unicode in headers.
809 9fefc052 Antony Chazapis
    meta = response.items()
810 9fefc052 Antony Chazapis
    for k, v in meta:
811 88283e9e Antony Chazapis
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
812 88283e9e Antony Chazapis
            k.startswith('X-Object-') or k.startswith('Content-')):
813 9fefc052 Antony Chazapis
            del(response[k])
814 88283e9e Antony Chazapis
            response[quote(k)] = quote(v, safe='/=,:@; ')
815 9fefc052 Antony Chazapis
    
816 b956618e Antony Chazapis
    if settings.TEST:
817 b956618e Antony Chazapis
        response['Date'] = format_date_time(time())
818 b956618e Antony Chazapis
819 b956618e Antony Chazapis
def render_fault(request, fault):
820 b956618e Antony Chazapis
    if settings.DEBUG or settings.TEST:
821 b956618e Antony Chazapis
        fault.details = format_exc(fault)
822 1993fea9 Antony Chazapis
    
823 b956618e Antony Chazapis
    request.serialization = 'text'
824 b956618e Antony Chazapis
    data = '\n'.join((fault.message, fault.details)) + '\n'
825 b956618e Antony Chazapis
    response = HttpResponse(data, status=fault.code)
826 b956618e Antony Chazapis
    update_response_headers(request, response)
827 b956618e Antony Chazapis
    return response
828 b956618e Antony Chazapis
829 b956618e Antony Chazapis
def request_serialization(request, format_allowed=False):
830 58a6c894 Antony Chazapis
    """Return the serialization format requested.
831 b956618e Antony Chazapis
    
832 b956618e Antony Chazapis
    Valid formats are 'text' and 'json', 'xml' if 'format_allowed' is True.
833 b956618e Antony Chazapis
    """
834 83dd59c5 Antony Chazapis
    
835 b956618e Antony Chazapis
    if not format_allowed:
836 b956618e Antony Chazapis
        return 'text'
837 b956618e Antony Chazapis
    
838 b956618e Antony Chazapis
    format = request.GET.get('format')
839 b956618e Antony Chazapis
    if format == 'json':
840 b956618e Antony Chazapis
        return 'json'
841 b956618e Antony Chazapis
    elif format == 'xml':
842 b956618e Antony Chazapis
        return 'xml'
843 b956618e Antony Chazapis
    
844 f9f15f92 Antony Chazapis
    for item in request.META.get('HTTP_ACCEPT', '').split(','):
845 f9f15f92 Antony Chazapis
        accept, sep, rest = item.strip().partition(';')
846 f9f15f92 Antony Chazapis
        if accept == 'application/json':
847 f9f15f92 Antony Chazapis
            return 'json'
848 f9f15f92 Antony Chazapis
        elif accept == 'application/xml' or accept == 'text/xml':
849 f9f15f92 Antony Chazapis
            return 'xml'
850 b956618e Antony Chazapis
    
851 b956618e Antony Chazapis
    return 'text'
852 b956618e Antony Chazapis
853 297513ba Antony Chazapis
def api_method(http_method=None, format_allowed=False, user_required=True):
854 58a6c894 Antony Chazapis
    """Decorator function for views that implement an API method."""
855 83dd59c5 Antony Chazapis
    
856 b956618e Antony Chazapis
    def decorator(func):
857 b956618e Antony Chazapis
        @wraps(func)
858 b956618e Antony Chazapis
        def wrapper(request, *args, **kwargs):
859 b956618e Antony Chazapis
            try:
860 b956618e Antony Chazapis
                if http_method and request.method != http_method:
861 b956618e Antony Chazapis
                    raise BadRequest('Method not allowed.')
862 297513ba Antony Chazapis
                if user_required and getattr(request, 'user', None) is None:
863 297513ba Antony Chazapis
                    raise Unauthorized('Access denied')
864 2c22e4ac Antony Chazapis
                
865 b956618e Antony Chazapis
                # The args variable may contain up to (account, container, object).
866 b956618e Antony Chazapis
                if len(args) > 1 and len(args[1]) > 256:
867 b956618e Antony Chazapis
                    raise BadRequest('Container name too large.')
868 b956618e Antony Chazapis
                if len(args) > 2 and len(args[2]) > 1024:
869 b956618e Antony Chazapis
                    raise BadRequest('Object name too large.')
870 b956618e Antony Chazapis
                
871 9fefc052 Antony Chazapis
                # Format and check headers.
872 9fefc052 Antony Chazapis
                update_request_headers(request)
873 9fefc052 Antony Chazapis
                
874 b956618e Antony Chazapis
                # Fill in custom request variables.
875 b956618e Antony Chazapis
                request.serialization = request_serialization(request, format_allowed)
876 228de81b Antony Chazapis
                request.backend = get_backend()
877 07813fd6 Sofia Papagiannaki
                
878 b956618e Antony Chazapis
                response = func(request, *args, **kwargs)
879 b956618e Antony Chazapis
                update_response_headers(request, response)
880 b956618e Antony Chazapis
                return response
881 b956618e Antony Chazapis
            except Fault, fault:
882 b956618e Antony Chazapis
                return render_fault(request, fault)
883 b956618e Antony Chazapis
            except BaseException, e:
884 b956618e Antony Chazapis
                logger.exception('Unexpected error: %s' % e)
885 b956618e Antony Chazapis
                fault = ServiceUnavailable('Unexpected error')
886 b956618e Antony Chazapis
                return render_fault(request, fault)
887 7b25e082 Antony Chazapis
            finally:
888 297513ba Antony Chazapis
                if getattr(request, 'backend', None) is not None:
889 d14fe290 Antony Chazapis
                    request.backend.close()
890 b956618e Antony Chazapis
        return wrapper
891 b956618e Antony Chazapis
    return decorator