Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ ec6f741b

History | View | Annotate | Download (45.2 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from functools import wraps
35
from datetime import datetime
36
from urllib import quote, unquote, urlencode
37

    
38
from django.http import (HttpResponse, Http404, HttpResponseRedirect,
39
                         HttpResponseNotAllowed)
40
from django.template.loader import render_to_string
41
from django.utils import simplejson as json
42
from django.utils.http import http_date, parse_etags
43
from django.utils.encoding import smart_unicode, smart_str
44
from django.core.files.uploadhandler import FileUploadHandler
45
from django.core.files.uploadedfile import UploadedFile
46
from django.core.urlresolvers import reverse
47
from django.core.exceptions import PermissionDenied
48

    
49
from snf_django.lib.api.parsedate import parse_http_date_safe, parse_http_date
50
from snf_django.lib import api
51
from snf_django.lib.api import faults, utils
52

    
53
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
54
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
55
                                 BACKEND_BLOCK_UMASK,
56
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
57
                                 BACKEND_QUEUE_EXCHANGE,
58
                                 ASTAKOSCLIENT_POOLSIZE,
59
                                 SERVICE_TOKEN,
60
                                 ASTAKOS_AUTH_URL,
61
                                 BACKEND_ACCOUNT_QUOTA,
62
                                 BACKEND_CONTAINER_QUOTA,
63
                                 BACKEND_VERSIONING, BACKEND_FREE_VERSIONING,
64
                                 BACKEND_POOL_ENABLED, BACKEND_POOL_SIZE,
65
                                 BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM,
66
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
67
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
68
                                 PUBLIC_URL_SECURITY, PUBLIC_URL_ALPHABET,
69
                                 BASE_HOST, UPDATE_MD5, VIEW_PREFIX,
70
                                 OAUTH2_CLIENT_CREDENTIALS, UNSAFE_DOMAIN)
71

    
72
from pithos.api.resources import resources
73
from pithos.backends import connect_backend
74
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
75
                                  VersionNotExists)
76

    
77
from synnefo.lib import join_urls
78
from synnefo.util import text
79

    
80
from astakosclient import AstakosClient
81
from astakosclient.errors import NoUserName, NoUUID, AstakosClientException
82

    
83
import logging
84
import re
85
import hashlib
86
import uuid
87
import decimal
88

    
89
logger = logging.getLogger(__name__)
90

    
91

    
92
def json_encode_decimal(obj):
93
    if isinstance(obj, decimal.Decimal):
94
        return str(obj)
95
    raise TypeError(repr(obj) + " is not JSON serializable")
96

    
97

    
98
def rename_meta_key(d, old, new):
99
    if old not in d:
100
        return
101
    d[new] = d[old]
102
    del(d[old])
103

    
104

    
105
def printable_header_dict(d):
106
    """Format a meta dictionary for printing out json/xml.
107

108
    Convert all keys to lower case and replace dashes with underscores.
109
    Format 'last_modified' timestamp.
110
    """
111

    
112
    timestamps = ('last_modified', 'x_container_until_timestamp',
113
                  'x_acount_until_timestamp')
114
    for timestamp in timestamps:
115
        if timestamp in d and d[timestamp]:
116
            d[timestamp] = utils.isoformat(
117
                datetime.fromtimestamp(d[timestamp]))
118
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
119

    
120

    
121
def format_header_key(k):
122
    """Convert underscores to dashes and capitalize intra-dash strings."""
123
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
124

    
125

    
126
def get_header_prefix(request, prefix):
127
    """Get all prefix-* request headers in a dict.
128
       Reformat keys with format_header_key()."""
129

    
130
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
131
    # TODO: Document or remove '~' replacing.
132
    return dict([(format_header_key(k[5:]), v.replace('~', ''))
133
                for k, v in request.META.iteritems()
134
                if k.startswith(prefix) and len(k) > len(prefix)])
135

    
136

    
137
def check_meta_headers(meta):
138
    if len(meta) > 90:
139
        raise faults.BadRequest('Too many headers.')
140
    for k, v in meta.iteritems():
141
        if len(k) > 128:
142
            raise faults.BadRequest('Header name too large.')
143
        if len(v) > 256:
144
            raise faults.BadRequest('Header value too large.')
145

    
146

    
147
def get_account_headers(request):
148
    meta = get_header_prefix(request, 'X-Account-Meta-')
149
    check_meta_headers(meta)
150
    groups = {}
151
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
152
        n = k[16:].lower()
153
        if '-' in n or '_' in n:
154
            raise faults.BadRequest('Bad characters in group name')
155
        groups[n] = v.replace(' ', '').split(',')
156
        while '' in groups[n]:
157
            groups[n].remove('')
158
    return meta, groups
159

    
160

    
161
def put_account_headers(response, meta, groups, policy):
162
    if 'count' in meta:
163
        response['X-Account-Container-Count'] = meta['count']
164
    if 'bytes' in meta:
165
        response['X-Account-Bytes-Used'] = meta['bytes']
166
    response['Last-Modified'] = http_date(int(meta['modified']))
167
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
168
        response[smart_str(
169
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
170
    if 'until_timestamp' in meta:
171
        response['X-Account-Until-Timestamp'] = http_date(
172
            int(meta['until_timestamp']))
173
    for k, v in groups.iteritems():
174
        k = smart_str(k, strings_only=True)
175
        k = format_header_key('X-Account-Group-' + k)
176
        v = smart_str(','.join(v), strings_only=True)
177
        response[k] = v
178
    for k, v in policy.iteritems():
179
        response[smart_str(format_header_key('X-Account-Policy-' + k),
180
                 strings_only=True)] = smart_str(v, strings_only=True)
181

    
182

    
183
def get_container_headers(request):
184
    meta = get_header_prefix(request, 'X-Container-Meta-')
185
    check_meta_headers(meta)
186
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in
187
                  get_header_prefix(request,
188
                                    'X-Container-Policy-').iteritems()])
189
    return meta, policy
190

    
191

    
192
def put_container_headers(request, response, meta, policy):
193
    if 'count' in meta:
194
        response['X-Container-Object-Count'] = meta['count']
195
    if 'bytes' in meta:
196
        response['X-Container-Bytes-Used'] = meta['bytes']
197
    response['Last-Modified'] = http_date(int(meta['modified']))
198
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
199
        response[smart_str(
200
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
201
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
202
         if x.startswith('X-Object-Meta-')]
203
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
204
    response['X-Container-Block-Size'] = request.backend.block_size
205
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
206
    if 'until_timestamp' in meta:
207
        response['X-Container-Until-Timestamp'] = http_date(
208
            int(meta['until_timestamp']))
209
    for k, v in policy.iteritems():
210
        response[smart_str(format_header_key('X-Container-Policy-' + k),
211
                           strings_only=True)] = smart_str(v,
212
                                                           strings_only=True)
213

    
214

    
215
def get_object_headers(request):
216
    content_type = request.META.get('CONTENT_TYPE', None)
217
    meta = get_header_prefix(request, 'X-Object-Meta-')
218
    check_meta_headers(meta)
219
    if request.META.get('HTTP_CONTENT_ENCODING'):
220
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
221
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
222
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
223
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
224
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
225
    return content_type, meta, get_sharing(request), get_public(request)
226

    
227

    
228
def put_object_headers(response, meta, restricted=False, token=None):
229
    response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
230
    response['Content-Length'] = meta['bytes']
231
    response.override_serialization = True
232
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
233
    response['Last-Modified'] = http_date(int(meta['modified']))
234
    if not restricted:
235
        response['X-Object-Hash'] = meta['hash']
236
        response['X-Object-UUID'] = meta['uuid']
237
        if TRANSLATE_UUIDS:
238
            meta['modified_by'] = \
239
                retrieve_displayname(token, meta['modified_by'])
240
        response['X-Object-Modified-By'] = smart_str(
241
            meta['modified_by'], strings_only=True)
242
        response['X-Object-Version'] = meta['version']
243
        response['X-Object-Version-Timestamp'] = http_date(
244
            int(meta['version_timestamp']))
245
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
246
            response[smart_str(
247
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
248
        for k in (
249
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
250
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
251
                'X-Object-Public'):
252
            if k in meta:
253
                response[k] = smart_str(meta[k], strings_only=True)
254
    else:
255
        for k in ('Content-Encoding', 'Content-Disposition'):
256
            if k in meta:
257
                response[k] = smart_str(meta[k], strings_only=True)
258

    
259

    
260
def update_manifest_meta(request, v_account, meta):
261
    """Update metadata if the object has an X-Object-Manifest."""
262

    
263
    if 'X-Object-Manifest' in meta:
264
        etag = ''
265
        bytes = 0
266
        try:
267
            src_container, src_name = split_container_object_string(
268
                '/' + meta['X-Object-Manifest'])
269
            objects = request.backend.list_objects(
270
                request.user_uniq, v_account,
271
                src_container, prefix=src_name, virtual=False)
272
            for x in objects:
273
                src_meta = request.backend.get_object_meta(
274
                    request.user_uniq, v_account, src_container, x[0],
275
                    'pithos', x[1])
276
                etag += (src_meta['hash'] if not UPDATE_MD5 else
277
                         src_meta['checksum'])
278
                bytes += src_meta['bytes']
279
        except:
280
            # Ignore errors.
281
            return
282
        meta['bytes'] = bytes
283
        md5 = hashlib.md5()
284
        md5.update(etag)
285
        meta['checksum'] = md5.hexdigest().lower()
286

    
287

    
288
def is_uuid(str):
289
    if str is None:
290
        return False
291
    try:
292
        uuid.UUID(str)
293
    except ValueError:
294
        return False
295
    else:
296
        return True
297

    
298

    
299
##########################
300
# USER CATALOG utilities #
301
##########################
302

    
303
def retrieve_displayname(token, uuid, fail_silently=True):
304
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
305
                            retry=2, use_pool=True,
306
                            logger=logger)
307
    try:
308
        displayname = astakos.get_username(uuid)
309
    except NoUserName:
310
        if not fail_silently:
311
            raise ItemNotExists(uuid)
312
        else:
313
            # just return the uuid
314
            return uuid
315
    return displayname
316

    
317

    
318
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
319
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
320
                            retry=2, use_pool=True,
321
                            logger=logger)
322
    catalog = astakos.get_usernames(uuids) or {}
323
    missing = list(set(uuids) - set(catalog))
324
    if missing and not fail_silently:
325
        raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
326
    return catalog if return_dict else [catalog.get(i) for i in uuids]
327

    
328

    
329
def retrieve_uuid(token, displayname):
330
    if is_uuid(displayname):
331
        return displayname
332

    
333
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
334
                            retry=2, use_pool=True,
335
                            logger=logger)
336
    try:
337
        uuid = astakos.get_uuid(displayname)
338
    except NoUUID:
339
        raise ItemNotExists(displayname)
340
    return uuid
341

    
342

    
343
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
344
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
345
                            retry=2, use_pool=True,
346
                            logger=logger)
347
    catalog = astakos.get_uuids(displaynames) or {}
348
    missing = list(set(displaynames) - set(catalog))
349
    if missing and not fail_silently:
350
        raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
351
    return catalog if return_dict else [catalog.get(i) for i in displaynames]
352

    
353

    
354
def replace_permissions_displayname(token, holder):
355
    if holder == '*':
356
        return holder
357
    try:
358
        # check first for a group permission
359
        account, group = holder.split(':', 1)
360
    except ValueError:
361
        return retrieve_uuid(token, holder)
362
    else:
363
        return ':'.join([retrieve_uuid(token, account), group])
364

    
365

    
366
def replace_permissions_uuid(token, holder):
367
    if holder == '*':
368
        return holder
369
    try:
370
        # check first for a group permission
371
        account, group = holder.split(':', 1)
372
    except ValueError:
373
        return retrieve_displayname(token, holder)
374
    else:
375
        return ':'.join([retrieve_displayname(token, account), group])
376

    
377

    
378
def update_sharing_meta(request, permissions, v_account,
379
                        v_container, v_object, meta):
380
    if permissions is None:
381
        return
382
    allowed, perm_path, perms = permissions
383
    if len(perms) == 0:
384
        return
385

    
386
    # replace uuid with displayname
387
    if TRANSLATE_UUIDS:
388
        perms['read'] = [replace_permissions_uuid(
389
            getattr(request, 'token', None), x)
390
            for x in perms.get('read', [])]
391
        perms['write'] = [replace_permissions_uuid(
392
            getattr(request, 'token', None), x)
393
            for x in perms.get('write', [])]
394

    
395
    ret = []
396

    
397
    r = ','.join(perms.get('read', []))
398
    if r:
399
        ret.append('read=' + r)
400
    w = ','.join(perms.get('write', []))
401
    if w:
402
        ret.append('write=' + w)
403
    meta['X-Object-Sharing'] = '; '.join(ret)
404
    if '/'.join((v_account, v_container, v_object)) != perm_path:
405
        meta['X-Object-Shared-By'] = perm_path
406
    if request.user_uniq != v_account:
407
        meta['X-Object-Allowed-To'] = allowed
408

    
409

    
410
def update_public_meta(public, meta):
411
    if not public:
412
        return
413
    meta['X-Object-Public'] = join_urls(
414
        BASE_HOST, reverse('pithos.api.public.public_demux', args=(public,)))
415

    
416

    
417
def validate_modification_preconditions(request, meta):
418
    """Check the modified timestamp conforms with the preconditions set."""
419

    
420
    if 'modified' not in meta:
421
        return  # TODO: Always return?
422

    
423
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
424
    if if_modified_since is not None:
425
        if_modified_since = parse_http_date_safe(if_modified_since)
426
    if (if_modified_since is not None
427
            and int(meta['modified']) <= if_modified_since):
428
        raise faults.NotModified('Resource has not been modified')
429

    
430
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
431
    if if_unmodified_since is not None:
432
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
433
    if (if_unmodified_since is not None
434
            and int(meta['modified']) > if_unmodified_since):
435
        raise faults.PreconditionFailed('Resource has been modified')
436

    
437

    
438
def validate_matching_preconditions(request, meta):
439
    """Check that the ETag conforms with the preconditions set."""
440

    
441
    etag = meta['hash'] if not UPDATE_MD5 else meta['checksum']
442
    if not etag:
443
        etag = None
444

    
445
    if_match = request.META.get('HTTP_IF_MATCH')
446
    if if_match is not None:
447
        if etag is None:
448
            raise faults.PreconditionFailed('Resource does not exist')
449
        if (if_match != '*'
450
                and etag not in [x.lower() for x in parse_etags(if_match)]):
451
            raise faults.PreconditionFailed('Resource ETag does not match')
452

    
453
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
454
    if if_none_match is not None:
455
        # TODO: If this passes, must ignore If-Modified-Since header.
456
        if etag is not None:
457
            if (if_none_match == '*' or etag in [x.lower() for x in
458
                                                 parse_etags(if_none_match)]):
459
                # TODO: Continue if an If-Modified-Since header is present.
460
                if request.method in ('HEAD', 'GET'):
461
                    raise faults.NotModified('Resource ETag matches')
462
                raise faults.PreconditionFailed(
463
                    'Resource exists or ETag matches')
464

    
465

    
466
def split_container_object_string(s):
467
    if not len(s) > 0 or s[0] != '/':
468
        raise ValueError
469
    s = s[1:]
470
    pos = s.find('/')
471
    if pos == -1 or pos == len(s) - 1:
472
        raise ValueError
473
    return s[:pos], s[(pos + 1):]
474

    
475

    
476
def copy_or_move_object(request, src_account, src_container, src_name,
477
                        dest_account, dest_container, dest_name,
478
                        move=False, delimiter=None):
479
    """Copy or move an object."""
480

    
481
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
482
        del(request.META['CONTENT_TYPE'])
483
    content_type, meta, permissions, public = get_object_headers(request)
484
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
485
    try:
486
        if move:
487
            version_id = request.backend.move_object(
488
                request.user_uniq, src_account, src_container, src_name,
489
                dest_account, dest_container, dest_name,
490
                content_type, 'pithos', meta, False, permissions, delimiter)
491
        else:
492
            version_id = request.backend.copy_object(
493
                request.user_uniq, src_account, src_container, src_name,
494
                dest_account, dest_container, dest_name,
495
                content_type, 'pithos', meta, False, permissions,
496
                src_version, delimiter)
497
    except NotAllowedError:
498
        raise faults.Forbidden('Not allowed')
499
    except (ItemNotExists, VersionNotExists):
500
        raise faults.ItemNotFound('Container or object does not exist')
501
    except ValueError:
502
        raise faults.BadRequest('Invalid sharing header')
503
    except QuotaError, e:
504
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
505
    if public is not None:
506
        try:
507
            request.backend.update_object_public(
508
                request.user_uniq, dest_account,
509
                dest_container, dest_name, public)
510
        except NotAllowedError:
511
            raise faults.Forbidden('Not allowed')
512
        except ItemNotExists:
513
            raise faults.ItemNotFound('Object does not exist')
514
    return version_id
515

    
516

    
517
def get_int_parameter(p):
518
    if p is not None:
519
        try:
520
            p = int(p)
521
        except ValueError:
522
            return None
523
        if p < 0:
524
            return None
525
    return p
526

    
527

    
528
def get_content_length(request):
529
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
530
    if content_length is None:
531
        raise faults.LengthRequired('Missing or invalid Content-Length header')
532
    return content_length
533

    
534

    
535
def get_range(request, size):
536
    """Parse a Range header from the request.
537

538
    Either returns None, when the header is not existent or should be ignored,
539
    or a list of (offset, length) tuples - should be further checked.
540
    """
541

    
542
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
543
    if not ranges.startswith('bytes='):
544
        return None
545

    
546
    ret = []
547
    for r in (x.strip() for x in ranges[6:].split(',')):
548
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
549
        m = p.match(r)
550
        if not m:
551
            return None
552
        offset = m.group('offset')
553
        upto = m.group('upto')
554
        if offset == '' and upto == '':
555
            return None
556

    
557
        if offset != '':
558
            offset = int(offset)
559
            if upto != '':
560
                upto = int(upto)
561
                if offset > upto:
562
                    return None
563
                ret.append((offset, upto - offset + 1))
564
            else:
565
                ret.append((offset, size - offset))
566
        else:
567
            length = int(upto)
568
            ret.append((size - length, length))
569

    
570
    return ret
571

    
572

    
573
def get_content_range(request):
574
    """Parse a Content-Range header from the request.
575

576
    Either returns None, when the header is not existent or should be ignored,
577
    or an (offset, length, total) tuple - check as length, total may be None.
578
    Returns (None, None, None) if the provided range is '*/*'.
579
    """
580

    
581
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
582
    if not ranges:
583
        return None
584

    
585
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
586
    m = p.match(ranges)
587
    if not m:
588
        if ranges == 'bytes */*':
589
            return (None, None, None)
590
        return None
591
    offset = int(m.group('offset'))
592
    upto = m.group('upto')
593
    total = m.group('total')
594
    if upto != '':
595
        upto = int(upto)
596
    else:
597
        upto = None
598
    if total != '*':
599
        total = int(total)
600
    else:
601
        total = None
602
    if (upto is not None and offset > upto) or \
603
        (total is not None and offset >= total) or \
604
            (total is not None and upto is not None and upto >= total):
605
        return None
606

    
607
    if upto is None:
608
        length = None
609
    else:
610
        length = upto - offset + 1
611
    return (offset, length, total)
612

    
613

    
614
def get_sharing(request):
615
    """Parse an X-Object-Sharing header from the request.
616

617
    Raises BadRequest on error.
618
    """
619

    
620
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
621
    if permissions is None:
622
        return None
623

    
624
    # TODO: Document or remove '~' replacing.
625
    permissions = permissions.replace('~', '')
626

    
627
    ret = {}
628
    permissions = permissions.replace(' ', '')
629
    if permissions == '':
630
        return ret
631
    for perm in (x for x in permissions.split(';')):
632
        if perm.startswith('read='):
633
            ret['read'] = list(set(
634
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
635
            if '' in ret['read']:
636
                ret['read'].remove('')
637
            if '*' in ret['read']:
638
                ret['read'] = ['*']
639
            if len(ret['read']) == 0:
640
                raise faults.BadRequest(
641
                    'Bad X-Object-Sharing header value: invalid length')
642
        elif perm.startswith('write='):
643
            ret['write'] = list(set(
644
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
645
            if '' in ret['write']:
646
                ret['write'].remove('')
647
            if '*' in ret['write']:
648
                ret['write'] = ['*']
649
            if len(ret['write']) == 0:
650
                raise faults.BadRequest(
651
                    'Bad X-Object-Sharing header value: invalid length')
652
        else:
653
            raise faults.BadRequest(
654
                'Bad X-Object-Sharing header value: missing prefix')
655

    
656
    # replace displayname with uuid
657
    if TRANSLATE_UUIDS:
658
        try:
659
            ret['read'] = [replace_permissions_displayname(
660
                getattr(request, 'token', None), x)
661
                for x in ret.get('read', [])]
662
            ret['write'] = [replace_permissions_displayname(
663
                getattr(request, 'token', None), x)
664
                for x in ret.get('write', [])]
665
        except ItemNotExists, e:
666
            raise faults.BadRequest(
667
                'Bad X-Object-Sharing header value: unknown account: %s' % e)
668

    
669
    # Keep duplicates only in write list.
670
    dups = [x for x in ret.get(
671
        'read', []) if x in ret.get('write', []) and x != '*']
672
    if dups:
673
        for x in dups:
674
            ret['read'].remove(x)
675
        if len(ret['read']) == 0:
676
            del(ret['read'])
677

    
678
    return ret
679

    
680

    
681
def get_public(request):
682
    """Parse an X-Object-Public header from the request.
683

684
    Raises BadRequest on error.
685
    """
686

    
687
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
688
    if public is None:
689
        return None
690

    
691
    public = public.replace(' ', '').lower()
692
    if public == 'true':
693
        return True
694
    elif public == 'false' or public == '':
695
        return False
696
    raise faults.BadRequest('Bad X-Object-Public header value')
697

    
698

    
699
def raw_input_socket(request):
700
    """Return the socket for reading the rest of the request."""
701

    
702
    server_software = request.META.get('SERVER_SOFTWARE')
703
    if server_software and server_software.startswith('mod_python'):
704
        return request._req
705
    if 'wsgi.input' in request.environ:
706
        return request.environ['wsgi.input']
707
    raise NotImplemented('Unknown server software')
708

    
709
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB
710

    
711

    
712
def socket_read_iterator(request, length=0, blocksize=4096):
713
    """Return maximum of blocksize data read from the socket in each iteration
714

715
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
716
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
717
    """
718

    
719
    sock = raw_input_socket(request)
720
    if length < 0:  # Chunked transfers
721
        # Small version (server does the dechunking).
722
        if (request.environ.get('mod_wsgi.input_chunked', None)
723
                or request.META['SERVER_SOFTWARE'].startswith('gunicorn')):
724
            while length < MAX_UPLOAD_SIZE:
725
                data = sock.read(blocksize)
726
                if data == '':
727
                    return
728
                yield data
729
            raise faults.BadRequest('Maximum size is reached')
730

    
731
        # Long version (do the dechunking).
732
        data = ''
733
        while length < MAX_UPLOAD_SIZE:
734
            # Get chunk size.
735
            if hasattr(sock, 'readline'):
736
                chunk_length = sock.readline()
737
            else:
738
                chunk_length = ''
739
                while chunk_length[-1:] != '\n':
740
                    chunk_length += sock.read(1)
741
                chunk_length.strip()
742
            pos = chunk_length.find(';')
743
            if pos >= 0:
744
                chunk_length = chunk_length[:pos]
745
            try:
746
                chunk_length = int(chunk_length, 16)
747
            except Exception:
748
                raise faults.BadRequest('Bad chunk size')
749
                                 # TODO: Change to something more appropriate.
750
            # Check if done.
751
            if chunk_length == 0:
752
                if len(data) > 0:
753
                    yield data
754
                return
755
            # Get the actual data.
756
            while chunk_length > 0:
757
                chunk = sock.read(min(chunk_length, blocksize))
758
                chunk_length -= len(chunk)
759
                if length > 0:
760
                    length += len(chunk)
761
                data += chunk
762
                if len(data) >= blocksize:
763
                    ret = data[:blocksize]
764
                    data = data[blocksize:]
765
                    yield ret
766
            sock.read(2)  # CRLF
767
        raise faults.BadRequest('Maximum size is reached')
768
    else:
769
        if length > MAX_UPLOAD_SIZE:
770
            raise faults.BadRequest('Maximum size is reached')
771
        while length > 0:
772
            data = sock.read(min(length, blocksize))
773
            if not data:
774
                raise faults.BadRequest()
775
            length -= len(data)
776
            yield data
777

    
778

    
779
class SaveToBackendHandler(FileUploadHandler):
780
    """Handle a file from an HTML form the django way."""
781

    
782
    def __init__(self, request=None):
783
        super(SaveToBackendHandler, self).__init__(request)
784
        self.backend = request.backend
785

    
786
    def put_data(self, length):
787
        if len(self.data) >= length:
788
            block = self.data[:length]
789
            self.file.hashmap.append(self.backend.put_block(block))
790
            self.checksum_compute.update(block)
791
            self.data = self.data[length:]
792

    
793
    def new_file(self, field_name, file_name, content_type,
794
                 content_length, charset=None):
795
        self.checksum_compute = NoChecksum() if not UPDATE_MD5 else Checksum()
796
        self.data = ''
797
        self.file = UploadedFile(
798
            name=file_name, content_type=content_type, charset=charset)
799
        self.file.size = 0
800
        self.file.hashmap = []
801

    
802
    def receive_data_chunk(self, raw_data, start):
803
        self.data += raw_data
804
        self.file.size += len(raw_data)
805
        self.put_data(self.request.backend.block_size)
806
        return None
807

    
808
    def file_complete(self, file_size):
809
        l = len(self.data)
810
        if l > 0:
811
            self.put_data(l)
812
        self.file.etag = self.checksum_compute.hexdigest()
813
        return self.file
814

    
815

    
816
class ObjectWrapper(object):
817
    """Return the object's data block-per-block in each iteration.
818

819
    Read from the object using the offset and length provided
820
    in each entry of the range list.
821
    """
822

    
823
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
824
        self.backend = backend
825
        self.ranges = ranges
826
        self.sizes = sizes
827
        self.hashmaps = hashmaps
828
        self.boundary = boundary
829
        self.size = sum(self.sizes)
830

    
831
        self.file_index = 0
832
        self.block_index = 0
833
        self.block_hash = -1
834
        self.block = ''
835

    
836
        self.range_index = -1
837
        self.offset, self.length = self.ranges[0]
838

    
839
    def __iter__(self):
840
        return self
841

    
842
    def part_iterator(self):
843
        if self.length > 0:
844
            # Get the file for the current offset.
845
            file_size = self.sizes[self.file_index]
846
            while self.offset >= file_size:
847
                self.offset -= file_size
848
                self.file_index += 1
849
                file_size = self.sizes[self.file_index]
850

    
851
            # Get the block for the current position.
852
            self.block_index = int(self.offset / self.backend.block_size)
853
            if self.block_hash != \
854
                    self.hashmaps[self.file_index][self.block_index]:
855
                self.block_hash = self.hashmaps[
856
                    self.file_index][self.block_index]
857
                try:
858
                    self.block = self.backend.get_block(self.block_hash)
859
                except ItemNotExists:
860
                    raise faults.ItemNotFound('Block does not exist')
861

    
862
            # Get the data from the block.
863
            bo = self.offset % self.backend.block_size
864
            bs = self.backend.block_size
865
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
866
                    self.sizes[self.file_index] % self.backend.block_size):
867
                bs = self.sizes[self.file_index] % self.backend.block_size
868
            bl = min(self.length, bs - bo)
869
            data = self.block[bo:bo + bl]
870
            self.offset += bl
871
            self.length -= bl
872
            return data
873
        else:
874
            raise StopIteration
875

    
876
    def next(self):
877
        if len(self.ranges) == 1:
878
            return self.part_iterator()
879
        if self.range_index == len(self.ranges):
880
            raise StopIteration
881
        try:
882
            if self.range_index == -1:
883
                raise StopIteration
884
            return self.part_iterator()
885
        except StopIteration:
886
            self.range_index += 1
887
            out = []
888
            if self.range_index < len(self.ranges):
889
                # Part header.
890
                self.offset, self.length = self.ranges[self.range_index]
891
                self.file_index = 0
892
                if self.range_index > 0:
893
                    out.append('')
894
                out.append('--' + self.boundary)
895
                out.append('Content-Range: bytes %d-%d/%d' % (
896
                    self.offset, self.offset + self.length - 1, self.size))
897
                out.append('Content-Transfer-Encoding: binary')
898
                out.append('')
899
                out.append('')
900
                return '\r\n'.join(out)
901
            else:
902
                # Footer.
903
                out.append('')
904
                out.append('--' + self.boundary + '--')
905
                out.append('')
906
                return '\r\n'.join(out)
907

    
908

    
909
def object_data_response(request, sizes, hashmaps, meta, public=False):
910
    """Get the HttpResponse object for replying with the object's data."""
911

    
912
    # Range handling.
913
    size = sum(sizes)
914
    ranges = get_range(request, size)
915
    if ranges is None:
916
        ranges = [(0, size)]
917
        ret = 200
918
    else:
919
        check = [True for offset, length in ranges if
920
                 length <= 0 or length > size or
921
                 offset < 0 or offset >= size or
922
                 offset + length > size]
923
        if len(check) > 0:
924
            raise faults.RangeNotSatisfiable(
925
                'Requested range exceeds object limits')
926
        ret = 206
927
        if_range = request.META.get('HTTP_IF_RANGE')
928
        if if_range:
929
            try:
930
                # Modification time has passed instead.
931
                last_modified = parse_http_date(if_range)
932
                if last_modified != meta['modified']:
933
                    ranges = [(0, size)]
934
                    ret = 200
935
            except ValueError:
936
                if if_range != meta['checksum']:
937
                    ranges = [(0, size)]
938
                    ret = 200
939

    
940
    if ret == 206 and len(ranges) > 1:
941
        boundary = uuid.uuid4().hex
942
    else:
943
        boundary = ''
944
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
945
    response = HttpResponse(wrapper, status=ret)
946
    put_object_headers(
947
        response, meta, restricted=public,
948
        token=getattr(request, 'token', None))
949
    if ret == 206:
950
        if len(ranges) == 1:
951
            offset, length = ranges[0]
952
            response[
953
                'Content-Length'] = length  # Update with the correct length.
954
            response['Content-Range'] = 'bytes %d-%d/%d' % (
955
                offset, offset + length - 1, size)
956
        else:
957
            del(response['Content-Length'])
958
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (
959
                boundary,)
960
    return response
961

    
962

    
963
def put_object_block(request, hashmap, data, offset):
964
    """Put one block of data at the given offset."""
965

    
966
    bi = int(offset / request.backend.block_size)
967
    bo = offset % request.backend.block_size
968
    bl = min(len(data), request.backend.block_size - bo)
969
    if bi < len(hashmap):
970
        hashmap[bi] = request.backend.update_block(hashmap[bi], data[:bl], bo)
971
    else:
972
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
973
    return bl  # Return ammount of data written.
974

    
975

    
976
def hashmap_md5(backend, hashmap, size):
977
    """Produce the MD5 sum from the data in the hashmap."""
978

    
979
    # TODO: Search backend for the MD5 of another object
980
    #       with the same hashmap and size...
981
    md5 = hashlib.md5()
982
    bs = backend.block_size
983
    for bi, hash in enumerate(hashmap):
984
        data = backend.get_block(hash)  # Blocks come in padded.
985
        if bi == len(hashmap) - 1:
986
            data = data[:size % bs]
987
        md5.update(data)
988
    return md5.hexdigest().lower()
989

    
990

    
991
def simple_list_response(request, l):
992
    if request.serialization == 'text':
993
        return '\n'.join(l) + '\n'
994
    if request.serialization == 'xml':
995
        return render_to_string('items.xml', {'items': l})
996
    if request.serialization == 'json':
997
        return json.dumps(l)
998

    
999

    
1000
from pithos.backends.util import PithosBackendPool
1001

    
1002
if RADOS_STORAGE:
1003
    BLOCK_PARAMS = {'mappool': RADOS_POOL_MAPS,
1004
                    'blockpool': RADOS_POOL_BLOCKS, }
1005
else:
1006
    BLOCK_PARAMS = {'mappool': None,
1007
                    'blockpool': None, }
1008

    
1009
BACKEND_KWARGS = dict(
1010
    db_module=BACKEND_DB_MODULE,
1011
    db_connection=BACKEND_DB_CONNECTION,
1012
    block_module=BACKEND_BLOCK_MODULE,
1013
    block_path=BACKEND_BLOCK_PATH,
1014
    block_umask=BACKEND_BLOCK_UMASK,
1015
    block_size=BACKEND_BLOCK_SIZE,
1016
    hash_algorithm=BACKEND_HASH_ALGORITHM,
1017
    queue_module=BACKEND_QUEUE_MODULE,
1018
    queue_hosts=BACKEND_QUEUE_HOSTS,
1019
    queue_exchange=BACKEND_QUEUE_EXCHANGE,
1020
    astakos_auth_url=ASTAKOS_AUTH_URL,
1021
    service_token=SERVICE_TOKEN,
1022
    astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
1023
    free_versioning=BACKEND_FREE_VERSIONING,
1024
    block_params=BLOCK_PARAMS,
1025
    public_url_security=PUBLIC_URL_SECURITY,
1026
    public_url_alphabet=PUBLIC_URL_ALPHABET,
1027
    account_quota_policy=BACKEND_ACCOUNT_QUOTA,
1028
    container_quota_policy=BACKEND_CONTAINER_QUOTA,
1029
    container_versioning_policy=BACKEND_VERSIONING)
1030

    
1031
_pithos_backend_pool = PithosBackendPool(size=BACKEND_POOL_SIZE,
1032
                                         **BACKEND_KWARGS)
1033

    
1034

    
1035
def get_backend():
1036
    if BACKEND_POOL_ENABLED:
1037
        backend = _pithos_backend_pool.pool_get()
1038
    else:
1039
        backend = connect_backend(**BACKEND_KWARGS)
1040
    backend.serials = []
1041
    backend.messages = []
1042
    return backend
1043

    
1044

    
1045
def update_request_headers(request):
1046
    # Handle URL-encoded keys and values.
1047
    meta = dict([(
1048
        k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
1049
    for k, v in meta.iteritems():
1050
        try:
1051
            k.decode('ascii')
1052
            v.decode('ascii')
1053
        except UnicodeDecodeError:
1054
            raise faults.BadRequest('Bad character in headers.')
1055
        if '%' in k or '%' in v:
1056
            del(request.META[k])
1057
            request.META[unquote(k)] = smart_unicode(unquote(
1058
                v), strings_only=True)
1059

    
1060

    
1061
def update_response_headers(request, response):
1062
    # URL-encode unicode in headers.
1063
    meta = response.items()
1064
    for k, v in meta:
1065
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
1066
                k.startswith('X-Object-') or k.startswith('Content-')):
1067
            del(response[k])
1068
            response[quote(k)] = quote(v, safe='/=,:@; ')
1069

    
1070

    
1071
def api_method(http_method=None, token_required=True, user_required=True,
1072
               logger=None, format_allowed=False, serializations=None,
1073
               strict_serlization=False, lock_container_path=False):
1074
    serializations = serializations or ['json', 'xml']
1075

    
1076
    def decorator(func):
1077
        @api.api_method(http_method=http_method, token_required=token_required,
1078
                        user_required=user_required,
1079
                        logger=logger, format_allowed=format_allowed,
1080
                        astakos_auth_url=ASTAKOS_AUTH_URL,
1081
                        serializations=serializations,
1082
                        strict_serlization=strict_serlization)
1083
        @wraps(func)
1084
        def wrapper(request, *args, **kwargs):
1085
            # The args variable may contain up to (account, container, object).
1086
            if len(args) > 1 and len(args[1]) > 256:
1087
                raise faults.BadRequest("Container name too large")
1088
            if len(args) > 2 and len(args[2]) > 1024:
1089
                raise faults.BadRequest('Object name too large.')
1090

    
1091
            success_status = False
1092
            try:
1093
                # Add a PithosBackend as attribute of the request object
1094
                request.backend = get_backend()
1095
                request.backend.pre_exec(lock_container_path)
1096

    
1097
                # Many API method expect thet X-Auth-Token in request,token
1098
                request.token = request.x_auth_token
1099
                update_request_headers(request)
1100
                response = func(request, *args, **kwargs)
1101
                update_response_headers(request, response)
1102

    
1103
                success_status = True
1104
                return response
1105
            finally:
1106
                # Always close PithosBackend connection
1107
                if getattr(request, "backend", None) is not None:
1108
                    request.backend.post_exec(success_status)
1109
                    request.backend.close()
1110
        return wrapper
1111
    return decorator
1112

    
1113

    
1114
def restrict_to_host(host=None):
1115
    """
1116
    View decorator which restricts wrapped view to be accessed only under the
1117
    host set. If an invalid host is identified and request HTTP method is one
1118
    of ``GET``, ``HOST``, the decorator will return a redirect response using a
1119
    clone of the request with host replaced to the one the restriction applies
1120
    to.
1121

1122
    e.g.
1123
    @restrict_to_host('files.example.com')
1124
    my_restricted_view(request, path):
1125
        return HttpResponse(file(path).read())
1126

1127
    A get to ``https://api.example.com/my_restricted_view/file_path/?param=1``
1128
    will return a redirect response with Location header set to
1129
    ``https://files.example.com/my_restricted_view/file_path/?param=1``.
1130

1131
    If host is set to ``None`` no restriction will be applied.
1132
    """
1133
    def decorator(func):
1134
        # skip decoration if no host is set
1135
        if not host:
1136
            return func
1137

    
1138
        @wraps(func)
1139
        def wrapper(request, *args, **kwargs):
1140
            request_host = request.get_host()
1141
            if host != request_host:
1142
                proto = 'https' if request.is_secure() else 'http'
1143
                if request.method in ['GET', 'HEAD']:
1144
                    full_path = request.get_full_path()
1145
                    redirect_uri = "%s://%s%s" % (proto, host, full_path)
1146
                    return HttpResponseRedirect(redirect_uri)
1147
                else:
1148
                    raise PermissionDenied
1149
            return func(request, *args, **kwargs)
1150
        return wrapper
1151
    return decorator
1152

    
1153

    
1154
def view_method():
1155
    """Decorator function for views."""
1156

    
1157
    def decorator(func):
1158
        @restrict_to_host(UNSAFE_DOMAIN)
1159
        @wraps(func)
1160
        def wrapper(request, *args, **kwargs):
1161
            if request.method not in ['GET', 'HEAD']:
1162
                return HttpResponseNotAllowed(['GET', 'HEAD'])
1163

    
1164
            try:
1165
                access_token = request.GET.get('access_token')
1166
                requested_resource = text.uenc(request.path.split(VIEW_PREFIX,
1167
                                                                  2)[-1])
1168
                astakos = AstakosClient(SERVICE_TOKEN, ASTAKOS_AUTH_URL,
1169
                                        retry=2, use_pool=True,
1170
                                        logger=logger)
1171
                if access_token is not None:
1172
                    # authenticate using the short-term access token
1173
                    try:
1174
                        request.user = astakos.validate_token(
1175
                            access_token, requested_resource)
1176
                    except AstakosClientException:
1177
                        return HttpResponseRedirect(request.path)
1178
                    request.user_uniq = request.user["access"]["user"]["id"]
1179

    
1180
                    _func = api_method(token_required=False,
1181
                                       user_required=False)(func)
1182
                    response = _func(request, *args, **kwargs)
1183
                    if response.status_code == 404:
1184
                        raise Http404
1185
                    elif response.status_code == 403:
1186
                        raise PermissionDenied
1187
                    return response
1188

    
1189
                client_id, client_secret = OAUTH2_CLIENT_CREDENTIALS
1190
                # TODO: check if client credentials are not set
1191
                authorization_code = request.GET.get('code')
1192
                if authorization_code is None:
1193
                    # request authorization code
1194
                    params = {'response_type': 'code',
1195
                              'client_id': client_id,
1196
                              'redirect_uri':
1197
                              request.build_absolute_uri(request.path),
1198
                              'state': '',  # TODO include state for security
1199
                              'scope': requested_resource}
1200
                    return HttpResponseRedirect('%s?%s' %
1201
                                                (join_urls(astakos.oauth2_url,
1202
                                                           'auth'),
1203
                                                 urlencode(params)))
1204
                else:
1205
                    # request short-term access token
1206
                    redirect_uri = request.build_absolute_uri(request.path)
1207
                    data = astakos.get_token('authorization_code',
1208
                                             *OAUTH2_CLIENT_CREDENTIALS,
1209
                                             redirect_uri=redirect_uri,
1210
                                             scope=requested_resource,
1211
                                             code=authorization_code)
1212
                    params = {'access_token': data.get('access_token', '')}
1213
                    return HttpResponseRedirect('%s?%s' % (redirect_uri,
1214
                                                           urlencode(params)))
1215
            except AstakosClientException, err:
1216
                logger.exception(err)
1217
                raise PermissionDenied
1218
        return wrapper
1219
    return decorator
1220

    
1221

    
1222
class Checksum:
1223
    def __init__(self):
1224
        self.md5 = hashlib.md5()
1225

    
1226
    def update(self, data):
1227
        self.md5.update(data)
1228

    
1229
    def hexdigest(self):
1230
        return self.md5.hexdigest().lower()
1231

    
1232

    
1233
class NoChecksum:
1234
    def update(self, data):
1235
        pass
1236

    
1237
    def hexdigest(self):
1238
        return ''