Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ 1e47e49d

History | View | Annotate | Download (45.4 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from functools import wraps
35
from datetime import datetime
36
from urllib import quote, unquote, urlencode
37

    
38
from django.http import (HttpResponse, Http404, HttpResponseRedirect,
39
                         HttpResponseNotAllowed)
40
from django.template.loader import render_to_string
41
from django.utils import simplejson as json
42
from django.utils.http import http_date, parse_etags
43
from django.utils.encoding import smart_unicode, smart_str
44
from django.core.files.uploadhandler import FileUploadHandler
45
from django.core.files.uploadedfile import UploadedFile
46
from django.core.urlresolvers import reverse
47
from django.core.exceptions import PermissionDenied
48

    
49
from snf_django.lib.api.parsedate import parse_http_date_safe, parse_http_date
50
from snf_django.lib import api
51
from snf_django.lib.api import faults, utils
52

    
53
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
54
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
55
                                 BACKEND_BLOCK_UMASK,
56
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
57
                                 BACKEND_QUEUE_EXCHANGE,
58
                                 ASTAKOSCLIENT_POOLSIZE,
59
                                 SERVICE_TOKEN,
60
                                 ASTAKOS_AUTH_URL,
61
                                 BACKEND_ACCOUNT_QUOTA,
62
                                 BACKEND_CONTAINER_QUOTA,
63
                                 BACKEND_VERSIONING, BACKEND_FREE_VERSIONING,
64
                                 BACKEND_POOL_ENABLED, BACKEND_POOL_SIZE,
65
                                 BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM,
66
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
67
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
68
                                 PUBLIC_URL_SECURITY, PUBLIC_URL_ALPHABET,
69
                                 BASE_HOST, UPDATE_MD5, VIEW_PREFIX,
70
                                 OAUTH2_CLIENT_CREDENTIALS, UNSAFE_DOMAIN)
71

    
72
from pithos.api.resources import resources
73
from pithos.backends import connect_backend
74
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
75
                                  VersionNotExists, IllegalOperationError)
76

    
77
from synnefo.lib import join_urls
78
from synnefo.util import text
79

    
80
from astakosclient import AstakosClient
81
from astakosclient.errors import NoUserName, NoUUID, AstakosClientException
82

    
83
import logging
84
import re
85
import hashlib
86
import uuid
87
import decimal
88

    
89
logger = logging.getLogger(__name__)
90

    
91

    
92
def json_encode_decimal(obj):
93
    if isinstance(obj, decimal.Decimal):
94
        return str(obj)
95
    raise TypeError(repr(obj) + " is not JSON serializable")
96

    
97

    
98
def rename_meta_key(d, old, new):
99
    if old not in d:
100
        return
101
    d[new] = d[old]
102
    del(d[old])
103

    
104

    
105
def printable_header_dict(d):
106
    """Format a meta dictionary for printing out json/xml.
107

108
    Convert all keys to lower case and replace dashes with underscores.
109
    Format 'last_modified' timestamp.
110
    """
111

    
112
    timestamps = ('last_modified', 'x_container_until_timestamp',
113
                  'x_acount_until_timestamp')
114
    for timestamp in timestamps:
115
        if timestamp in d and d[timestamp]:
116
            d[timestamp] = utils.isoformat(
117
                datetime.fromtimestamp(d[timestamp]))
118
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
119

    
120

    
121
def format_header_key(k):
122
    """Convert underscores to dashes and capitalize intra-dash strings."""
123
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
124

    
125

    
126
def get_header_prefix(request, prefix):
127
    """Get all prefix-* request headers in a dict.
128
       Reformat keys with format_header_key()."""
129

    
130
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
131
    # TODO: Document or remove '~' replacing.
132
    return dict([(format_header_key(k[5:]), v.replace('~', ''))
133
                for k, v in request.META.iteritems()
134
                if k.startswith(prefix) and len(k) > len(prefix)])
135

    
136

    
137
def check_meta_headers(meta):
138
    if len(meta) > 90:
139
        raise faults.BadRequest('Too many headers.')
140
    for k, v in meta.iteritems():
141
        if len(k) > 128:
142
            raise faults.BadRequest('Header name too large.')
143
        if len(v) > 256:
144
            raise faults.BadRequest('Header value too large.')
145

    
146

    
147
def get_account_headers(request):
148
    meta = get_header_prefix(request, 'X-Account-Meta-')
149
    check_meta_headers(meta)
150
    groups = {}
151
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
152
        n = k[16:].lower()
153
        if '-' in n or '_' in n:
154
            raise faults.BadRequest('Bad characters in group name')
155
        groups[n] = v.replace(' ', '').split(',')
156
        while '' in groups[n]:
157
            groups[n].remove('')
158
    return meta, groups
159

    
160

    
161
def put_account_headers(response, meta, groups, policy):
162
    if 'count' in meta:
163
        response['X-Account-Container-Count'] = meta['count']
164
    if 'bytes' in meta:
165
        response['X-Account-Bytes-Used'] = meta['bytes']
166
    response['Last-Modified'] = http_date(int(meta['modified']))
167
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
168
        response[smart_str(
169
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
170
    if 'until_timestamp' in meta:
171
        response['X-Account-Until-Timestamp'] = http_date(
172
            int(meta['until_timestamp']))
173
    for k, v in groups.iteritems():
174
        k = smart_str(k, strings_only=True)
175
        k = format_header_key('X-Account-Group-' + k)
176
        v = smart_str(','.join(v), strings_only=True)
177
        response[k] = v
178
    for k, v in policy.iteritems():
179
        response[smart_str(format_header_key('X-Account-Policy-' + k),
180
                 strings_only=True)] = smart_str(v, strings_only=True)
181

    
182

    
183
def get_container_headers(request):
184
    meta = get_header_prefix(request, 'X-Container-Meta-')
185
    check_meta_headers(meta)
186
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in
187
                  get_header_prefix(request,
188
                                    'X-Container-Policy-').iteritems()])
189
    return meta, policy
190

    
191

    
192
def put_container_headers(request, response, meta, policy):
193
    if 'count' in meta:
194
        response['X-Container-Object-Count'] = meta['count']
195
    if 'bytes' in meta:
196
        response['X-Container-Bytes-Used'] = meta['bytes']
197
    response['Last-Modified'] = http_date(int(meta['modified']))
198
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
199
        response[smart_str(
200
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
201
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
202
         if x.startswith('X-Object-Meta-')]
203
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
204
    response['X-Container-Block-Size'] = request.backend.block_size
205
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
206
    if 'until_timestamp' in meta:
207
        response['X-Container-Until-Timestamp'] = http_date(
208
            int(meta['until_timestamp']))
209
    for k, v in policy.iteritems():
210
        response[smart_str(format_header_key('X-Container-Policy-' + k),
211
                           strings_only=True)] = smart_str(v,
212
                                                           strings_only=True)
213

    
214

    
215
def get_object_headers(request):
216
    content_type = request.META.get('CONTENT_TYPE', None)
217
    meta = get_header_prefix(request, 'X-Object-Meta-')
218
    check_meta_headers(meta)
219
    if request.META.get('HTTP_CONTENT_ENCODING'):
220
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
221
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
222
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
223
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
224
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
225
    return content_type, meta, get_sharing(request), get_public(request)
226

    
227

    
228
def put_object_headers(response, meta, restricted=False, token=None):
229
    response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
230
    response['Content-Length'] = meta['bytes']
231
    response.override_serialization = True
232
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
233
    response['Last-Modified'] = http_date(int(meta['modified']))
234
    if not restricted:
235
        response['X-Object-Hash'] = meta['hash']
236
        response['X-Object-UUID'] = meta['uuid']
237
        if TRANSLATE_UUIDS:
238
            meta['modified_by'] = \
239
                retrieve_displayname(token, meta['modified_by'])
240
        response['X-Object-Modified-By'] = smart_str(
241
            meta['modified_by'], strings_only=True)
242
        response['X-Object-Version'] = meta['version']
243
        response['X-Object-Version-Timestamp'] = http_date(
244
            int(meta['version_timestamp']))
245
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
246
            response[smart_str(
247
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
248
        for k in (
249
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
250
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
251
                'X-Object-Public'):
252
            if k in meta:
253
                response[k] = smart_str(meta[k], strings_only=True)
254
    else:
255
        for k in ('Content-Encoding', 'Content-Disposition'):
256
            if k in meta:
257
                response[k] = smart_str(meta[k], strings_only=True)
258

    
259

    
260
def update_manifest_meta(request, v_account, meta):
261
    """Update metadata if the object has an X-Object-Manifest."""
262

    
263
    if 'X-Object-Manifest' in meta:
264
        etag = ''
265
        bytes = 0
266
        try:
267
            src_container, src_name = split_container_object_string(
268
                '/' + meta['X-Object-Manifest'])
269
            objects = request.backend.list_objects(
270
                request.user_uniq, v_account,
271
                src_container, prefix=src_name, virtual=False)
272
            for x in objects:
273
                src_meta = request.backend.get_object_meta(
274
                    request.user_uniq, v_account, src_container, x[0],
275
                    'pithos', x[1])
276
                etag += (src_meta['hash'] if not UPDATE_MD5 else
277
                         src_meta['checksum'])
278
                bytes += src_meta['bytes']
279
        except:
280
            # Ignore errors.
281
            return
282
        meta['bytes'] = bytes
283
        md5 = hashlib.md5()
284
        md5.update(etag)
285
        meta['checksum'] = md5.hexdigest().lower()
286

    
287

    
288
def is_uuid(str):
289
    if str is None:
290
        return False
291
    try:
292
        uuid.UUID(str)
293
    except ValueError:
294
        return False
295
    else:
296
        return True
297

    
298

    
299
##########################
300
# USER CATALOG utilities #
301
##########################
302

    
303
def retrieve_displayname(token, uuid, fail_silently=True):
304
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
305
                            retry=2, use_pool=True,
306
                            logger=logger)
307
    try:
308
        displayname = astakos.get_username(uuid)
309
    except NoUserName:
310
        if not fail_silently:
311
            raise ItemNotExists(uuid)
312
        else:
313
            # just return the uuid
314
            return uuid
315
    return displayname
316

    
317

    
318
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
319
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
320
                            retry=2, use_pool=True,
321
                            logger=logger)
322
    catalog = astakos.get_usernames(uuids) or {}
323
    missing = list(set(uuids) - set(catalog))
324
    if missing and not fail_silently:
325
        raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
326
    return catalog if return_dict else [catalog.get(i) for i in uuids]
327

    
328

    
329
def retrieve_uuid(token, displayname):
330
    if is_uuid(displayname):
331
        return displayname
332

    
333
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
334
                            retry=2, use_pool=True,
335
                            logger=logger)
336
    try:
337
        uuid = astakos.get_uuid(displayname)
338
    except NoUUID:
339
        raise ItemNotExists(displayname)
340
    return uuid
341

    
342

    
343
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
344
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
345
                            retry=2, use_pool=True,
346
                            logger=logger)
347
    catalog = astakos.get_uuids(displaynames) or {}
348
    missing = list(set(displaynames) - set(catalog))
349
    if missing and not fail_silently:
350
        raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
351
    return catalog if return_dict else [catalog.get(i) for i in displaynames]
352

    
353

    
354
def replace_permissions_displayname(token, holder):
355
    if holder == '*':
356
        return holder
357
    try:
358
        # check first for a group permission
359
        account, group = holder.split(':', 1)
360
    except ValueError:
361
        return retrieve_uuid(token, holder)
362
    else:
363
        return ':'.join([retrieve_uuid(token, account), group])
364

    
365

    
366
def replace_permissions_uuid(token, holder):
367
    if holder == '*':
368
        return holder
369
    try:
370
        # check first for a group permission
371
        account, group = holder.split(':', 1)
372
    except ValueError:
373
        return retrieve_displayname(token, holder)
374
    else:
375
        return ':'.join([retrieve_displayname(token, account), group])
376

    
377

    
378
def update_sharing_meta(request, permissions, v_account,
379
                        v_container, v_object, meta):
380
    if permissions is None:
381
        return
382
    allowed, perm_path, perms = permissions
383
    if len(perms) == 0:
384
        return
385

    
386
    # replace uuid with displayname
387
    if TRANSLATE_UUIDS:
388
        perms['read'] = [replace_permissions_uuid(
389
            getattr(request, 'token', None), x)
390
            for x in perms.get('read', [])]
391
        perms['write'] = [replace_permissions_uuid(
392
            getattr(request, 'token', None), x)
393
            for x in perms.get('write', [])]
394

    
395
    ret = []
396

    
397
    r = ','.join(perms.get('read', []))
398
    if r:
399
        ret.append('read=' + r)
400
    w = ','.join(perms.get('write', []))
401
    if w:
402
        ret.append('write=' + w)
403
    meta['X-Object-Sharing'] = '; '.join(ret)
404
    if '/'.join((v_account, v_container, v_object)) != perm_path:
405
        meta['X-Object-Shared-By'] = perm_path
406
    if request.user_uniq != v_account:
407
        meta['X-Object-Allowed-To'] = allowed
408

    
409

    
410
def update_public_meta(public, meta):
411
    if not public:
412
        return
413
    meta['X-Object-Public'] = join_urls(
414
        BASE_HOST, reverse('pithos.api.public.public_demux', args=(public,)))
415

    
416

    
417
def validate_modification_preconditions(request, meta):
418
    """Check the modified timestamp conforms with the preconditions set."""
419

    
420
    if 'modified' not in meta:
421
        return  # TODO: Always return?
422

    
423
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
424
    if if_modified_since is not None:
425
        if_modified_since = parse_http_date_safe(if_modified_since)
426
    if (if_modified_since is not None
427
            and int(meta['modified']) <= if_modified_since):
428
        raise faults.NotModified('Resource has not been modified')
429

    
430
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
431
    if if_unmodified_since is not None:
432
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
433
    if (if_unmodified_since is not None
434
            and int(meta['modified']) > if_unmodified_since):
435
        raise faults.PreconditionFailed('Resource has been modified')
436

    
437

    
438
def validate_matching_preconditions(request, meta):
439
    """Check that the ETag conforms with the preconditions set."""
440

    
441
    etag = meta['hash'] if not UPDATE_MD5 else meta['checksum']
442
    if not etag:
443
        etag = None
444

    
445
    if_match = request.META.get('HTTP_IF_MATCH')
446
    if if_match is not None:
447
        if etag is None:
448
            raise faults.PreconditionFailed('Resource does not exist')
449
        if (if_match != '*'
450
                and etag not in [x.lower() for x in parse_etags(if_match)]):
451
            raise faults.PreconditionFailed('Resource ETag does not match')
452

    
453
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
454
    if if_none_match is not None:
455
        # TODO: If this passes, must ignore If-Modified-Since header.
456
        if etag is not None:
457
            if (if_none_match == '*' or etag in [x.lower() for x in
458
                                                 parse_etags(if_none_match)]):
459
                # TODO: Continue if an If-Modified-Since header is present.
460
                if request.method in ('HEAD', 'GET'):
461
                    raise faults.NotModified('Resource ETag matches')
462
                raise faults.PreconditionFailed(
463
                    'Resource exists or ETag matches')
464

    
465

    
466
def split_container_object_string(s):
467
    if not len(s) > 0 or s[0] != '/':
468
        raise ValueError
469
    s = s[1:]
470
    pos = s.find('/')
471
    if pos == -1 or pos == len(s) - 1:
472
        raise ValueError
473
    return s[:pos], s[(pos + 1):]
474

    
475

    
476
def copy_or_move_object(request, src_account, src_container, src_name,
477
                        dest_account, dest_container, dest_name,
478
                        move=False, delimiter=None):
479
    """Copy or move an object."""
480

    
481
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
482
        del(request.META['CONTENT_TYPE'])
483
    content_type, meta, permissions, public = get_object_headers(request)
484
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
485
    try:
486
        if move:
487
            version_id = request.backend.move_object(
488
                request.user_uniq, src_account, src_container, src_name,
489
                dest_account, dest_container, dest_name,
490
                content_type, 'pithos', meta, False, permissions, delimiter)
491
        else:
492
            version_id = request.backend.copy_object(
493
                request.user_uniq, src_account, src_container, src_name,
494
                dest_account, dest_container, dest_name,
495
                content_type, 'pithos', meta, False, permissions,
496
                src_version, delimiter)
497
    except NotAllowedError:
498
        raise faults.Forbidden('Not allowed')
499
    except (ItemNotExists, VersionNotExists):
500
        raise faults.ItemNotFound('Container or object does not exist')
501
    except ValueError:
502
        raise faults.BadRequest('Invalid sharing header')
503
    except QuotaError, e:
504
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
505
    if public is not None:
506
        try:
507
            request.backend.update_object_public(
508
                request.user_uniq, dest_account,
509
                dest_container, dest_name, public)
510
        except NotAllowedError:
511
            raise faults.Forbidden('Not allowed')
512
        except ItemNotExists:
513
            raise faults.ItemNotFound('Object does not exist')
514
    return version_id
515

    
516

    
517
def get_int_parameter(p):
518
    if p is not None:
519
        try:
520
            p = int(p)
521
        except ValueError:
522
            return None
523
        if p < 0:
524
            return None
525
    return p
526

    
527

    
528
def get_content_length(request):
529
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
530
    if content_length is None:
531
        raise faults.LengthRequired('Missing or invalid Content-Length header')
532
    return content_length
533

    
534

    
535
def get_range(request, size):
536
    """Parse a Range header from the request.
537

538
    Either returns None, when the header is not existent or should be ignored,
539
    or a list of (offset, length) tuples - should be further checked.
540
    """
541

    
542
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
543
    if not ranges.startswith('bytes='):
544
        return None
545

    
546
    ret = []
547
    for r in (x.strip() for x in ranges[6:].split(',')):
548
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
549
        m = p.match(r)
550
        if not m:
551
            return None
552
        offset = m.group('offset')
553
        upto = m.group('upto')
554
        if offset == '' and upto == '':
555
            return None
556

    
557
        if offset != '':
558
            offset = int(offset)
559
            if upto != '':
560
                upto = int(upto)
561
                if offset > upto:
562
                    return None
563
                ret.append((offset, upto - offset + 1))
564
            else:
565
                ret.append((offset, size - offset))
566
        else:
567
            length = int(upto)
568
            ret.append((size - length, length))
569

    
570
    return ret
571

    
572

    
573
def get_content_range(request):
574
    """Parse a Content-Range header from the request.
575

576
    Either returns None, when the header is not existent or should be ignored,
577
    or an (offset, length, total) tuple - check as length, total may be None.
578
    Returns (None, None, None) if the provided range is '*/*'.
579
    """
580

    
581
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
582
    if not ranges:
583
        return None
584

    
585
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
586
    m = p.match(ranges)
587
    if not m:
588
        if ranges == 'bytes */*':
589
            return (None, None, None)
590
        return None
591
    offset = int(m.group('offset'))
592
    upto = m.group('upto')
593
    total = m.group('total')
594
    if upto != '':
595
        upto = int(upto)
596
    else:
597
        upto = None
598
    if total != '*':
599
        total = int(total)
600
    else:
601
        total = None
602
    if (upto is not None and offset > upto) or \
603
        (total is not None and offset >= total) or \
604
            (total is not None and upto is not None and upto >= total):
605
        return None
606

    
607
    if upto is None:
608
        length = None
609
    else:
610
        length = upto - offset + 1
611
    return (offset, length, total)
612

    
613

    
614
def get_sharing(request):
615
    """Parse an X-Object-Sharing header from the request.
616

617
    Raises BadRequest on error.
618
    """
619

    
620
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
621
    if permissions is None:
622
        return None
623

    
624
    # TODO: Document or remove '~' replacing.
625
    permissions = permissions.replace('~', '')
626

    
627
    ret = {}
628
    permissions = permissions.replace(' ', '')
629
    if permissions == '':
630
        return ret
631
    for perm in (x for x in permissions.split(';')):
632
        if perm.startswith('read='):
633
            ret['read'] = list(set(
634
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
635
            if '' in ret['read']:
636
                ret['read'].remove('')
637
            if '*' in ret['read']:
638
                ret['read'] = ['*']
639
            if len(ret['read']) == 0:
640
                raise faults.BadRequest(
641
                    'Bad X-Object-Sharing header value: invalid length')
642
        elif perm.startswith('write='):
643
            ret['write'] = list(set(
644
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
645
            if '' in ret['write']:
646
                ret['write'].remove('')
647
            if '*' in ret['write']:
648
                ret['write'] = ['*']
649
            if len(ret['write']) == 0:
650
                raise faults.BadRequest(
651
                    'Bad X-Object-Sharing header value: invalid length')
652
        else:
653
            raise faults.BadRequest(
654
                'Bad X-Object-Sharing header value: missing prefix')
655

    
656
    # replace displayname with uuid
657
    if TRANSLATE_UUIDS:
658
        try:
659
            ret['read'] = [replace_permissions_displayname(
660
                getattr(request, 'token', None), x)
661
                for x in ret.get('read', [])]
662
            ret['write'] = [replace_permissions_displayname(
663
                getattr(request, 'token', None), x)
664
                for x in ret.get('write', [])]
665
        except ItemNotExists, e:
666
            raise faults.BadRequest(
667
                'Bad X-Object-Sharing header value: unknown account: %s' % e)
668

    
669
    # Keep duplicates only in write list.
670
    dups = [x for x in ret.get(
671
        'read', []) if x in ret.get('write', []) and x != '*']
672
    if dups:
673
        for x in dups:
674
            ret['read'].remove(x)
675
        if len(ret['read']) == 0:
676
            del(ret['read'])
677

    
678
    return ret
679

    
680

    
681
def get_public(request):
682
    """Parse an X-Object-Public header from the request.
683

684
    Raises BadRequest on error.
685
    """
686

    
687
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
688
    if public is None:
689
        return None
690

    
691
    public = public.replace(' ', '').lower()
692
    if public == 'true':
693
        return True
694
    elif public == 'false' or public == '':
695
        return False
696
    raise faults.BadRequest('Bad X-Object-Public header value')
697

    
698

    
699
def raw_input_socket(request):
700
    """Return the socket for reading the rest of the request."""
701

    
702
    server_software = request.META.get('SERVER_SOFTWARE')
703
    if server_software and server_software.startswith('mod_python'):
704
        return request._req
705
    if 'wsgi.input' in request.environ:
706
        return request.environ['wsgi.input']
707
    raise NotImplemented('Unknown server software')
708

    
709
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB
710

    
711

    
712
def socket_read_iterator(request, length=0, blocksize=4096):
713
    """Return maximum of blocksize data read from the socket in each iteration
714

715
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
716
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
717
    """
718

    
719
    sock = raw_input_socket(request)
720
    if length < 0:  # Chunked transfers
721
        # Small version (server does the dechunking).
722
        if (request.environ.get('mod_wsgi.input_chunked', None)
723
                or request.META['SERVER_SOFTWARE'].startswith('gunicorn')):
724
            while length < MAX_UPLOAD_SIZE:
725
                data = sock.read(blocksize)
726
                if data == '':
727
                    return
728
                yield data
729
            raise faults.BadRequest('Maximum size is reached')
730

    
731
        # Long version (do the dechunking).
732
        data = ''
733
        while length < MAX_UPLOAD_SIZE:
734
            # Get chunk size.
735
            if hasattr(sock, 'readline'):
736
                chunk_length = sock.readline()
737
            else:
738
                chunk_length = ''
739
                while chunk_length[-1:] != '\n':
740
                    chunk_length += sock.read(1)
741
                chunk_length.strip()
742
            pos = chunk_length.find(';')
743
            if pos >= 0:
744
                chunk_length = chunk_length[:pos]
745
            try:
746
                chunk_length = int(chunk_length, 16)
747
            except Exception:
748
                raise faults.BadRequest('Bad chunk size')
749
                                 # TODO: Change to something more appropriate.
750
            # Check if done.
751
            if chunk_length == 0:
752
                if len(data) > 0:
753
                    yield data
754
                return
755
            # Get the actual data.
756
            while chunk_length > 0:
757
                chunk = sock.read(min(chunk_length, blocksize))
758
                chunk_length -= len(chunk)
759
                if length > 0:
760
                    length += len(chunk)
761
                data += chunk
762
                if len(data) >= blocksize:
763
                    ret = data[:blocksize]
764
                    data = data[blocksize:]
765
                    yield ret
766
            sock.read(2)  # CRLF
767
        raise faults.BadRequest('Maximum size is reached')
768
    else:
769
        if length > MAX_UPLOAD_SIZE:
770
            raise faults.BadRequest('Maximum size is reached')
771
        while length > 0:
772
            data = sock.read(min(length, blocksize))
773
            if not data:
774
                raise faults.BadRequest()
775
            length -= len(data)
776
            yield data
777

    
778

    
779
class SaveToBackendHandler(FileUploadHandler):
780
    """Handle a file from an HTML form the django way."""
781

    
782
    def __init__(self, request=None):
783
        super(SaveToBackendHandler, self).__init__(request)
784
        self.backend = request.backend
785

    
786
    def put_data(self, length):
787
        if len(self.data) >= length:
788
            block = self.data[:length]
789
            self.file.hashmap.append(self.backend.put_block(block))
790
            self.checksum_compute.update(block)
791
            self.data = self.data[length:]
792

    
793
    def new_file(self, field_name, file_name, content_type,
794
                 content_length, charset=None):
795
        self.checksum_compute = NoChecksum() if not UPDATE_MD5 else Checksum()
796
        self.data = ''
797
        self.file = UploadedFile(
798
            name=file_name, content_type=content_type, charset=charset)
799
        self.file.size = 0
800
        self.file.hashmap = []
801

    
802
    def receive_data_chunk(self, raw_data, start):
803
        self.data += raw_data
804
        self.file.size += len(raw_data)
805
        self.put_data(self.request.backend.block_size)
806
        return None
807

    
808
    def file_complete(self, file_size):
809
        l = len(self.data)
810
        if l > 0:
811
            self.put_data(l)
812
        self.file.etag = self.checksum_compute.hexdigest()
813
        return self.file
814

    
815

    
816
class ObjectWrapper(object):
817
    """Return the object's data block-per-block in each iteration.
818

819
    Read from the object using the offset and length provided
820
    in each entry of the range list.
821
    """
822

    
823
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
824
        self.backend = backend
825
        self.ranges = ranges
826
        self.sizes = sizes
827
        self.hashmaps = hashmaps
828
        self.boundary = boundary
829
        self.size = sum(self.sizes)
830

    
831
        self.file_index = 0
832
        self.block_index = 0
833
        self.block_hash = -1
834
        self.block = ''
835

    
836
        self.range_index = -1
837
        self.offset, self.length = self.ranges[0]
838

    
839
    def __iter__(self):
840
        return self
841

    
842
    def part_iterator(self):
843
        if self.length > 0:
844
            # Get the file for the current offset.
845
            file_size = self.sizes[self.file_index]
846
            while self.offset >= file_size:
847
                self.offset -= file_size
848
                self.file_index += 1
849
                file_size = self.sizes[self.file_index]
850

    
851
            # Get the block for the current position.
852
            self.block_index = int(self.offset / self.backend.block_size)
853
            if self.block_hash != \
854
                    self.hashmaps[self.file_index][self.block_index]:
855
                self.block_hash = self.hashmaps[
856
                    self.file_index][self.block_index]
857
                try:
858
                    self.block = self.backend.get_block(self.block_hash)
859
                except ItemNotExists:
860
                    raise faults.ItemNotFound('Block does not exist')
861

    
862
            # Get the data from the block.
863
            bo = self.offset % self.backend.block_size
864
            bs = self.backend.block_size
865
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
866
                    self.sizes[self.file_index] % self.backend.block_size):
867
                bs = self.sizes[self.file_index] % self.backend.block_size
868
            bl = min(self.length, bs - bo)
869
            data = self.block[bo:bo + bl]
870
            self.offset += bl
871
            self.length -= bl
872
            return data
873
        else:
874
            raise StopIteration
875

    
876
    def next(self):
877
        if len(self.ranges) == 1:
878
            return self.part_iterator()
879
        if self.range_index == len(self.ranges):
880
            raise StopIteration
881
        try:
882
            if self.range_index == -1:
883
                raise StopIteration
884
            return self.part_iterator()
885
        except StopIteration:
886
            self.range_index += 1
887
            out = []
888
            if self.range_index < len(self.ranges):
889
                # Part header.
890
                self.offset, self.length = self.ranges[self.range_index]
891
                self.file_index = 0
892
                if self.range_index > 0:
893
                    out.append('')
894
                out.append('--' + self.boundary)
895
                out.append('Content-Range: bytes %d-%d/%d' % (
896
                    self.offset, self.offset + self.length - 1, self.size))
897
                out.append('Content-Transfer-Encoding: binary')
898
                out.append('')
899
                out.append('')
900
                return '\r\n'.join(out)
901
            else:
902
                # Footer.
903
                out.append('')
904
                out.append('--' + self.boundary + '--')
905
                out.append('')
906
                return '\r\n'.join(out)
907

    
908

    
909
def object_data_response(request, sizes, hashmaps, meta, public=False):
910
    """Get the HttpResponse object for replying with the object's data."""
911

    
912
    # Range handling.
913
    size = sum(sizes)
914
    ranges = get_range(request, size)
915
    if ranges is None:
916
        ranges = [(0, size)]
917
        ret = 200
918
    else:
919
        check = [True for offset, length in ranges if
920
                 length <= 0 or length > size or
921
                 offset < 0 or offset >= size or
922
                 offset + length > size]
923
        if len(check) > 0:
924
            raise faults.RangeNotSatisfiable(
925
                'Requested range exceeds object limits')
926
        ret = 206
927
        if_range = request.META.get('HTTP_IF_RANGE')
928
        if if_range:
929
            try:
930
                # Modification time has passed instead.
931
                last_modified = parse_http_date(if_range)
932
                if last_modified != meta['modified']:
933
                    ranges = [(0, size)]
934
                    ret = 200
935
            except ValueError:
936
                if if_range != meta['checksum']:
937
                    ranges = [(0, size)]
938
                    ret = 200
939

    
940
    if ret == 206 and len(ranges) > 1:
941
        boundary = uuid.uuid4().hex
942
    else:
943
        boundary = ''
944
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
945
    response = HttpResponse(wrapper, status=ret)
946
    put_object_headers(
947
        response, meta, restricted=public,
948
        token=getattr(request, 'token', None))
949
    if ret == 206:
950
        if len(ranges) == 1:
951
            offset, length = ranges[0]
952
            response[
953
                'Content-Length'] = length  # Update with the correct length.
954
            response['Content-Range'] = 'bytes %d-%d/%d' % (
955
                offset, offset + length - 1, size)
956
        else:
957
            del(response['Content-Length'])
958
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (
959
                boundary,)
960
    return response
961

    
962

    
963
def put_object_block(request, hashmap, data, offset):
964
    """Put one block of data at the given offset."""
965

    
966
    bi = int(offset / request.backend.block_size)
967
    bo = offset % request.backend.block_size
968
    bl = min(len(data), request.backend.block_size - bo)
969
    if bi < len(hashmap):
970
        try:
971
            hashmap[bi] = request.backend.update_block(hashmap[bi],
972
                                                       data[:bl], bo)
973
        except IllegalOperationError, e:
974
            raise faults.Forbidden(e)
975
    else:
976
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
977
    return bl  # Return ammount of data written.
978

    
979

    
980
def hashmap_md5(backend, hashmap, size):
981
    """Produce the MD5 sum from the data in the hashmap."""
982

    
983
    # TODO: Search backend for the MD5 of another object
984
    #       with the same hashmap and size...
985
    md5 = hashlib.md5()
986
    bs = backend.block_size
987
    for bi, hash in enumerate(hashmap):
988
        data = backend.get_block(hash)  # Blocks come in padded.
989
        if bi == len(hashmap) - 1:
990
            data = data[:size % bs]
991
        md5.update(data)
992
    return md5.hexdigest().lower()
993

    
994

    
995
def simple_list_response(request, l):
996
    if request.serialization == 'text':
997
        return '\n'.join(l) + '\n'
998
    if request.serialization == 'xml':
999
        return render_to_string('items.xml', {'items': l})
1000
    if request.serialization == 'json':
1001
        return json.dumps(l)
1002

    
1003

    
1004
from pithos.backends.util import PithosBackendPool
1005

    
1006
if RADOS_STORAGE:
1007
    BLOCK_PARAMS = {'mappool': RADOS_POOL_MAPS,
1008
                    'blockpool': RADOS_POOL_BLOCKS, }
1009
else:
1010
    BLOCK_PARAMS = {'mappool': None,
1011
                    'blockpool': None, }
1012

    
1013
BACKEND_KWARGS = dict(
1014
    db_module=BACKEND_DB_MODULE,
1015
    db_connection=BACKEND_DB_CONNECTION,
1016
    block_module=BACKEND_BLOCK_MODULE,
1017
    block_path=BACKEND_BLOCK_PATH,
1018
    block_umask=BACKEND_BLOCK_UMASK,
1019
    block_size=BACKEND_BLOCK_SIZE,
1020
    hash_algorithm=BACKEND_HASH_ALGORITHM,
1021
    queue_module=BACKEND_QUEUE_MODULE,
1022
    queue_hosts=BACKEND_QUEUE_HOSTS,
1023
    queue_exchange=BACKEND_QUEUE_EXCHANGE,
1024
    astakos_auth_url=ASTAKOS_AUTH_URL,
1025
    service_token=SERVICE_TOKEN,
1026
    astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
1027
    free_versioning=BACKEND_FREE_VERSIONING,
1028
    block_params=BLOCK_PARAMS,
1029
    public_url_security=PUBLIC_URL_SECURITY,
1030
    public_url_alphabet=PUBLIC_URL_ALPHABET,
1031
    account_quota_policy=BACKEND_ACCOUNT_QUOTA,
1032
    container_quota_policy=BACKEND_CONTAINER_QUOTA,
1033
    container_versioning_policy=BACKEND_VERSIONING)
1034

    
1035
_pithos_backend_pool = PithosBackendPool(size=BACKEND_POOL_SIZE,
1036
                                         **BACKEND_KWARGS)
1037

    
1038

    
1039
def get_backend():
1040
    if BACKEND_POOL_ENABLED:
1041
        backend = _pithos_backend_pool.pool_get()
1042
    else:
1043
        backend = connect_backend(**BACKEND_KWARGS)
1044
    backend.serials = []
1045
    backend.messages = []
1046
    return backend
1047

    
1048

    
1049
def update_request_headers(request):
1050
    # Handle URL-encoded keys and values.
1051
    meta = dict([(
1052
        k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
1053
    for k, v in meta.iteritems():
1054
        try:
1055
            k.decode('ascii')
1056
            v.decode('ascii')
1057
        except UnicodeDecodeError:
1058
            raise faults.BadRequest('Bad character in headers.')
1059
        if '%' in k or '%' in v:
1060
            del(request.META[k])
1061
            request.META[unquote(k)] = smart_unicode(unquote(
1062
                v), strings_only=True)
1063

    
1064

    
1065
def update_response_headers(request, response):
1066
    # URL-encode unicode in headers.
1067
    meta = response.items()
1068
    for k, v in meta:
1069
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
1070
                k.startswith('X-Object-') or k.startswith('Content-')):
1071
            del(response[k])
1072
            response[quote(k)] = quote(v, safe='/=,:@; ')
1073

    
1074

    
1075
def api_method(http_method=None, token_required=True, user_required=True,
1076
               logger=None, format_allowed=False, serializations=None,
1077
               strict_serlization=False, lock_container_path=False):
1078
    serializations = serializations or ['json', 'xml']
1079

    
1080
    def decorator(func):
1081
        @api.api_method(http_method=http_method, token_required=token_required,
1082
                        user_required=user_required,
1083
                        logger=logger, format_allowed=format_allowed,
1084
                        astakos_auth_url=ASTAKOS_AUTH_URL,
1085
                        serializations=serializations,
1086
                        strict_serlization=strict_serlization)
1087
        @wraps(func)
1088
        def wrapper(request, *args, **kwargs):
1089
            # The args variable may contain up to (account, container, object).
1090
            if len(args) > 1 and len(args[1]) > 256:
1091
                raise faults.BadRequest("Container name too large")
1092
            if len(args) > 2 and len(args[2]) > 1024:
1093
                raise faults.BadRequest('Object name too large.')
1094

    
1095
            success_status = False
1096
            try:
1097
                # Add a PithosBackend as attribute of the request object
1098
                request.backend = get_backend()
1099
                request.backend.pre_exec(lock_container_path)
1100

    
1101
                # Many API method expect thet X-Auth-Token in request,token
1102
                request.token = request.x_auth_token
1103
                update_request_headers(request)
1104
                response = func(request, *args, **kwargs)
1105
                update_response_headers(request, response)
1106

    
1107
                success_status = True
1108
                return response
1109
            finally:
1110
                # Always close PithosBackend connection
1111
                if getattr(request, "backend", None) is not None:
1112
                    request.backend.post_exec(success_status)
1113
                    request.backend.close()
1114
        return wrapper
1115
    return decorator
1116

    
1117

    
1118
def restrict_to_host(host=None):
1119
    """
1120
    View decorator which restricts wrapped view to be accessed only under the
1121
    host set. If an invalid host is identified and request HTTP method is one
1122
    of ``GET``, ``HOST``, the decorator will return a redirect response using a
1123
    clone of the request with host replaced to the one the restriction applies
1124
    to.
1125

1126
    e.g.
1127
    @restrict_to_host('files.example.com')
1128
    my_restricted_view(request, path):
1129
        return HttpResponse(file(path).read())
1130

1131
    A get to ``https://api.example.com/my_restricted_view/file_path/?param=1``
1132
    will return a redirect response with Location header set to
1133
    ``https://files.example.com/my_restricted_view/file_path/?param=1``.
1134

1135
    If host is set to ``None`` no restriction will be applied.
1136
    """
1137
    def decorator(func):
1138
        # skip decoration if no host is set
1139
        if not host:
1140
            return func
1141

    
1142
        @wraps(func)
1143
        def wrapper(request, *args, **kwargs):
1144
            request_host = request.get_host()
1145
            if host != request_host:
1146
                proto = 'https' if request.is_secure() else 'http'
1147
                if request.method in ['GET', 'HEAD']:
1148
                    full_path = request.get_full_path()
1149
                    redirect_uri = "%s://%s%s" % (proto, host, full_path)
1150
                    return HttpResponseRedirect(redirect_uri)
1151
                else:
1152
                    raise PermissionDenied
1153
            return func(request, *args, **kwargs)
1154
        return wrapper
1155
    return decorator
1156

    
1157

    
1158
def view_method():
1159
    """Decorator function for views."""
1160

    
1161
    def decorator(func):
1162
        @restrict_to_host(UNSAFE_DOMAIN)
1163
        @wraps(func)
1164
        def wrapper(request, *args, **kwargs):
1165
            if request.method not in ['GET', 'HEAD']:
1166
                return HttpResponseNotAllowed(['GET', 'HEAD'])
1167

    
1168
            try:
1169
                access_token = request.GET.get('access_token')
1170
                requested_resource = text.uenc(request.path.split(VIEW_PREFIX,
1171
                                                                  2)[-1])
1172
                astakos = AstakosClient(SERVICE_TOKEN, ASTAKOS_AUTH_URL,
1173
                                        retry=2, use_pool=True,
1174
                                        logger=logger)
1175
                if access_token is not None:
1176
                    # authenticate using the short-term access token
1177
                    try:
1178
                        request.user = astakos.validate_token(
1179
                            access_token, requested_resource)
1180
                    except AstakosClientException:
1181
                        return HttpResponseRedirect(request.path)
1182
                    request.user_uniq = request.user["access"]["user"]["id"]
1183

    
1184
                    _func = api_method(token_required=False,
1185
                                       user_required=False)(func)
1186
                    response = _func(request, *args, **kwargs)
1187
                    if response.status_code == 404:
1188
                        raise Http404
1189
                    elif response.status_code == 403:
1190
                        raise PermissionDenied
1191
                    return response
1192

    
1193
                client_id, client_secret = OAUTH2_CLIENT_CREDENTIALS
1194
                # TODO: check if client credentials are not set
1195
                authorization_code = request.GET.get('code')
1196
                if authorization_code is None:
1197
                    # request authorization code
1198
                    params = {'response_type': 'code',
1199
                              'client_id': client_id,
1200
                              'redirect_uri':
1201
                              request.build_absolute_uri(request.path),
1202
                              'state': '',  # TODO include state for security
1203
                              'scope': requested_resource}
1204
                    return HttpResponseRedirect('%s?%s' %
1205
                                                (join_urls(astakos.oauth2_url,
1206
                                                           'auth'),
1207
                                                 urlencode(params)))
1208
                else:
1209
                    # request short-term access token
1210
                    redirect_uri = request.build_absolute_uri(request.path)
1211
                    data = astakos.get_token('authorization_code',
1212
                                             *OAUTH2_CLIENT_CREDENTIALS,
1213
                                             redirect_uri=redirect_uri,
1214
                                             scope=requested_resource,
1215
                                             code=authorization_code)
1216
                    params = {'access_token': data.get('access_token', '')}
1217
                    return HttpResponseRedirect('%s?%s' % (redirect_uri,
1218
                                                           urlencode(params)))
1219
            except AstakosClientException, err:
1220
                logger.exception(err)
1221
                raise PermissionDenied
1222
        return wrapper
1223
    return decorator
1224

    
1225

    
1226
class Checksum:
1227
    def __init__(self):
1228
        self.md5 = hashlib.md5()
1229

    
1230
    def update(self, data):
1231
        self.md5.update(data)
1232

    
1233
    def hexdigest(self):
1234
        return self.md5.hexdigest().lower()
1235

    
1236

    
1237
class NoChecksum:
1238
    def update(self, data):
1239
        pass
1240

    
1241
    def hexdigest(self):
1242
        return ''