Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ e7f0ec5e

History | View | Annotate | Download (45.6 kB)

1
# Copyright 2011-2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from functools import wraps
35
from datetime import datetime
36
from urllib import quote, unquote, urlencode
37

    
38
from django.http import (HttpResponse, Http404, HttpResponseRedirect,
39
                         HttpResponseNotAllowed)
40
from django.template.loader import render_to_string
41
from django.utils import simplejson as json
42
from django.utils.http import http_date, parse_etags
43
from django.utils.encoding import smart_unicode, smart_str
44
from django.core.files.uploadhandler import FileUploadHandler
45
from django.core.files.uploadedfile import UploadedFile
46
from django.core.urlresolvers import reverse
47
from django.core.exceptions import PermissionDenied
48

    
49
from snf_django.lib.api.parsedate import parse_http_date_safe, parse_http_date
50
from snf_django.lib import api
51
from snf_django.lib.api import faults, utils
52

    
53
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
54
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
55
                                 BACKEND_BLOCK_UMASK,
56
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
57
                                 BACKEND_QUEUE_EXCHANGE,
58
                                 ASTAKOSCLIENT_POOLSIZE,
59
                                 SERVICE_TOKEN,
60
                                 ASTAKOS_AUTH_URL,
61
                                 BACKEND_ACCOUNT_QUOTA,
62
                                 BACKEND_CONTAINER_QUOTA,
63
                                 BACKEND_VERSIONING, BACKEND_FREE_VERSIONING,
64
                                 BACKEND_POOL_ENABLED, BACKEND_POOL_SIZE,
65
                                 BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM,
66
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
67
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
68
                                 PUBLIC_URL_SECURITY, PUBLIC_URL_ALPHABET,
69
                                 BASE_HOST, UPDATE_MD5, VIEW_PREFIX,
70
                                 OAUTH2_CLIENT_CREDENTIALS, SERVE_API_DOMAIN)
71

    
72
from pithos.api.resources import resources
73
from pithos.backends import connect_backend
74
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
75
                                  VersionNotExists)
76

    
77
from synnefo.lib import join_urls
78

    
79
from astakosclient import AstakosClient
80
from astakosclient.errors import NoUserName, NoUUID, AstakosClientException
81

    
82
import logging
83
import re
84
import hashlib
85
import uuid
86
import decimal
87

    
88
logger = logging.getLogger(__name__)
89

    
90

    
91
def json_encode_decimal(obj):
92
    if isinstance(obj, decimal.Decimal):
93
        return str(obj)
94
    raise TypeError(repr(obj) + " is not JSON serializable")
95

    
96

    
97
def rename_meta_key(d, old, new):
98
    if old not in d:
99
        return
100
    d[new] = d[old]
101
    del(d[old])
102

    
103

    
104
def printable_header_dict(d):
105
    """Format a meta dictionary for printing out json/xml.
106

107
    Convert all keys to lower case and replace dashes with underscores.
108
    Format 'last_modified' timestamp.
109
    """
110

    
111
    timestamps = ('last_modified', 'x_container_until_timestamp',
112
                  'x_acount_until_timestamp')
113
    for timestamp in timestamps:
114
        if timestamp in d and d[timestamp]:
115
            d[timestamp] = utils.isoformat(
116
                datetime.fromtimestamp(d[timestamp]))
117
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
118

    
119

    
120
def format_header_key(k):
121
    """Convert underscores to dashes and capitalize intra-dash strings."""
122
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
123

    
124

    
125
def get_header_prefix(request, prefix):
126
    """Get all prefix-* request headers in a dict.
127
       Reformat keys with format_header_key()."""
128

    
129
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
130
    # TODO: Document or remove '~' replacing.
131
    return dict([(format_header_key(k[5:]), v.replace('~', ''))
132
                for k, v in request.META.iteritems()
133
                if k.startswith(prefix) and len(k) > len(prefix)])
134

    
135

    
136
def check_meta_headers(meta):
137
    if len(meta) > 90:
138
        raise faults.BadRequest('Too many headers.')
139
    for k, v in meta.iteritems():
140
        if len(k) > 128:
141
            raise faults.BadRequest('Header name too large.')
142
        if len(v) > 256:
143
            raise faults.BadRequest('Header value too large.')
144

    
145

    
146
def get_account_headers(request):
147
    meta = get_header_prefix(request, 'X-Account-Meta-')
148
    check_meta_headers(meta)
149
    groups = {}
150
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
151
        n = k[16:].lower()
152
        if '-' in n or '_' in n:
153
            raise faults.BadRequest('Bad characters in group name')
154
        groups[n] = v.replace(' ', '').split(',')
155
        while '' in groups[n]:
156
            groups[n].remove('')
157
    return meta, groups
158

    
159

    
160
def put_account_headers(response, meta, groups, policy):
161
    if 'count' in meta:
162
        response['X-Account-Container-Count'] = meta['count']
163
    if 'bytes' in meta:
164
        response['X-Account-Bytes-Used'] = meta['bytes']
165
    response['Last-Modified'] = http_date(int(meta['modified']))
166
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
167
        response[smart_str(
168
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
169
    if 'until_timestamp' in meta:
170
        response['X-Account-Until-Timestamp'] = http_date(
171
            int(meta['until_timestamp']))
172
    for k, v in groups.iteritems():
173
        k = smart_str(k, strings_only=True)
174
        k = format_header_key('X-Account-Group-' + k)
175
        v = smart_str(','.join(v), strings_only=True)
176
        response[k] = v
177
    for k, v in policy.iteritems():
178
        response[smart_str(format_header_key('X-Account-Policy-' + k),
179
                 strings_only=True)] = smart_str(v, strings_only=True)
180

    
181

    
182
def get_container_headers(request):
183
    meta = get_header_prefix(request, 'X-Container-Meta-')
184
    check_meta_headers(meta)
185
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in
186
                  get_header_prefix(request,
187
                                    'X-Container-Policy-').iteritems()])
188
    return meta, policy
189

    
190

    
191
def put_container_headers(request, response, meta, policy):
192
    if 'count' in meta:
193
        response['X-Container-Object-Count'] = meta['count']
194
    if 'bytes' in meta:
195
        response['X-Container-Bytes-Used'] = meta['bytes']
196
    response['Last-Modified'] = http_date(int(meta['modified']))
197
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
198
        response[smart_str(
199
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
200
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
201
         if x.startswith('X-Object-Meta-')]
202
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
203
    response['X-Container-Block-Size'] = request.backend.block_size
204
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
205
    if 'until_timestamp' in meta:
206
        response['X-Container-Until-Timestamp'] = http_date(
207
            int(meta['until_timestamp']))
208
    for k, v in policy.iteritems():
209
        response[smart_str(format_header_key('X-Container-Policy-' + k),
210
                           strings_only=True)] = smart_str(v,
211
                                                           strings_only=True)
212

    
213

    
214
def get_object_headers(request):
215
    content_type = request.META.get('CONTENT_TYPE', None)
216
    meta = get_header_prefix(request, 'X-Object-Meta-')
217
    check_meta_headers(meta)
218
    if request.META.get('HTTP_CONTENT_ENCODING'):
219
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
220
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
221
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
222
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
223
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
224
    return content_type, meta, get_sharing(request), get_public(request)
225

    
226

    
227
def put_object_headers(response, meta, restricted=False, token=None):
228
    response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
229
    response['Content-Length'] = meta['bytes']
230
    response.override_serialization = True
231
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
232
    response['Last-Modified'] = http_date(int(meta['modified']))
233
    if not restricted:
234
        response['X-Object-Hash'] = meta['hash']
235
        response['X-Object-UUID'] = meta['uuid']
236
        if TRANSLATE_UUIDS:
237
            meta['modified_by'] = \
238
                retrieve_displayname(token, meta['modified_by'])
239
        response['X-Object-Modified-By'] = smart_str(
240
            meta['modified_by'], strings_only=True)
241
        response['X-Object-Version'] = meta['version']
242
        response['X-Object-Version-Timestamp'] = http_date(
243
            int(meta['version_timestamp']))
244
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
245
            response[smart_str(
246
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
247
        for k in (
248
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
249
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
250
                'X-Object-Public'):
251
            if k in meta:
252
                response[k] = smart_str(meta[k], strings_only=True)
253
    else:
254
        for k in ('Content-Encoding', 'Content-Disposition'):
255
            if k in meta:
256
                response[k] = smart_str(meta[k], strings_only=True)
257

    
258

    
259
def update_manifest_meta(request, v_account, meta):
260
    """Update metadata if the object has an X-Object-Manifest."""
261

    
262
    if 'X-Object-Manifest' in meta:
263
        etag = ''
264
        bytes = 0
265
        try:
266
            src_container, src_name = split_container_object_string(
267
                '/' + meta['X-Object-Manifest'])
268
            objects = request.backend.list_objects(
269
                request.user_uniq, v_account,
270
                src_container, prefix=src_name, virtual=False)
271
            for x in objects:
272
                src_meta = request.backend.get_object_meta(
273
                    request.user_uniq, v_account, src_container, x[0],
274
                    'pithos', x[1])
275
                etag += (src_meta['hash'] if not UPDATE_MD5 else
276
                         src_meta['checksum'])
277
                bytes += src_meta['bytes']
278
        except:
279
            # Ignore errors.
280
            return
281
        meta['bytes'] = bytes
282
        md5 = hashlib.md5()
283
        md5.update(etag)
284
        meta['checksum'] = md5.hexdigest().lower()
285

    
286

    
287
def is_uuid(str):
288
    if str is None:
289
        return False
290
    try:
291
        uuid.UUID(str)
292
    except ValueError:
293
        return False
294
    else:
295
        return True
296

    
297

    
298
##########################
299
# USER CATALOG utilities #
300
##########################
301

    
302
def retrieve_displayname(token, uuid, fail_silently=True):
303
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
304
                            retry=2, use_pool=True,
305
                            logger=logger)
306
    try:
307
        displayname = astakos.get_username(uuid)
308
    except NoUserName:
309
        if not fail_silently:
310
            raise ItemNotExists(uuid)
311
        else:
312
            # just return the uuid
313
            return uuid
314
    return displayname
315

    
316

    
317
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
318
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
319
                            retry=2, use_pool=True,
320
                            logger=logger)
321
    catalog = astakos.get_usernames(uuids) or {}
322
    missing = list(set(uuids) - set(catalog))
323
    if missing and not fail_silently:
324
        raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
325
    return catalog if return_dict else [catalog.get(i) for i in uuids]
326

    
327

    
328
def retrieve_uuid(token, displayname):
329
    if is_uuid(displayname):
330
        return displayname
331

    
332
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
333
                            retry=2, use_pool=True,
334
                            logger=logger)
335
    try:
336
        uuid = astakos.get_uuid(displayname)
337
    except NoUUID:
338
        raise ItemNotExists(displayname)
339
    return uuid
340

    
341

    
342
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
343
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
344
                            retry=2, use_pool=True,
345
                            logger=logger)
346
    catalog = astakos.get_uuids(displaynames) or {}
347
    missing = list(set(displaynames) - set(catalog))
348
    if missing and not fail_silently:
349
        raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
350
    return catalog if return_dict else [catalog.get(i) for i in displaynames]
351

    
352

    
353
def replace_permissions_displayname(token, holder):
354
    if holder == '*':
355
        return holder
356
    try:
357
        # check first for a group permission
358
        account, group = holder.split(':', 1)
359
    except ValueError:
360
        return retrieve_uuid(token, holder)
361
    else:
362
        return ':'.join([retrieve_uuid(token, account), group])
363

    
364

    
365
def replace_permissions_uuid(token, holder):
366
    if holder == '*':
367
        return holder
368
    try:
369
        # check first for a group permission
370
        account, group = holder.split(':', 1)
371
    except ValueError:
372
        return retrieve_displayname(token, holder)
373
    else:
374
        return ':'.join([retrieve_displayname(token, account), group])
375

    
376

    
377
def update_sharing_meta(request, permissions, v_account,
378
                        v_container, v_object, meta):
379
    if permissions is None:
380
        return
381
    allowed, perm_path, perms = permissions
382
    if len(perms) == 0:
383
        return
384

    
385
    # replace uuid with displayname
386
    if TRANSLATE_UUIDS:
387
        perms['read'] = [replace_permissions_uuid(
388
            getattr(request, 'token', None), x)
389
            for x in perms.get('read', [])]
390
        perms['write'] = [replace_permissions_uuid(
391
            getattr(request, 'token', None), x)
392
            for x in perms.get('write', [])]
393

    
394
    ret = []
395

    
396
    r = ','.join(perms.get('read', []))
397
    if r:
398
        ret.append('read=' + r)
399
    w = ','.join(perms.get('write', []))
400
    if w:
401
        ret.append('write=' + w)
402
    meta['X-Object-Sharing'] = '; '.join(ret)
403
    if '/'.join((v_account, v_container, v_object)) != perm_path:
404
        meta['X-Object-Shared-By'] = perm_path
405
    if request.user_uniq != v_account:
406
        meta['X-Object-Allowed-To'] = allowed
407

    
408

    
409
def update_public_meta(public, meta):
410
    if not public:
411
        return
412
    meta['X-Object-Public'] = join_urls(
413
        BASE_HOST, reverse('pithos.api.public.public_demux', args=(public,)))
414

    
415

    
416
def validate_modification_preconditions(request, meta):
417
    """Check the modified timestamp conforms with the preconditions set."""
418

    
419
    if 'modified' not in meta:
420
        return  # TODO: Always return?
421

    
422
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
423
    if if_modified_since is not None:
424
        if_modified_since = parse_http_date_safe(if_modified_since)
425
    if (if_modified_since is not None
426
            and int(meta['modified']) <= if_modified_since):
427
        raise faults.NotModified('Resource has not been modified')
428

    
429
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
430
    if if_unmodified_since is not None:
431
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
432
    if (if_unmodified_since is not None
433
            and int(meta['modified']) > if_unmodified_since):
434
        raise faults.PreconditionFailed('Resource has been modified')
435

    
436

    
437
def validate_matching_preconditions(request, meta):
438
    """Check that the ETag conforms with the preconditions set."""
439

    
440
    etag = meta['hash'] if not UPDATE_MD5 else meta['checksum']
441
    if not etag:
442
        etag = None
443

    
444
    if_match = request.META.get('HTTP_IF_MATCH')
445
    if if_match is not None:
446
        if etag is None:
447
            raise faults.PreconditionFailed('Resource does not exist')
448
        if (if_match != '*'
449
                and etag not in [x.lower() for x in parse_etags(if_match)]):
450
            raise faults.PreconditionFailed('Resource ETag does not match')
451

    
452
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
453
    if if_none_match is not None:
454
        # TODO: If this passes, must ignore If-Modified-Since header.
455
        if etag is not None:
456
            if (if_none_match == '*' or etag in [x.lower() for x in
457
                                                 parse_etags(if_none_match)]):
458
                # TODO: Continue if an If-Modified-Since header is present.
459
                if request.method in ('HEAD', 'GET'):
460
                    raise faults.NotModified('Resource ETag matches')
461
                raise faults.PreconditionFailed(
462
                    'Resource exists or ETag matches')
463

    
464

    
465
def split_container_object_string(s):
466
    if not len(s) > 0 or s[0] != '/':
467
        raise ValueError
468
    s = s[1:]
469
    pos = s.find('/')
470
    if pos == -1 or pos == len(s) - 1:
471
        raise ValueError
472
    return s[:pos], s[(pos + 1):]
473

    
474

    
475
def copy_or_move_object(request, src_account, src_container, src_name,
476
                        dest_account, dest_container, dest_name,
477
                        move=False, delimiter=None):
478
    """Copy or move an object."""
479

    
480
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
481
        del(request.META['CONTENT_TYPE'])
482
    content_type, meta, permissions, public = get_object_headers(request)
483
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
484
    try:
485
        if move:
486
            version_id = request.backend.move_object(
487
                request.user_uniq, src_account, src_container, src_name,
488
                dest_account, dest_container, dest_name,
489
                content_type, 'pithos', meta, False, permissions, delimiter)
490
        else:
491
            version_id = request.backend.copy_object(
492
                request.user_uniq, src_account, src_container, src_name,
493
                dest_account, dest_container, dest_name,
494
                content_type, 'pithos', meta, False, permissions,
495
                src_version, delimiter)
496
    except NotAllowedError:
497
        raise faults.Forbidden('Not allowed')
498
    except (ItemNotExists, VersionNotExists):
499
        raise faults.ItemNotFound('Container or object does not exist')
500
    except ValueError:
501
        raise faults.BadRequest('Invalid sharing header')
502
    except QuotaError, e:
503
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
504
    if public is not None:
505
        try:
506
            request.backend.update_object_public(
507
                request.user_uniq, dest_account,
508
                dest_container, dest_name, public)
509
        except NotAllowedError:
510
            raise faults.Forbidden('Not allowed')
511
        except ItemNotExists:
512
            raise faults.ItemNotFound('Object does not exist')
513
    return version_id
514

    
515

    
516
def get_int_parameter(p):
517
    if p is not None:
518
        try:
519
            p = int(p)
520
        except ValueError:
521
            return None
522
        if p < 0:
523
            return None
524
    return p
525

    
526

    
527
def get_content_length(request):
528
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
529
    if content_length is None:
530
        raise faults.LengthRequired('Missing or invalid Content-Length header')
531
    return content_length
532

    
533

    
534
def get_range(request, size):
535
    """Parse a Range header from the request.
536

537
    Either returns None, when the header is not existent or should be ignored,
538
    or a list of (offset, length) tuples - should be further checked.
539
    """
540

    
541
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
542
    if not ranges.startswith('bytes='):
543
        return None
544

    
545
    ret = []
546
    for r in (x.strip() for x in ranges[6:].split(',')):
547
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
548
        m = p.match(r)
549
        if not m:
550
            return None
551
        offset = m.group('offset')
552
        upto = m.group('upto')
553
        if offset == '' and upto == '':
554
            return None
555

    
556
        if offset != '':
557
            offset = int(offset)
558
            if upto != '':
559
                upto = int(upto)
560
                if offset > upto:
561
                    return None
562
                ret.append((offset, upto - offset + 1))
563
            else:
564
                ret.append((offset, size - offset))
565
        else:
566
            length = int(upto)
567
            ret.append((size - length, length))
568

    
569
    return ret
570

    
571

    
572
def get_content_range(request):
573
    """Parse a Content-Range header from the request.
574

575
    Either returns None, when the header is not existent or should be ignored,
576
    or an (offset, length, total) tuple - check as length, total may be None.
577
    Returns (None, None, None) if the provided range is '*/*'.
578
    """
579

    
580
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
581
    if not ranges:
582
        return None
583

    
584
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
585
    m = p.match(ranges)
586
    if not m:
587
        if ranges == 'bytes */*':
588
            return (None, None, None)
589
        return None
590
    offset = int(m.group('offset'))
591
    upto = m.group('upto')
592
    total = m.group('total')
593
    if upto != '':
594
        upto = int(upto)
595
    else:
596
        upto = None
597
    if total != '*':
598
        total = int(total)
599
    else:
600
        total = None
601
    if (upto is not None and offset > upto) or \
602
        (total is not None and offset >= total) or \
603
            (total is not None and upto is not None and upto >= total):
604
        return None
605

    
606
    if upto is None:
607
        length = None
608
    else:
609
        length = upto - offset + 1
610
    return (offset, length, total)
611

    
612

    
613
def get_sharing(request):
614
    """Parse an X-Object-Sharing header from the request.
615

616
    Raises BadRequest on error.
617
    """
618

    
619
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
620
    if permissions is None:
621
        return None
622

    
623
    # TODO: Document or remove '~' replacing.
624
    permissions = permissions.replace('~', '')
625

    
626
    ret = {}
627
    permissions = permissions.replace(' ', '')
628
    if permissions == '':
629
        return ret
630
    for perm in (x for x in permissions.split(';')):
631
        if perm.startswith('read='):
632
            ret['read'] = list(set(
633
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
634
            if '' in ret['read']:
635
                ret['read'].remove('')
636
            if '*' in ret['read']:
637
                ret['read'] = ['*']
638
            if len(ret['read']) == 0:
639
                raise faults.BadRequest(
640
                    'Bad X-Object-Sharing header value: invalid length')
641
        elif perm.startswith('write='):
642
            ret['write'] = list(set(
643
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
644
            if '' in ret['write']:
645
                ret['write'].remove('')
646
            if '*' in ret['write']:
647
                ret['write'] = ['*']
648
            if len(ret['write']) == 0:
649
                raise faults.BadRequest(
650
                    'Bad X-Object-Sharing header value: invalid length')
651
        else:
652
            raise faults.BadRequest(
653
                'Bad X-Object-Sharing header value: missing prefix')
654

    
655
    # replace displayname with uuid
656
    if TRANSLATE_UUIDS:
657
        try:
658
            ret['read'] = [replace_permissions_displayname(
659
                getattr(request, 'token', None), x)
660
                for x in ret.get('read', [])]
661
            ret['write'] = [replace_permissions_displayname(
662
                getattr(request, 'token', None), x)
663
                for x in ret.get('write', [])]
664
        except ItemNotExists, e:
665
            raise faults.BadRequest(
666
                'Bad X-Object-Sharing header value: unknown account: %s' % e)
667

    
668
    # Keep duplicates only in write list.
669
    dups = [x for x in ret.get(
670
        'read', []) if x in ret.get('write', []) and x != '*']
671
    if dups:
672
        for x in dups:
673
            ret['read'].remove(x)
674
        if len(ret['read']) == 0:
675
            del(ret['read'])
676

    
677
    return ret
678

    
679

    
680
def get_public(request):
681
    """Parse an X-Object-Public header from the request.
682

683
    Raises BadRequest on error.
684
    """
685

    
686
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
687
    if public is None:
688
        return None
689

    
690
    public = public.replace(' ', '').lower()
691
    if public == 'true':
692
        return True
693
    elif public == 'false' or public == '':
694
        return False
695
    raise faults.BadRequest('Bad X-Object-Public header value')
696

    
697

    
698
def raw_input_socket(request):
699
    """Return the socket for reading the rest of the request."""
700

    
701
    server_software = request.META.get('SERVER_SOFTWARE')
702
    if server_software and server_software.startswith('mod_python'):
703
        return request._req
704
    if 'wsgi.input' in request.environ:
705
        return request.environ['wsgi.input']
706
    raise NotImplemented('Unknown server software')
707

    
708
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB
709

    
710

    
711
def socket_read_iterator(request, length=0, blocksize=4096):
712
    """Return maximum of blocksize data read from the socket in each iteration
713

714
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
715
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
716
    """
717

    
718
    sock = raw_input_socket(request)
719
    if length < 0:  # Chunked transfers
720
        # Small version (server does the dechunking).
721
        if (request.environ.get('mod_wsgi.input_chunked', None)
722
                or request.META['SERVER_SOFTWARE'].startswith('gunicorn')):
723
            while length < MAX_UPLOAD_SIZE:
724
                data = sock.read(blocksize)
725
                if data == '':
726
                    return
727
                yield data
728
            raise faults.BadRequest('Maximum size is reached')
729

    
730
        # Long version (do the dechunking).
731
        data = ''
732
        while length < MAX_UPLOAD_SIZE:
733
            # Get chunk size.
734
            if hasattr(sock, 'readline'):
735
                chunk_length = sock.readline()
736
            else:
737
                chunk_length = ''
738
                while chunk_length[-1:] != '\n':
739
                    chunk_length += sock.read(1)
740
                chunk_length.strip()
741
            pos = chunk_length.find(';')
742
            if pos >= 0:
743
                chunk_length = chunk_length[:pos]
744
            try:
745
                chunk_length = int(chunk_length, 16)
746
            except Exception:
747
                raise faults.BadRequest('Bad chunk size')
748
                                 # TODO: Change to something more appropriate.
749
            # Check if done.
750
            if chunk_length == 0:
751
                if len(data) > 0:
752
                    yield data
753
                return
754
            # Get the actual data.
755
            while chunk_length > 0:
756
                chunk = sock.read(min(chunk_length, blocksize))
757
                chunk_length -= len(chunk)
758
                if length > 0:
759
                    length += len(chunk)
760
                data += chunk
761
                if len(data) >= blocksize:
762
                    ret = data[:blocksize]
763
                    data = data[blocksize:]
764
                    yield ret
765
            sock.read(2)  # CRLF
766
        raise faults.BadRequest('Maximum size is reached')
767
    else:
768
        if length > MAX_UPLOAD_SIZE:
769
            raise faults.BadRequest('Maximum size is reached')
770
        while length > 0:
771
            data = sock.read(min(length, blocksize))
772
            if not data:
773
                raise faults.BadRequest()
774
            length -= len(data)
775
            yield data
776

    
777

    
778
class SaveToBackendHandler(FileUploadHandler):
779
    """Handle a file from an HTML form the django way."""
780

    
781
    def __init__(self, request=None):
782
        super(SaveToBackendHandler, self).__init__(request)
783
        self.backend = request.backend
784

    
785
    def put_data(self, length):
786
        if len(self.data) >= length:
787
            block = self.data[:length]
788
            self.file.hashmap.append(self.backend.put_block(block))
789
            self.checksum_compute.update(block)
790
            self.data = self.data[length:]
791

    
792
    def new_file(self, field_name, file_name, content_type,
793
                 content_length, charset=None):
794
        self.checksum_compute = NoChecksum() if not UPDATE_MD5 else Checksum()
795
        self.data = ''
796
        self.file = UploadedFile(
797
            name=file_name, content_type=content_type, charset=charset)
798
        self.file.size = 0
799
        self.file.hashmap = []
800

    
801
    def receive_data_chunk(self, raw_data, start):
802
        self.data += raw_data
803
        self.file.size += len(raw_data)
804
        self.put_data(self.request.backend.block_size)
805
        return None
806

    
807
    def file_complete(self, file_size):
808
        l = len(self.data)
809
        if l > 0:
810
            self.put_data(l)
811
        self.file.etag = self.checksum_compute.hexdigest()
812
        return self.file
813

    
814

    
815
class ObjectWrapper(object):
816
    """Return the object's data block-per-block in each iteration.
817

818
    Read from the object using the offset and length provided
819
    in each entry of the range list.
820
    """
821

    
822
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
823
        self.backend = backend
824
        self.ranges = ranges
825
        self.sizes = sizes
826
        self.hashmaps = hashmaps
827
        self.boundary = boundary
828
        self.size = sum(self.sizes)
829

    
830
        self.file_index = 0
831
        self.block_index = 0
832
        self.block_hash = -1
833
        self.block = ''
834

    
835
        self.range_index = -1
836
        self.offset, self.length = self.ranges[0]
837

    
838
    def __iter__(self):
839
        return self
840

    
841
    def part_iterator(self):
842
        if self.length > 0:
843
            # Get the file for the current offset.
844
            file_size = self.sizes[self.file_index]
845
            while self.offset >= file_size:
846
                self.offset -= file_size
847
                self.file_index += 1
848
                file_size = self.sizes[self.file_index]
849

    
850
            # Get the block for the current position.
851
            self.block_index = int(self.offset / self.backend.block_size)
852
            if self.block_hash != \
853
                    self.hashmaps[self.file_index][self.block_index]:
854
                self.block_hash = self.hashmaps[
855
                    self.file_index][self.block_index]
856
                try:
857
                    self.block = self.backend.get_block(self.block_hash)
858
                except ItemNotExists:
859
                    raise faults.ItemNotFound('Block does not exist')
860

    
861
            # Get the data from the block.
862
            bo = self.offset % self.backend.block_size
863
            bs = self.backend.block_size
864
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
865
                    self.sizes[self.file_index] % self.backend.block_size):
866
                bs = self.sizes[self.file_index] % self.backend.block_size
867
            bl = min(self.length, bs - bo)
868
            data = self.block[bo:bo + bl]
869
            self.offset += bl
870
            self.length -= bl
871
            return data
872
        else:
873
            raise StopIteration
874

    
875
    def next(self):
876
        if len(self.ranges) == 1:
877
            return self.part_iterator()
878
        if self.range_index == len(self.ranges):
879
            raise StopIteration
880
        try:
881
            if self.range_index == -1:
882
                raise StopIteration
883
            return self.part_iterator()
884
        except StopIteration:
885
            self.range_index += 1
886
            out = []
887
            if self.range_index < len(self.ranges):
888
                # Part header.
889
                self.offset, self.length = self.ranges[self.range_index]
890
                self.file_index = 0
891
                if self.range_index > 0:
892
                    out.append('')
893
                out.append('--' + self.boundary)
894
                out.append('Content-Range: bytes %d-%d/%d' % (
895
                    self.offset, self.offset + self.length - 1, self.size))
896
                out.append('Content-Transfer-Encoding: binary')
897
                out.append('')
898
                out.append('')
899
                return '\r\n'.join(out)
900
            else:
901
                # Footer.
902
                out.append('')
903
                out.append('--' + self.boundary + '--')
904
                out.append('')
905
                return '\r\n'.join(out)
906

    
907

    
908
def object_data_response(request, sizes, hashmaps, meta, public=False):
909
    """Get the HttpResponse object for replying with the object's data."""
910

    
911
    # Range handling.
912
    size = sum(sizes)
913
    ranges = get_range(request, size)
914
    if ranges is None:
915
        ranges = [(0, size)]
916
        ret = 200
917
    else:
918
        check = [True for offset, length in ranges if
919
                 length <= 0 or length > size or
920
                 offset < 0 or offset >= size or
921
                 offset + length > size]
922
        if len(check) > 0:
923
            raise faults.RangeNotSatisfiable(
924
                'Requested range exceeds object limits')
925
        ret = 206
926
        if_range = request.META.get('HTTP_IF_RANGE')
927
        if if_range:
928
            try:
929
                # Modification time has passed instead.
930
                last_modified = parse_http_date(if_range)
931
                if last_modified != meta['modified']:
932
                    ranges = [(0, size)]
933
                    ret = 200
934
            except ValueError:
935
                if if_range != meta['checksum']:
936
                    ranges = [(0, size)]
937
                    ret = 200
938

    
939
    if ret == 206 and len(ranges) > 1:
940
        boundary = uuid.uuid4().hex
941
    else:
942
        boundary = ''
943
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
944
    response = HttpResponse(wrapper, status=ret)
945
    put_object_headers(
946
        response, meta, restricted=public,
947
        token=getattr(request, 'token', None))
948
    if ret == 206:
949
        if len(ranges) == 1:
950
            offset, length = ranges[0]
951
            response[
952
                'Content-Length'] = length  # Update with the correct length.
953
            response['Content-Range'] = 'bytes %d-%d/%d' % (
954
                offset, offset + length - 1, size)
955
        else:
956
            del(response['Content-Length'])
957
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (
958
                boundary,)
959
    return response
960

    
961

    
962
def put_object_block(request, hashmap, data, offset):
963
    """Put one block of data at the given offset."""
964

    
965
    bi = int(offset / request.backend.block_size)
966
    bo = offset % request.backend.block_size
967
    bl = min(len(data), request.backend.block_size - bo)
968
    if bi < len(hashmap):
969
        hashmap[bi] = request.backend.update_block(hashmap[bi], data[:bl], bo)
970
    else:
971
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
972
    return bl  # Return ammount of data written.
973

    
974

    
975
def hashmap_md5(backend, hashmap, size):
976
    """Produce the MD5 sum from the data in the hashmap."""
977

    
978
    # TODO: Search backend for the MD5 of another object
979
    #       with the same hashmap and size...
980
    md5 = hashlib.md5()
981
    bs = backend.block_size
982
    for bi, hash in enumerate(hashmap):
983
        data = backend.get_block(hash)  # Blocks come in padded.
984
        if bi == len(hashmap) - 1:
985
            data = data[:size % bs]
986
        md5.update(data)
987
    return md5.hexdigest().lower()
988

    
989

    
990
def simple_list_response(request, l):
991
    if request.serialization == 'text':
992
        return '\n'.join(l) + '\n'
993
    if request.serialization == 'xml':
994
        return render_to_string('items.xml', {'items': l})
995
    if request.serialization == 'json':
996
        return json.dumps(l)
997

    
998

    
999
from pithos.backends.util import PithosBackendPool
1000

    
1001
if RADOS_STORAGE:
1002
    BLOCK_PARAMS = {'mappool': RADOS_POOL_MAPS,
1003
                    'blockpool': RADOS_POOL_BLOCKS, }
1004
else:
1005
    BLOCK_PARAMS = {'mappool': None,
1006
                    'blockpool': None, }
1007

    
1008
BACKEND_KWARGS = dict(
1009
    db_module=BACKEND_DB_MODULE,
1010
    db_connection=BACKEND_DB_CONNECTION,
1011
    block_module=BACKEND_BLOCK_MODULE,
1012
    block_path=BACKEND_BLOCK_PATH,
1013
    block_umask=BACKEND_BLOCK_UMASK,
1014
    block_size=BACKEND_BLOCK_SIZE,
1015
    hash_algorithm=BACKEND_HASH_ALGORITHM,
1016
    queue_module=BACKEND_QUEUE_MODULE,
1017
    queue_hosts=BACKEND_QUEUE_HOSTS,
1018
    queue_exchange=BACKEND_QUEUE_EXCHANGE,
1019
    astakos_auth_url=ASTAKOS_AUTH_URL,
1020
    service_token=SERVICE_TOKEN,
1021
    astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
1022
    free_versioning=BACKEND_FREE_VERSIONING,
1023
    block_params=BLOCK_PARAMS,
1024
    public_url_security=PUBLIC_URL_SECURITY,
1025
    public_url_alphabet=PUBLIC_URL_ALPHABET,
1026
    account_quota_policy=BACKEND_ACCOUNT_QUOTA,
1027
    container_quota_policy=BACKEND_CONTAINER_QUOTA,
1028
    container_versioning_policy=BACKEND_VERSIONING)
1029

    
1030
_pithos_backend_pool = PithosBackendPool(size=BACKEND_POOL_SIZE,
1031
                                         **BACKEND_KWARGS)
1032

    
1033

    
1034
def get_backend():
1035
    if BACKEND_POOL_ENABLED:
1036
        backend = _pithos_backend_pool.pool_get()
1037
    else:
1038
        backend = connect_backend(**BACKEND_KWARGS)
1039
    backend.serials = []
1040
    backend.messages = []
1041
    return backend
1042

    
1043

    
1044
def update_request_headers(request):
1045
    # Handle URL-encoded keys and values.
1046
    meta = dict([(
1047
        k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
1048
    for k, v in meta.iteritems():
1049
        try:
1050
            k.decode('ascii')
1051
            v.decode('ascii')
1052
        except UnicodeDecodeError:
1053
            raise faults.BadRequest('Bad character in headers.')
1054
        if '%' in k or '%' in v:
1055
            del(request.META[k])
1056
            request.META[unquote(k)] = smart_unicode(unquote(
1057
                v), strings_only=True)
1058

    
1059

    
1060
def update_response_headers(request, response):
1061
    # URL-encode unicode in headers.
1062
    meta = response.items()
1063
    for k, v in meta:
1064
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
1065
                k.startswith('X-Object-') or k.startswith('Content-')):
1066
            del(response[k])
1067
            response[quote(k)] = quote(v, safe='/=,:@; ')
1068

    
1069

    
1070
def get_pithos_usage(token):
1071
    """Get Pithos Usage from astakos."""
1072
    astakos = AstakosClient(token, ASTAKOS_AUTH_URL,
1073
                            retry=2, use_pool=True,
1074
                            logger=logger)
1075
    quotas = astakos.get_quotas()['system']
1076
    pithos_resources = [r['name'] for r in resources]
1077
    map(quotas.pop, filter(lambda k: k not in pithos_resources, quotas.keys()))
1078
    return quotas.popitem()[-1]  # assume only one resource
1079

    
1080

    
1081
def api_method(http_method=None, token_required=True, user_required=True,
1082
               logger=None, format_allowed=False, serializations=None,
1083
               strict_serlization=False, lock_container_path=False):
1084
    serializations = serializations or ['json', 'xml']
1085

    
1086
    def decorator(func):
1087
        @api.api_method(http_method=http_method, token_required=token_required,
1088
                        user_required=user_required,
1089
                        logger=logger, format_allowed=format_allowed,
1090
                        astakos_auth_url=ASTAKOS_AUTH_URL,
1091
                        serializations=serializations,
1092
                        strict_serlization=strict_serlization)
1093
        @wraps(func)
1094
        def wrapper(request, *args, **kwargs):
1095
            # The args variable may contain up to (account, container, object).
1096
            if len(args) > 1 and len(args[1]) > 256:
1097
                raise faults.BadRequest("Container name too large")
1098
            if len(args) > 2 and len(args[2]) > 1024:
1099
                raise faults.BadRequest('Object name too large.')
1100

    
1101
            success_status = False
1102
            try:
1103
                # Add a PithosBackend as attribute of the request object
1104
                request.backend = get_backend()
1105
                request.backend.pre_exec(lock_container_path)
1106

    
1107
                # Many API method expect thet X-Auth-Token in request,token
1108
                request.token = request.x_auth_token
1109
                update_request_headers(request)
1110
                response = func(request, *args, **kwargs)
1111
                update_response_headers(request, response)
1112

    
1113
                success_status = True
1114
                return response
1115
            finally:
1116
                # Always close PithosBackend connection
1117
                if getattr(request, "backend", None) is not None:
1118
                    request.backend.post_exec(success_status)
1119
                    request.backend.close()
1120
        return wrapper
1121
    return decorator
1122

    
1123

    
1124
def restrict_to_host(host=None):
1125
    """
1126
    View decorator which restricts wrapped view to be accessed only under the
1127
    host set. If an invalid host is identified and request HTTP method is one
1128
    of ``GET``, ``HOST``, the decorator will return a redirect response using a
1129
    clone of the request with host replaced to the one the restriction applies
1130
    to.
1131

1132
    e.g.
1133
    @restrict_to_host('files.example.com')
1134
    my_restricted_view(request, path):
1135
        return HttpResponse(file(path).read())
1136

1137
    A get to ``https://api.example.com/my_restricted_view/file_path/?param=1``
1138
    will return a redirect response with Location header set to
1139
    ``https://files.example.com/my_restricted_view/file_path/?param=1``.
1140

1141
    If host is set to ``None`` no restriction will be applied.
1142
    """
1143
    def decorator(func):
1144
        # skip decoration if no host is set
1145
        if not host:
1146
            return func
1147

    
1148
        @wraps(func)
1149
        def wrapper(request, *args, **kwargs):
1150
            request_host = request.get_host()
1151
            if host != request_host:
1152
                proto = 'https' if request.is_secure() else 'http'
1153
                if request.method in ['GET', 'HEAD']:
1154
                    full_path = request.get_full_path()
1155
                    redirect_uri = "%s://%s%s" % (proto, host, full_path)
1156
                    return HttpResponseRedirect(redirect_uri)
1157
                else:
1158
                    raise PermissionDenied
1159
            return func(request, *args, **kwargs)
1160
        return wrapper
1161
    return decorator
1162

    
1163

    
1164
def view_method():
1165
    """Decorator function for views."""
1166

    
1167
    def decorator(func):
1168
        @restrict_to_host(SERVE_API_DOMAIN)
1169
        @wraps(func)
1170
        def wrapper(request, *args, **kwargs):
1171
            if request.method not in ['GET', 'HEAD']:
1172
                return HttpResponseNotAllowed(['GET', 'HEAD'])
1173

    
1174
            try:
1175
                access_token = request.GET.get('access_token')
1176
                requested_resource = request.path.split(VIEW_PREFIX, 2)[-1]
1177
                astakos = AstakosClient(SERVICE_TOKEN, ASTAKOS_AUTH_URL,
1178
                                        retry=2, use_pool=True,
1179
                                        logger=logger)
1180
                if access_token is not None:
1181
                    # authenticate using the short-term access token
1182
                    try:
1183
                        request.user = astakos.validate_token(
1184
                            access_token, requested_resource)
1185
                    except AstakosClientException:
1186
                        return HttpResponseRedirect(request.path)
1187
                    request.user_uniq = request.user["access"]["user"]["id"]
1188

    
1189
                    _func = api_method(token_required=False,
1190
                                       user_required=False)(func)
1191
                    response = _func(request, *args, **kwargs)
1192
                    if response.status_code == 404:
1193
                        raise Http404
1194
                    elif response.status_code == 403:
1195
                        raise PermissionDenied
1196
                    return response
1197

    
1198
                client_id, client_secret = OAUTH2_CLIENT_CREDENTIALS
1199
                # TODO: check if client credentials are not set
1200
                authorization_code = request.GET.get('code')
1201
                if authorization_code is None:
1202
                    # request authorization code
1203
                    params = {'response_type': 'code',
1204
                              'client_id': client_id,
1205
                              'redirect_uri':
1206
                              request.build_absolute_uri(request.path),
1207
                              'state': '',  # TODO include state for security
1208
                              'scope': request.path.split(VIEW_PREFIX, 2)[-1]}
1209
                    return HttpResponseRedirect('%s?%s' %
1210
                                                (join_urls(astakos.oauth2_url,
1211
                                                           'auth'),
1212
                                                 urlencode(params)))
1213
                else:
1214
                    # request short-term access token
1215
                    redirect_uri = request.build_absolute_uri(request.path)
1216
                    data = astakos.get_token('authorization_code',
1217
                                             *OAUTH2_CLIENT_CREDENTIALS,
1218
                                             redirect_uri=redirect_uri,
1219
                                             scope=requested_resource,
1220
                                             code=authorization_code)
1221
                    params = {'access_token': data.get('access_token', '')}
1222
                    return HttpResponseRedirect('%s?%s' % (redirect_uri,
1223
                                                           urlencode(params)))
1224
            except AstakosClientException, err:
1225
                logger.exception(err)
1226
                raise PermissionDenied
1227
        return wrapper
1228
    return decorator
1229

    
1230

    
1231
class Checksum:
1232
    def __init__(self):
1233
        self.md5 = hashlib.md5()
1234

    
1235
    def update(self, data):
1236
        self.md5.update(data)
1237

    
1238
    def hexdigest(self):
1239
        return self.md5.hexdigest().lower()
1240

    
1241

    
1242
class NoChecksum:
1243
    def update(self, data):
1244
        pass
1245

    
1246
    def hexdigest(self):
1247
        return ''