Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ 981d3b0d

History | View | Annotate | Download (40.6 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from functools import wraps
35
from datetime import datetime
36
from urllib import quote, unquote
37

    
38
from django.http import (HttpResponse, HttpResponseRedirect, Http404,
39
                         HttpResponseForbidden)
40
from django.template.loader import render_to_string
41
from django.utils import simplejson as json
42
from django.utils.http import http_date, parse_etags
43
from django.utils.encoding import smart_unicode, smart_str
44
from django.core.files.uploadhandler import FileUploadHandler
45
from django.core.files.uploadedfile import UploadedFile
46
from django.core.urlresolvers import reverse
47

    
48
from snf_django.lib.api.parsedate import parse_http_date_safe, parse_http_date
49
from snf_django.lib import api
50
from snf_django.lib.api import faults, utils
51

    
52
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
53
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
54
                                 BACKEND_BLOCK_UMASK,
55
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
56
                                 BACKEND_QUEUE_EXCHANGE,
57
                                 ASTAKOSCLIENT_POOLSIZE,
58
                                 SERVICE_TOKEN,
59
                                 ASTAKOS_BASE_URL,
60
                                 BACKEND_ACCOUNT_QUOTA, BACKEND_CONTAINER_QUOTA,
61
                                 BACKEND_VERSIONING,
62
                                 BACKEND_FREE_VERSIONING, BACKEND_POOL_SIZE,
63
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
64
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
65
                                 PUBLIC_URL_SECURITY,
66
                                 PUBLIC_URL_ALPHABET,
67
                                 COOKIE_NAME, BASE_HOST, LOGIN_URL)
68
from pithos.api.resources import resources
69
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
70
                                  VersionNotExists)
71

    
72
from synnefo.lib import join_urls
73

    
74
from astakosclient import AstakosClient
75
from astakosclient.errors import NoUserName, NoUUID
76

    
77
import logging
78
import re
79
import hashlib
80
import uuid
81
import decimal
82

    
83
logger = logging.getLogger(__name__)
84

    
85

    
86
def json_encode_decimal(obj):
87
    if isinstance(obj, decimal.Decimal):
88
        return str(obj)
89
    raise TypeError(repr(obj) + " is not JSON serializable")
90

    
91

    
92
def rename_meta_key(d, old, new):
93
    if old not in d:
94
        return
95
    d[new] = d[old]
96
    del(d[old])
97

    
98

    
99
def printable_header_dict(d):
100
    """Format a meta dictionary for printing out json/xml.
101

102
    Convert all keys to lower case and replace dashes with underscores.
103
    Format 'last_modified' timestamp.
104
    """
105

    
106
    if 'last_modified' in d and d['last_modified']:
107
        d['last_modified'] = utils.isoformat(
108
            datetime.fromtimestamp(d['last_modified']))
109
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
110

    
111

    
112
def format_header_key(k):
113
    """Convert underscores to dashes and capitalize intra-dash strings."""
114
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
115

    
116

    
117
def get_header_prefix(request, prefix):
118
    """Get all prefix-* request headers in a dict.
119
       Reformat keys with format_header_key()."""
120

    
121
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
122
    # TODO: Document or remove '~' replacing.
123
    return dict([(format_header_key(k[5:]), v.replace('~', ''))
124
                for k, v in request.META.iteritems()
125
                if k.startswith(prefix) and len(k) > len(prefix)])
126

    
127

    
128
def check_meta_headers(meta):
129
    if len(meta) > 90:
130
        raise faults.BadRequest('Too many headers.')
131
    for k, v in meta.iteritems():
132
        if len(k) > 128:
133
            raise faults.BadRequest('Header name too large.')
134
        if len(v) > 256:
135
            raise faults.BadRequest('Header value too large.')
136

    
137

    
138
def get_account_headers(request):
139
    meta = get_header_prefix(request, 'X-Account-Meta-')
140
    check_meta_headers(meta)
141
    groups = {}
142
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
143
        n = k[16:].lower()
144
        if '-' in n or '_' in n:
145
            raise faults.BadRequest('Bad characters in group name')
146
        groups[n] = v.replace(' ', '').split(',')
147
        while '' in groups[n]:
148
            groups[n].remove('')
149
    return meta, groups
150

    
151

    
152
def put_account_headers(response, meta, groups, policy):
153
    if 'count' in meta:
154
        response['X-Account-Container-Count'] = meta['count']
155
    if 'bytes' in meta:
156
        response['X-Account-Bytes-Used'] = meta['bytes']
157
    response['Last-Modified'] = http_date(int(meta['modified']))
158
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
159
        response[smart_str(
160
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
161
    if 'until_timestamp' in meta:
162
        response['X-Account-Until-Timestamp'] = http_date(
163
            int(meta['until_timestamp']))
164
    for k, v in groups.iteritems():
165
        k = smart_str(k, strings_only=True)
166
        k = format_header_key('X-Account-Group-' + k)
167
        v = smart_str(','.join(v), strings_only=True)
168
        response[k] = v
169
    for k, v in policy.iteritems():
170
        response[smart_str(format_header_key('X-Account-Policy-' + k),
171
                 strings_only=True)] = smart_str(v, strings_only=True)
172

    
173

    
174
def get_container_headers(request):
175
    meta = get_header_prefix(request, 'X-Container-Meta-')
176
    check_meta_headers(meta)
177
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in
178
                  get_header_prefix(request,
179
                                    'X-Container-Policy-').iteritems()])
180
    return meta, policy
181

    
182

    
183
def put_container_headers(request, response, meta, policy):
184
    if 'count' in meta:
185
        response['X-Container-Object-Count'] = meta['count']
186
    if 'bytes' in meta:
187
        response['X-Container-Bytes-Used'] = meta['bytes']
188
    response['Last-Modified'] = http_date(int(meta['modified']))
189
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
190
        response[smart_str(
191
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
192
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
193
         if x.startswith('X-Object-Meta-')]
194
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
195
    response['X-Container-Block-Size'] = request.backend.block_size
196
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
197
    if 'until_timestamp' in meta:
198
        response['X-Container-Until-Timestamp'] = http_date(
199
            int(meta['until_timestamp']))
200
    for k, v in policy.iteritems():
201
        response[smart_str(format_header_key('X-Container-Policy-' + k),
202
                           strings_only=True)] = smart_str(v, strings_only=True)
203

    
204

    
205
def get_object_headers(request):
206
    content_type = request.META.get('CONTENT_TYPE', None)
207
    meta = get_header_prefix(request, 'X-Object-Meta-')
208
    check_meta_headers(meta)
209
    if request.META.get('HTTP_CONTENT_ENCODING'):
210
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
211
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
212
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
213
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
214
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
215
    return content_type, meta, get_sharing(request), get_public(request)
216

    
217

    
218
def put_object_headers(response, meta, restricted=False, token=None):
219
    response['ETag'] = meta['checksum']
220
    response['Content-Length'] = meta['bytes']
221
    response.override_serialization = True
222
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
223
    response['Last-Modified'] = http_date(int(meta['modified']))
224
    if not restricted:
225
        response['X-Object-Hash'] = meta['hash']
226
        response['X-Object-UUID'] = meta['uuid']
227
        if TRANSLATE_UUIDS:
228
            meta['modified_by'] = \
229
                retrieve_displayname(token, meta['modified_by'])
230
        response['X-Object-Modified-By'] = smart_str(
231
            meta['modified_by'], strings_only=True)
232
        response['X-Object-Version'] = meta['version']
233
        response['X-Object-Version-Timestamp'] = http_date(
234
            int(meta['version_timestamp']))
235
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
236
            response[smart_str(
237
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
238
        for k in (
239
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
240
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
241
                'X-Object-Public'):
242
            if k in meta:
243
                response[k] = smart_str(meta[k], strings_only=True)
244
    else:
245
        for k in ('Content-Encoding', 'Content-Disposition'):
246
            if k in meta:
247
                response[k] = smart_str(meta[k], strings_only=True)
248

    
249

    
250
def update_manifest_meta(request, v_account, meta):
251
    """Update metadata if the object has an X-Object-Manifest."""
252

    
253
    if 'X-Object-Manifest' in meta:
254
        etag = ''
255
        bytes = 0
256
        try:
257
            src_container, src_name = split_container_object_string(
258
                '/' + meta['X-Object-Manifest'])
259
            objects = request.backend.list_objects(
260
                request.user_uniq, v_account,
261
                src_container, prefix=src_name, virtual=False)
262
            for x in objects:
263
                src_meta = request.backend.get_object_meta(request.user_uniq,
264
                                                           v_account,
265
                                                           src_container,
266
                                                           x[0], 'pithos', x[1])
267
                etag += src_meta['checksum']
268
                bytes += src_meta['bytes']
269
        except:
270
            # Ignore errors.
271
            return
272
        meta['bytes'] = bytes
273
        md5 = hashlib.md5()
274
        md5.update(etag)
275
        meta['checksum'] = md5.hexdigest().lower()
276

    
277

    
278
def is_uuid(str):
279
    if str is None:
280
        return False
281
    try:
282
        uuid.UUID(str)
283
    except ValueError:
284
        return False
285
    else:
286
        return True
287

    
288

    
289
##########################
290
# USER CATALOG utilities #
291
##########################
292

    
293
def retrieve_displayname(token, uuid, fail_silently=True):
294
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
295
                            logger=logger)
296
    try:
297
        displayname = astakos.get_username(token, uuid)
298
    except NoUserName:
299
        if not fail_silently:
300
            raise ItemNotExists(uuid)
301
        else:
302
            # just return the uuid
303
            return uuid
304
    return displayname
305

    
306

    
307
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
308
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
309
                            logger=logger)
310
    catalog = astakos.get_usernames(token, uuids) or {}
311
    missing = list(set(uuids) - set(catalog))
312
    if missing and not fail_silently:
313
        raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
314
    return catalog if return_dict else [catalog.get(i) for i in uuids]
315

    
316

    
317
def retrieve_uuid(token, displayname):
318
    if is_uuid(displayname):
319
        return displayname
320

    
321
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
322
                            logger=logger)
323
    try:
324
        uuid = astakos.get_uuid(token, displayname)
325
    except NoUUID:
326
        raise ItemNotExists(displayname)
327
    return uuid
328

    
329

    
330
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
331
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
332
                            logger=logger)
333
    catalog = astakos.get_uuids(token, displaynames) or {}
334
    missing = list(set(displaynames) - set(catalog))
335
    if missing and not fail_silently:
336
        raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
337
    return catalog if return_dict else [catalog.get(i) for i in displaynames]
338

    
339

    
340
def replace_permissions_displayname(token, holder):
341
    if holder == '*':
342
        return holder
343
    try:
344
        # check first for a group permission
345
        account, group = holder.split(':', 1)
346
    except ValueError:
347
        return retrieve_uuid(token, holder)
348
    else:
349
        return ':'.join([retrieve_uuid(token, account), group])
350

    
351

    
352
def replace_permissions_uuid(token, holder):
353
    if holder == '*':
354
        return holder
355
    try:
356
        # check first for a group permission
357
        account, group = holder.split(':', 1)
358
    except ValueError:
359
        return retrieve_displayname(token, holder)
360
    else:
361
        return ':'.join([retrieve_displayname(token, account), group])
362

    
363

    
364
def update_sharing_meta(request, permissions, v_account,
365
                        v_container, v_object, meta):
366
    if permissions is None:
367
        return
368
    allowed, perm_path, perms = permissions
369
    if len(perms) == 0:
370
        return
371

    
372
    # replace uuid with displayname
373
    if TRANSLATE_UUIDS:
374
        perms['read'] = [replace_permissions_uuid(
375
            getattr(request, 'token', None), x)
376
            for x in perms.get('read', [])]
377
        perms['write'] = [replace_permissions_uuid(
378
            getattr(request, 'token', None), x)
379
            for x in perms.get('write', [])]
380

    
381
    ret = []
382

    
383
    r = ','.join(perms.get('read', []))
384
    if r:
385
        ret.append('read=' + r)
386
    w = ','.join(perms.get('write', []))
387
    if w:
388
        ret.append('write=' + w)
389
    meta['X-Object-Sharing'] = '; '.join(ret)
390
    if '/'.join((v_account, v_container, v_object)) != perm_path:
391
        meta['X-Object-Shared-By'] = perm_path
392
    if request.user_uniq != v_account:
393
        meta['X-Object-Allowed-To'] = allowed
394

    
395

    
396
def update_public_meta(public, meta):
397
    if not public:
398
        return
399
    meta['X-Object-Public'] = join_urls(
400
        BASE_HOST, reverse('pithos.api.public.public_demux', args=(public,)))
401

    
402

    
403
def validate_modification_preconditions(request, meta):
404
    """Check that the modified timestamp conforms with the preconditions set."""
405

    
406
    if 'modified' not in meta:
407
        return  # TODO: Always return?
408

    
409
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
410
    if if_modified_since is not None:
411
        if_modified_since = parse_http_date_safe(if_modified_since)
412
    if (if_modified_since is not None
413
            and int(meta['modified']) <= if_modified_since):
414
        raise faults.NotModified('Resource has not been modified')
415

    
416
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
417
    if if_unmodified_since is not None:
418
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
419
    if (if_unmodified_since is not None
420
            and int(meta['modified']) > if_unmodified_since):
421
        raise faults.PreconditionFailed('Resource has been modified')
422

    
423

    
424
def validate_matching_preconditions(request, meta):
425
    """Check that the ETag conforms with the preconditions set."""
426

    
427
    etag = meta['checksum']
428
    if not etag:
429
        etag = None
430

    
431
    if_match = request.META.get('HTTP_IF_MATCH')
432
    if if_match is not None:
433
        if etag is None:
434
            raise faults.PreconditionFailed('Resource does not exist')
435
        if (if_match != '*'
436
                and etag not in [x.lower() for x in parse_etags(if_match)]):
437
            raise faults.PreconditionFailed('Resource ETag does not match')
438

    
439
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
440
    if if_none_match is not None:
441
        # TODO: If this passes, must ignore If-Modified-Since header.
442
        if etag is not None:
443
            if (if_none_match == '*'
444
                    or etag in [x.lower() for x in parse_etags(if_none_match)]):
445
                # TODO: Continue if an If-Modified-Since header is present.
446
                if request.method in ('HEAD', 'GET'):
447
                    raise faults.NotModified('Resource ETag matches')
448
                raise faults.PreconditionFailed(
449
                    'Resource exists or ETag matches')
450

    
451

    
452
def split_container_object_string(s):
453
    if not len(s) > 0 or s[0] != '/':
454
        raise ValueError
455
    s = s[1:]
456
    pos = s.find('/')
457
    if pos == -1 or pos == len(s) - 1:
458
        raise ValueError
459
    return s[:pos], s[(pos + 1):]
460

    
461

    
462
def copy_or_move_object(request, src_account, src_container, src_name,
463
                        dest_account, dest_container, dest_name,
464
                        move=False, delimiter=None):
465
    """Copy or move an object."""
466

    
467
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
468
        del(request.META['CONTENT_TYPE'])
469
    content_type, meta, permissions, public = get_object_headers(request)
470
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
471
    try:
472
        if move:
473
            version_id = request.backend.move_object(
474
                request.user_uniq, src_account, src_container, src_name,
475
                dest_account, dest_container, dest_name,
476
                content_type, 'pithos', meta, False, permissions, delimiter)
477
        else:
478
            version_id = request.backend.copy_object(
479
                request.user_uniq, src_account, src_container, src_name,
480
                dest_account, dest_container, dest_name,
481
                content_type, 'pithos', meta, False, permissions,
482
                src_version, delimiter)
483
    except NotAllowedError:
484
        raise faults.Forbidden('Not allowed')
485
    except (ItemNotExists, VersionNotExists):
486
        raise faults.ItemNotFound('Container or object does not exist')
487
    except ValueError:
488
        raise faults.BadRequest('Invalid sharing header')
489
    except QuotaError, e:
490
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
491
    if public is not None:
492
        try:
493
            request.backend.update_object_public(
494
                request.user_uniq, dest_account,
495
                dest_container, dest_name, public)
496
        except NotAllowedError:
497
            raise faults.Forbidden('Not allowed')
498
        except ItemNotExists:
499
            raise faults.ItemNotFound('Object does not exist')
500
    return version_id
501

    
502

    
503
def get_int_parameter(p):
504
    if p is not None:
505
        try:
506
            p = int(p)
507
        except ValueError:
508
            return None
509
        if p < 0:
510
            return None
511
    return p
512

    
513

    
514
def get_content_length(request):
515
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
516
    if content_length is None:
517
        raise faults.LengthRequired('Missing or invalid Content-Length header')
518
    return content_length
519

    
520

    
521
def get_range(request, size):
522
    """Parse a Range header from the request.
523

524
    Either returns None, when the header is not existent or should be ignored,
525
    or a list of (offset, length) tuples - should be further checked.
526
    """
527

    
528
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
529
    if not ranges.startswith('bytes='):
530
        return None
531

    
532
    ret = []
533
    for r in (x.strip() for x in ranges[6:].split(',')):
534
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
535
        m = p.match(r)
536
        if not m:
537
            return None
538
        offset = m.group('offset')
539
        upto = m.group('upto')
540
        if offset == '' and upto == '':
541
            return None
542

    
543
        if offset != '':
544
            offset = int(offset)
545
            if upto != '':
546
                upto = int(upto)
547
                if offset > upto:
548
                    return None
549
                ret.append((offset, upto - offset + 1))
550
            else:
551
                ret.append((offset, size - offset))
552
        else:
553
            length = int(upto)
554
            ret.append((size - length, length))
555

    
556
    return ret
557

    
558

    
559
def get_content_range(request):
560
    """Parse a Content-Range header from the request.
561

562
    Either returns None, when the header is not existent or should be ignored,
563
    or an (offset, length, total) tuple - check as length, total may be None.
564
    Returns (None, None, None) if the provided range is '*/*'.
565
    """
566

    
567
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
568
    if not ranges:
569
        return None
570

    
571
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
572
    m = p.match(ranges)
573
    if not m:
574
        if ranges == 'bytes */*':
575
            return (None, None, None)
576
        return None
577
    offset = int(m.group('offset'))
578
    upto = m.group('upto')
579
    total = m.group('total')
580
    if upto != '':
581
        upto = int(upto)
582
    else:
583
        upto = None
584
    if total != '*':
585
        total = int(total)
586
    else:
587
        total = None
588
    if (upto is not None and offset > upto) or \
589
        (total is not None and offset >= total) or \
590
            (total is not None and upto is not None and upto >= total):
591
        return None
592

    
593
    if upto is None:
594
        length = None
595
    else:
596
        length = upto - offset + 1
597
    return (offset, length, total)
598

    
599

    
600
def get_sharing(request):
601
    """Parse an X-Object-Sharing header from the request.
602

603
    Raises BadRequest on error.
604
    """
605

    
606
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
607
    if permissions is None:
608
        return None
609

    
610
    # TODO: Document or remove '~' replacing.
611
    permissions = permissions.replace('~', '')
612

    
613
    ret = {}
614
    permissions = permissions.replace(' ', '')
615
    if permissions == '':
616
        return ret
617
    for perm in (x for x in permissions.split(';')):
618
        if perm.startswith('read='):
619
            ret['read'] = list(set(
620
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
621
            if '' in ret['read']:
622
                ret['read'].remove('')
623
            if '*' in ret['read']:
624
                ret['read'] = ['*']
625
            if len(ret['read']) == 0:
626
                raise faults.BadRequest(
627
                    'Bad X-Object-Sharing header value: invalid length')
628
        elif perm.startswith('write='):
629
            ret['write'] = list(set(
630
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
631
            if '' in ret['write']:
632
                ret['write'].remove('')
633
            if '*' in ret['write']:
634
                ret['write'] = ['*']
635
            if len(ret['write']) == 0:
636
                raise faults.BadRequest(
637
                    'Bad X-Object-Sharing header value: invalid length')
638
        else:
639
            raise faults.BadRequest(
640
                'Bad X-Object-Sharing header value: missing prefix')
641

    
642
    # replace displayname with uuid
643
    if TRANSLATE_UUIDS:
644
        try:
645
            ret['read'] = [replace_permissions_displayname(
646
                getattr(request, 'token', None), x)
647
                for x in ret.get('read', [])]
648
            ret['write'] = [replace_permissions_displayname(
649
                getattr(request, 'token', None), x)
650
                for x in ret.get('write', [])]
651
        except ItemNotExists, e:
652
            raise faults.BadRequest(
653
                'Bad X-Object-Sharing header value: unknown account: %s' % e)
654

    
655
    # Keep duplicates only in write list.
656
    dups = [x for x in ret.get(
657
        'read', []) if x in ret.get('write', []) and x != '*']
658
    if dups:
659
        for x in dups:
660
            ret['read'].remove(x)
661
        if len(ret['read']) == 0:
662
            del(ret['read'])
663

    
664
    return ret
665

    
666

    
667
def get_public(request):
668
    """Parse an X-Object-Public header from the request.
669

670
    Raises BadRequest on error.
671
    """
672

    
673
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
674
    if public is None:
675
        return None
676

    
677
    public = public.replace(' ', '').lower()
678
    if public == 'true':
679
        return True
680
    elif public == 'false' or public == '':
681
        return False
682
    raise faults.BadRequest('Bad X-Object-Public header value')
683

    
684

    
685
def raw_input_socket(request):
686
    """Return the socket for reading the rest of the request."""
687

    
688
    server_software = request.META.get('SERVER_SOFTWARE')
689
    if server_software and server_software.startswith('mod_python'):
690
        return request._req
691
    if 'wsgi.input' in request.environ:
692
        return request.environ['wsgi.input']
693
    raise NotImplemented('Unknown server software')
694

    
695
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB
696

    
697

    
698
def socket_read_iterator(request, length=0, blocksize=4096):
699
    """Return a maximum of blocksize data read from the socket in each iteration
700

701
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
702
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
703
    """
704

    
705
    sock = raw_input_socket(request)
706
    if length < 0:  # Chunked transfers
707
        # Small version (server does the dechunking).
708
        if (request.environ.get('mod_wsgi.input_chunked', None)
709
                or request.META['SERVER_SOFTWARE'].startswith('gunicorn')):
710
            while length < MAX_UPLOAD_SIZE:
711
                data = sock.read(blocksize)
712
                if data == '':
713
                    return
714
                yield data
715
            raise faults.BadRequest('Maximum size is reached')
716

    
717
        # Long version (do the dechunking).
718
        data = ''
719
        while length < MAX_UPLOAD_SIZE:
720
            # Get chunk size.
721
            if hasattr(sock, 'readline'):
722
                chunk_length = sock.readline()
723
            else:
724
                chunk_length = ''
725
                while chunk_length[-1:] != '\n':
726
                    chunk_length += sock.read(1)
727
                chunk_length.strip()
728
            pos = chunk_length.find(';')
729
            if pos >= 0:
730
                chunk_length = chunk_length[:pos]
731
            try:
732
                chunk_length = int(chunk_length, 16)
733
            except Exception:
734
                raise faults.BadRequest('Bad chunk size')
735
                                 # TODO: Change to something more appropriate.
736
            # Check if done.
737
            if chunk_length == 0:
738
                if len(data) > 0:
739
                    yield data
740
                return
741
            # Get the actual data.
742
            while chunk_length > 0:
743
                chunk = sock.read(min(chunk_length, blocksize))
744
                chunk_length -= len(chunk)
745
                if length > 0:
746
                    length += len(chunk)
747
                data += chunk
748
                if len(data) >= blocksize:
749
                    ret = data[:blocksize]
750
                    data = data[blocksize:]
751
                    yield ret
752
            sock.read(2)  # CRLF
753
        raise faults.BadRequest('Maximum size is reached')
754
    else:
755
        if length > MAX_UPLOAD_SIZE:
756
            raise faults.BadRequest('Maximum size is reached')
757
        while length > 0:
758
            data = sock.read(min(length, blocksize))
759
            if not data:
760
                raise faults.BadRequest()
761
            length -= len(data)
762
            yield data
763

    
764

    
765
class SaveToBackendHandler(FileUploadHandler):
766
    """Handle a file from an HTML form the django way."""
767

    
768
    def __init__(self, request=None):
769
        super(SaveToBackendHandler, self).__init__(request)
770
        self.backend = request.backend
771

    
772
    def put_data(self, length):
773
        if len(self.data) >= length:
774
            block = self.data[:length]
775
            self.file.hashmap.append(self.backend.put_block(block))
776
            self.md5.update(block)
777
            self.data = self.data[length:]
778

    
779
    def new_file(self, field_name, file_name, content_type,
780
                 content_length, charset=None):
781
        self.md5 = hashlib.md5()
782
        self.data = ''
783
        self.file = UploadedFile(
784
            name=file_name, content_type=content_type, charset=charset)
785
        self.file.size = 0
786
        self.file.hashmap = []
787

    
788
    def receive_data_chunk(self, raw_data, start):
789
        self.data += raw_data
790
        self.file.size += len(raw_data)
791
        self.put_data(self.request.backend.block_size)
792
        return None
793

    
794
    def file_complete(self, file_size):
795
        l = len(self.data)
796
        if l > 0:
797
            self.put_data(l)
798
        self.file.etag = self.md5.hexdigest().lower()
799
        return self.file
800

    
801

    
802
class ObjectWrapper(object):
803
    """Return the object's data block-per-block in each iteration.
804

805
    Read from the object using the offset and length provided
806
    in each entry of the range list.
807
    """
808

    
809
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
810
        self.backend = backend
811
        self.ranges = ranges
812
        self.sizes = sizes
813
        self.hashmaps = hashmaps
814
        self.boundary = boundary
815
        self.size = sum(self.sizes)
816

    
817
        self.file_index = 0
818
        self.block_index = 0
819
        self.block_hash = -1
820
        self.block = ''
821

    
822
        self.range_index = -1
823
        self.offset, self.length = self.ranges[0]
824

    
825
    def __iter__(self):
826
        return self
827

    
828
    def part_iterator(self):
829
        if self.length > 0:
830
            # Get the file for the current offset.
831
            file_size = self.sizes[self.file_index]
832
            while self.offset >= file_size:
833
                self.offset -= file_size
834
                self.file_index += 1
835
                file_size = self.sizes[self.file_index]
836

    
837
            # Get the block for the current position.
838
            self.block_index = int(self.offset / self.backend.block_size)
839
            if self.block_hash != \
840
                    self.hashmaps[self.file_index][self.block_index]:
841
                self.block_hash = self.hashmaps[
842
                    self.file_index][self.block_index]
843
                try:
844
                    self.block = self.backend.get_block(self.block_hash)
845
                except ItemNotExists:
846
                    raise faults.ItemNotFound('Block does not exist')
847

    
848
            # Get the data from the block.
849
            bo = self.offset % self.backend.block_size
850
            bs = self.backend.block_size
851
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
852
                    self.sizes[self.file_index] % self.backend.block_size):
853
                bs = self.sizes[self.file_index] % self.backend.block_size
854
            bl = min(self.length, bs - bo)
855
            data = self.block[bo:bo + bl]
856
            self.offset += bl
857
            self.length -= bl
858
            return data
859
        else:
860
            raise StopIteration
861

    
862
    def next(self):
863
        if len(self.ranges) == 1:
864
            return self.part_iterator()
865
        if self.range_index == len(self.ranges):
866
            raise StopIteration
867
        try:
868
            if self.range_index == -1:
869
                raise StopIteration
870
            return self.part_iterator()
871
        except StopIteration:
872
            self.range_index += 1
873
            out = []
874
            if self.range_index < len(self.ranges):
875
                # Part header.
876
                self.offset, self.length = self.ranges[self.range_index]
877
                self.file_index = 0
878
                if self.range_index > 0:
879
                    out.append('')
880
                out.append('--' + self.boundary)
881
                out.append('Content-Range: bytes %d-%d/%d' % (
882
                    self.offset, self.offset + self.length - 1, self.size))
883
                out.append('Content-Transfer-Encoding: binary')
884
                out.append('')
885
                out.append('')
886
                return '\r\n'.join(out)
887
            else:
888
                # Footer.
889
                out.append('')
890
                out.append('--' + self.boundary + '--')
891
                out.append('')
892
                return '\r\n'.join(out)
893

    
894

    
895
def object_data_response(request, sizes, hashmaps, meta, public=False):
896
    """Get the HttpResponse object for replying with the object's data."""
897

    
898
    # Range handling.
899
    size = sum(sizes)
900
    ranges = get_range(request, size)
901
    if ranges is None:
902
        ranges = [(0, size)]
903
        ret = 200
904
    else:
905
        check = [True for offset, length in ranges if
906
                 length <= 0 or length > size or
907
                 offset < 0 or offset >= size or
908
                 offset + length > size]
909
        if len(check) > 0:
910
            raise faults.RangeNotSatisfiable(
911
                'Requested range exceeds object limits')
912
        ret = 206
913
        if_range = request.META.get('HTTP_IF_RANGE')
914
        if if_range:
915
            try:
916
                # Modification time has passed instead.
917
                last_modified = parse_http_date(if_range)
918
                if last_modified != meta['modified']:
919
                    ranges = [(0, size)]
920
                    ret = 200
921
            except ValueError:
922
                if if_range != meta['checksum']:
923
                    ranges = [(0, size)]
924
                    ret = 200
925

    
926
    if ret == 206 and len(ranges) > 1:
927
        boundary = uuid.uuid4().hex
928
    else:
929
        boundary = ''
930
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
931
    response = HttpResponse(wrapper, status=ret)
932
    put_object_headers(
933
        response, meta, restricted=public,
934
        token=getattr(request, 'token', None))
935
    if ret == 206:
936
        if len(ranges) == 1:
937
            offset, length = ranges[0]
938
            response[
939
                'Content-Length'] = length  # Update with the correct length.
940
            response['Content-Range'] = 'bytes %d-%d/%d' % (
941
                offset, offset + length - 1, size)
942
        else:
943
            del(response['Content-Length'])
944
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (
945
                boundary,)
946
    return response
947

    
948

    
949
def put_object_block(request, hashmap, data, offset):
950
    """Put one block of data at the given offset."""
951

    
952
    bi = int(offset / request.backend.block_size)
953
    bo = offset % request.backend.block_size
954
    bl = min(len(data), request.backend.block_size - bo)
955
    if bi < len(hashmap):
956
        hashmap[bi] = request.backend.update_block(hashmap[bi], data[:bl], bo)
957
    else:
958
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
959
    return bl  # Return ammount of data written.
960

    
961

    
962
def hashmap_md5(backend, hashmap, size):
963
    """Produce the MD5 sum from the data in the hashmap."""
964

    
965
    # TODO: Search backend for the MD5 of another object
966
    #       with the same hashmap and size...
967
    md5 = hashlib.md5()
968
    bs = backend.block_size
969
    for bi, hash in enumerate(hashmap):
970
        data = backend.get_block(hash)  # Blocks come in padded.
971
        if bi == len(hashmap) - 1:
972
            data = data[:size % bs]
973
        md5.update(data)
974
    return md5.hexdigest().lower()
975

    
976

    
977
def simple_list_response(request, l):
978
    if request.serialization == 'text':
979
        return '\n'.join(l) + '\n'
980
    if request.serialization == 'xml':
981
        return render_to_string('items.xml', {'items': l})
982
    if request.serialization == 'json':
983
        return json.dumps(l)
984

    
985

    
986
from pithos.backends.util import PithosBackendPool
987

    
988
if RADOS_STORAGE:
989
    BLOCK_PARAMS = {'mappool': RADOS_POOL_MAPS,
990
                    'blockpool': RADOS_POOL_BLOCKS, }
991
else:
992
    BLOCK_PARAMS = {'mappool': None,
993
                    'blockpool': None, }
994

    
995

    
996
_pithos_backend_pool = PithosBackendPool(
997
        size=BACKEND_POOL_SIZE,
998
        db_module=BACKEND_DB_MODULE,
999
        db_connection=BACKEND_DB_CONNECTION,
1000
        block_module=BACKEND_BLOCK_MODULE,
1001
        block_path=BACKEND_BLOCK_PATH,
1002
        block_umask=BACKEND_BLOCK_UMASK,
1003
        queue_module=BACKEND_QUEUE_MODULE,
1004
        queue_hosts=BACKEND_QUEUE_HOSTS,
1005
        queue_exchange=BACKEND_QUEUE_EXCHANGE,
1006
        astakos_url=ASTAKOS_BASE_URL,
1007
        service_token=SERVICE_TOKEN,
1008
        astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
1009
        free_versioning=BACKEND_FREE_VERSIONING,
1010
        block_params=BLOCK_PARAMS,
1011
        public_url_security=PUBLIC_URL_SECURITY,
1012
        public_url_alphabet=PUBLIC_URL_ALPHABET,
1013
        account_quota_policy=BACKEND_ACCOUNT_QUOTA,
1014
        container_quota_policy=BACKEND_CONTAINER_QUOTA,
1015
        container_versioning_policy=BACKEND_VERSIONING)
1016

    
1017

    
1018
def get_backend():
1019
    backend = _pithos_backend_pool.pool_get()
1020
    backend.messages = []
1021
    return backend
1022

    
1023

    
1024
def update_request_headers(request):
1025
    # Handle URL-encoded keys and values.
1026
    meta = dict([(
1027
        k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
1028
    for k, v in meta.iteritems():
1029
        try:
1030
            k.decode('ascii')
1031
            v.decode('ascii')
1032
        except UnicodeDecodeError:
1033
            raise faults.BadRequest('Bad character in headers.')
1034
        if '%' in k or '%' in v:
1035
            del(request.META[k])
1036
            request.META[unquote(k)] = smart_unicode(unquote(
1037
                v), strings_only=True)
1038

    
1039

    
1040
def update_response_headers(request, response):
1041
    # URL-encode unicode in headers.
1042
    meta = response.items()
1043
    for k, v in meta:
1044
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
1045
                k.startswith('X-Object-') or k.startswith('Content-')):
1046
            del(response[k])
1047
            response[quote(k)] = quote(v, safe='/=,:@; ')
1048

    
1049

    
1050
def get_pithos_usage(token):
1051
    """Get Pithos Usage from astakos."""
1052
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
1053
                            logger=logger)
1054
    quotas = astakos.get_quotas(token)['system']
1055
    pithos_resources = [r['name'] for r in resources]
1056
    map(quotas.pop, filter(lambda k: k not in pithos_resources, quotas.keys()))
1057
    return quotas.popitem()[-1] # assume only one resource
1058

    
1059

    
1060
def api_method(http_method=None, token_required=True, user_required=True, logger=None,
1061
               format_allowed=False, serializations=None,
1062
               strict_serlization=False):
1063
    serializations = serializations or ['json', 'xml']
1064
    def decorator(func):
1065
        @api.api_method(http_method=http_method, token_required=token_required,
1066
                        user_required=user_required,
1067
                        logger=logger, format_allowed=format_allowed,
1068
                        astakos_url=ASTAKOS_BASE_URL,
1069
                        serializations=serializations,
1070
                        strict_serlization=strict_serlization)
1071
        @wraps(func)
1072
        def wrapper(request, *args, **kwargs):
1073
            # The args variable may contain up to (account, container, object).
1074
            if len(args) > 1 and len(args[1]) > 256:
1075
                raise faults.BadRequest("Container name too large")
1076
            if len(args) > 2 and len(args[2]) > 1024:
1077
                raise faults.BadRequest('Object name too large.')
1078

    
1079
            try:
1080
                # Add a PithosBackend as attribute of the request object
1081
                request.backend = get_backend()
1082
                # Many API method expect thet X-Auth-Token in request,token
1083
                request.token = request.x_auth_token
1084
                update_request_headers(request)
1085
                response = func(request, *args, **kwargs)
1086
                update_response_headers(request, response)
1087
                return response
1088
            finally:
1089
                # Always close PithosBackend connection
1090
                if getattr(request, "backend", None) is not None:
1091
                    request.backend.close()
1092
        return wrapper
1093
    return decorator
1094

    
1095

    
1096
def get_token_from_cookie(request):
1097
    assert(request.method == 'GET'),\
1098
        "Cookie based authentication is only allowed to GET requests"
1099
    token = None
1100
    if COOKIE_NAME in request.COOKIES:
1101
        cookie_value = unquote(request.COOKIES.get(COOKIE_NAME, ''))
1102
        account, sep, token = cookie_value.partition('|')
1103
    return token
1104

    
1105

    
1106
def view_method():
1107
    """Decorator function for views."""
1108

    
1109
    def decorator(func):
1110
        @wraps(func)
1111
        def wrapper(request, *args, **kwargs):
1112
            token = get_token_from_cookie(request)
1113
            if token is None:
1114
                return HttpResponseRedirect('%s?next=%s' % (
1115
                    LOGIN_URL, join_urls(BASE_HOST, request.path)))
1116
            request.META['HTTP_X_AUTH_TOKEN'] = token
1117
            # Get the response object
1118
            response = func(request, *args, **kwargs)
1119
            if response.status_code in [200, 206, 304, 412, 416]:
1120
                return response
1121
            elif response.status_code == 404:
1122
                raise Http404()
1123
            elif response.status_code in [401, 403]:
1124
                return HttpResponseForbidden()
1125
            else:
1126
                # unexpected response status
1127
                raise Exception(response.status_code)
1128
        return wrapper
1129
    return decorator