Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ ad0efdb3

History | View | Annotate | Download (38.5 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from functools import wraps
35
from datetime import datetime
36
from urllib import quote, unquote
37

    
38
from django.http import HttpResponse
39
from django.template.loader import render_to_string
40
from django.utils import simplejson as json
41
from django.utils.http import http_date, parse_etags
42
from django.utils.encoding import smart_unicode, smart_str
43
from django.core.files.uploadhandler import FileUploadHandler
44
from django.core.files.uploadedfile import UploadedFile
45

    
46
from snf_django.lib.api.parsedate import parse_http_date_safe, parse_http_date
47
from snf_django.lib.astakos import user_for_token
48
from snf_django.lib import api
49
from snf_django.lib.api import faults, utils
50

    
51
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
52
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
53
                                 BACKEND_BLOCK_UMASK,
54
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
55
                                 BACKEND_QUEUE_EXCHANGE, USE_QUOTAHOLDER,
56
                                 QUOTAHOLDER_URL, QUOTAHOLDER_TOKEN,
57
                                 QUOTAHOLDER_POOLSIZE,
58
                                 ASTAKOS_URL,
59
                                 BACKEND_ACCOUNT_QUOTA, BACKEND_CONTAINER_QUOTA,
60
                                 BACKEND_VERSIONING,
61
                                 BACKEND_FREE_VERSIONING, BACKEND_POOL_SIZE,
62
                                 USER_CATALOG_URL,
63
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
64
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
65
                                 PUBLIC_URL_SECURITY,
66
                                 PUBLIC_URL_ALPHABET)
67
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
68
                                  VersionNotExists)
69
from snf_django.lib.astakos import (get_user_uuid, get_displayname,
70
                                    get_uuids, get_displaynames)
71

    
72
import logging
73
import re
74
import hashlib
75
import uuid
76
import decimal
77

    
78
logger = logging.getLogger(__name__)
79

    
80

    
81
def json_encode_decimal(obj):
82
    if isinstance(obj, decimal.Decimal):
83
        return str(obj)
84
    raise TypeError(repr(obj) + " is not JSON serializable")
85

    
86

    
87
def rename_meta_key(d, old, new):
88
    if old not in d:
89
        return
90
    d[new] = d[old]
91
    del(d[old])
92

    
93

    
94
def printable_header_dict(d):
95
    """Format a meta dictionary for printing out json/xml.
96

97
    Convert all keys to lower case and replace dashes with underscores.
98
    Format 'last_modified' timestamp.
99
    """
100

    
101
    if 'last_modified' in d and d['last_modified']:
102
        d['last_modified'] = utils.isoformat(
103
            datetime.fromtimestamp(d['last_modified']))
104
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
105

    
106

    
107
def format_header_key(k):
108
    """Convert underscores to dashes and capitalize intra-dash strings."""
109
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
110

    
111

    
112
def get_header_prefix(request, prefix):
113
    """Get all prefix-* request headers in a dict.
114
       Reformat keys with format_header_key()."""
115

    
116
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
117
    # TODO: Document or remove '~' replacing.
118
    return dict([(format_header_key(k[5:]), v.replace('~', ''))
119
                for k, v in request.META.iteritems()
120
                if k.startswith(prefix) and len(k) > len(prefix)])
121

    
122

    
123
def check_meta_headers(meta):
124
    if len(meta) > 90:
125
        raise faults.BadRequest('Too many headers.')
126
    for k, v in meta.iteritems():
127
        if len(k) > 128:
128
            raise faults.BadRequest('Header name too large.')
129
        if len(v) > 256:
130
            raise faults.BadRequest('Header value too large.')
131

    
132

    
133
def get_account_headers(request):
134
    meta = get_header_prefix(request, 'X-Account-Meta-')
135
    check_meta_headers(meta)
136
    groups = {}
137
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
138
        n = k[16:].lower()
139
        if '-' in n or '_' in n:
140
            raise faults.BadRequest('Bad characters in group name')
141
        groups[n] = v.replace(' ', '').split(',')
142
        while '' in groups[n]:
143
            groups[n].remove('')
144
    return meta, groups
145

    
146

    
147
def put_account_headers(response, meta, groups, policy):
148
    if 'count' in meta:
149
        response['X-Account-Container-Count'] = meta['count']
150
    if 'bytes' in meta:
151
        response['X-Account-Bytes-Used'] = meta['bytes']
152
    response['Last-Modified'] = http_date(int(meta['modified']))
153
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
154
        response[smart_str(
155
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
156
    if 'until_timestamp' in meta:
157
        response['X-Account-Until-Timestamp'] = http_date(
158
            int(meta['until_timestamp']))
159
    for k, v in groups.iteritems():
160
        k = smart_str(k, strings_only=True)
161
        k = format_header_key('X-Account-Group-' + k)
162
        v = smart_str(','.join(v), strings_only=True)
163
        response[k] = v
164
    for k, v in policy.iteritems():
165
        response[smart_str(format_header_key('X-Account-Policy-' + k),
166
                 strings_only=True)] = smart_str(v, strings_only=True)
167

    
168

    
169
def get_container_headers(request):
170
    meta = get_header_prefix(request, 'X-Container-Meta-')
171
    check_meta_headers(meta)
172
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in
173
                  get_header_prefix(request,
174
                                    'X-Container-Policy-').iteritems()])
175
    return meta, policy
176

    
177

    
178
def put_container_headers(request, response, meta, policy):
179
    if 'count' in meta:
180
        response['X-Container-Object-Count'] = meta['count']
181
    if 'bytes' in meta:
182
        response['X-Container-Bytes-Used'] = meta['bytes']
183
    response['Last-Modified'] = http_date(int(meta['modified']))
184
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
185
        response[smart_str(
186
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
187
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
188
         if x.startswith('X-Object-Meta-')]
189
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
190
    response['X-Container-Block-Size'] = request.backend.block_size
191
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
192
    if 'until_timestamp' in meta:
193
        response['X-Container-Until-Timestamp'] = http_date(
194
            int(meta['until_timestamp']))
195
    for k, v in policy.iteritems():
196
        response[smart_str(format_header_key('X-Container-Policy-' + k),
197
                           strings_only=True)] = smart_str(v, strings_only=True)
198

    
199

    
200
def get_object_headers(request):
201
    content_type = request.META.get('CONTENT_TYPE', None)
202
    meta = get_header_prefix(request, 'X-Object-Meta-')
203
    check_meta_headers(meta)
204
    if request.META.get('HTTP_CONTENT_ENCODING'):
205
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
206
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
207
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
208
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
209
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
210
    return content_type, meta, get_sharing(request), get_public(request)
211

    
212

    
213
def put_object_headers(response, meta, restricted=False, token=None):
214
    response['ETag'] = meta['checksum']
215
    response['Content-Length'] = meta['bytes']
216
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
217
    response['Last-Modified'] = http_date(int(meta['modified']))
218
    if not restricted:
219
        response['X-Object-Hash'] = meta['hash']
220
        response['X-Object-UUID'] = meta['uuid']
221
        if TRANSLATE_UUIDS:
222
            meta['modified_by'] = \
223
                retrieve_displayname(token, meta['modified_by'])
224
        response['X-Object-Modified-By'] = smart_str(
225
            meta['modified_by'], strings_only=True)
226
        response['X-Object-Version'] = meta['version']
227
        response['X-Object-Version-Timestamp'] = http_date(
228
            int(meta['version_timestamp']))
229
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
230
            response[smart_str(
231
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
232
        for k in (
233
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
234
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
235
                'X-Object-Public'):
236
            if k in meta:
237
                response[k] = smart_str(meta[k], strings_only=True)
238
    else:
239
        for k in ('Content-Encoding', 'Content-Disposition'):
240
            if k in meta:
241
                response[k] = smart_str(meta[k], strings_only=True)
242

    
243

    
244
def update_manifest_meta(request, v_account, meta):
245
    """Update metadata if the object has an X-Object-Manifest."""
246

    
247
    if 'X-Object-Manifest' in meta:
248
        etag = ''
249
        bytes = 0
250
        try:
251
            src_container, src_name = split_container_object_string(
252
                '/' + meta['X-Object-Manifest'])
253
            objects = request.backend.list_objects(
254
                request.user_uniq, v_account,
255
                src_container, prefix=src_name, virtual=False)
256
            for x in objects:
257
                src_meta = request.backend.get_object_meta(request.user_uniq,
258
                                                           v_account,
259
                                                           src_container,
260
                                                           x[0], 'pithos', x[1])
261
                etag += src_meta['checksum']
262
                bytes += src_meta['bytes']
263
        except:
264
            # Ignore errors.
265
            return
266
        meta['bytes'] = bytes
267
        md5 = hashlib.md5()
268
        md5.update(etag)
269
        meta['checksum'] = md5.hexdigest().lower()
270

    
271

    
272
def is_uuid(str):
273
    if str is None:
274
        return False
275
    try:
276
        uuid.UUID(str)
277
    except ValueError:
278
        return False
279
    else:
280
        return True
281

    
282

    
283
##########################
284
# USER CATALOG utilities #
285
##########################
286

    
287
def retrieve_displayname(token, uuid, fail_silently=True):
288
    displayname = get_displayname(token, uuid, USER_CATALOG_URL)
289
    if not displayname and not fail_silently:
290
        raise ItemNotExists(uuid)
291
    elif not displayname:
292
        # just return the uuid
293
        return uuid
294
    return displayname
295

    
296

    
297
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
298
    catalog = get_displaynames(token, uuids, USER_CATALOG_URL) or {}
299
    missing = list(set(uuids) - set(catalog))
300
    if missing and not fail_silently:
301
        raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
302
    return catalog if return_dict else [catalog.get(i) for i in uuids]
303

    
304

    
305
def retrieve_uuid(token, displayname):
306
    if is_uuid(displayname):
307
        return displayname
308

    
309
    uuid = get_user_uuid(token, displayname, USER_CATALOG_URL)
310
    if not uuid:
311
        raise ItemNotExists(displayname)
312
    return uuid
313

    
314

    
315
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
316
    catalog = get_uuids(token, displaynames, USER_CATALOG_URL) or {}
317
    missing = list(set(displaynames) - set(catalog))
318
    if missing and not fail_silently:
319
        raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
320
    return catalog if return_dict else [catalog.get(i) for i in displaynames]
321

    
322

    
323
def replace_permissions_displayname(token, holder):
324
    if holder == '*':
325
        return holder
326
    try:
327
        # check first for a group permission
328
        account, group = holder.split(':', 1)
329
    except ValueError:
330
        return retrieve_uuid(token, holder)
331
    else:
332
        return ':'.join([retrieve_uuid(token, account), group])
333

    
334

    
335
def replace_permissions_uuid(token, holder):
336
    if holder == '*':
337
        return holder
338
    try:
339
        # check first for a group permission
340
        account, group = holder.split(':', 1)
341
    except ValueError:
342
        return retrieve_displayname(token, holder)
343
    else:
344
        return ':'.join([retrieve_displayname(token, account), group])
345

    
346

    
347
def update_sharing_meta(request, permissions, v_account,
348
                        v_container, v_object, meta):
349
    if permissions is None:
350
        return
351
    allowed, perm_path, perms = permissions
352
    if len(perms) == 0:
353
        return
354

    
355
    # replace uuid with displayname
356
    if TRANSLATE_UUIDS:
357
        perms['read'] = [replace_permissions_uuid(
358
            getattr(request, 'token', None), x)
359
            for x in perms.get('read', [])]
360
        perms['write'] = [replace_permissions_uuid(
361
            getattr(request, 'token', None), x)
362
            for x in perms.get('write', [])]
363

    
364
    ret = []
365

    
366
    r = ','.join(perms.get('read', []))
367
    if r:
368
        ret.append('read=' + r)
369
    w = ','.join(perms.get('write', []))
370
    if w:
371
        ret.append('write=' + w)
372
    meta['X-Object-Sharing'] = '; '.join(ret)
373
    if '/'.join((v_account, v_container, v_object)) != perm_path:
374
        meta['X-Object-Shared-By'] = perm_path
375
    if request.user_uniq != v_account:
376
        meta['X-Object-Allowed-To'] = allowed
377

    
378

    
379
def update_public_meta(public, meta):
380
    if not public:
381
        return
382
    meta['X-Object-Public'] = '/public/' + public
383

    
384

    
385
def validate_modification_preconditions(request, meta):
386
    """Check that the modified timestamp conforms with the preconditions set."""
387

    
388
    if 'modified' not in meta:
389
        return  # TODO: Always return?
390

    
391
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
392
    if if_modified_since is not None:
393
        if_modified_since = parse_http_date_safe(if_modified_since)
394
    if (if_modified_since is not None
395
            and int(meta['modified']) <= if_modified_since):
396
        raise faults.NotModified('Resource has not been modified')
397

    
398
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
399
    if if_unmodified_since is not None:
400
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
401
    if (if_unmodified_since is not None
402
            and int(meta['modified']) > if_unmodified_since):
403
        raise faults.PreconditionFailed('Resource has been modified')
404

    
405

    
406
def validate_matching_preconditions(request, meta):
407
    """Check that the ETag conforms with the preconditions set."""
408

    
409
    etag = meta['checksum']
410
    if not etag:
411
        etag = None
412

    
413
    if_match = request.META.get('HTTP_IF_MATCH')
414
    if if_match is not None:
415
        if etag is None:
416
            raise faults.PreconditionFailed('Resource does not exist')
417
        if (if_match != '*'
418
                and etag not in [x.lower() for x in parse_etags(if_match)]):
419
            raise faults.PreconditionFailed('Resource ETag does not match')
420

    
421
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
422
    if if_none_match is not None:
423
        # TODO: If this passes, must ignore If-Modified-Since header.
424
        if etag is not None:
425
            if (if_none_match == '*'
426
                    or etag in [x.lower() for x in parse_etags(if_none_match)]):
427
                # TODO: Continue if an If-Modified-Since header is present.
428
                if request.method in ('HEAD', 'GET'):
429
                    raise faults.NotModified('Resource ETag matches')
430
                raise faults.PreconditionFailed(
431
                    'Resource exists or ETag matches')
432

    
433

    
434
def split_container_object_string(s):
435
    if not len(s) > 0 or s[0] != '/':
436
        raise ValueError
437
    s = s[1:]
438
    pos = s.find('/')
439
    if pos == -1 or pos == len(s) - 1:
440
        raise ValueError
441
    return s[:pos], s[(pos + 1):]
442

    
443

    
444
def copy_or_move_object(request, src_account, src_container, src_name,
445
                        dest_account, dest_container, dest_name,
446
                        move=False, delimiter=None):
447
    """Copy or move an object."""
448

    
449
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
450
        del(request.META['CONTENT_TYPE'])
451
    content_type, meta, permissions, public = get_object_headers(request)
452
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
453
    try:
454
        if move:
455
            version_id = request.backend.move_object(
456
                request.user_uniq, src_account, src_container, src_name,
457
                dest_account, dest_container, dest_name,
458
                content_type, 'pithos', meta, False, permissions, delimiter)
459
        else:
460
            version_id = request.backend.copy_object(
461
                request.user_uniq, src_account, src_container, src_name,
462
                dest_account, dest_container, dest_name,
463
                content_type, 'pithos', meta, False, permissions,
464
                src_version, delimiter)
465
    except NotAllowedError:
466
        raise faults.Forbidden('Not allowed')
467
    except (ItemNotExists, VersionNotExists):
468
        raise faults.ItemNotFound('Container or object does not exist')
469
    except ValueError:
470
        raise faults.BadRequest('Invalid sharing header')
471
    except QuotaError, e:
472
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
473
    if public is not None:
474
        try:
475
            request.backend.update_object_public(
476
                request.user_uniq, dest_account,
477
                dest_container, dest_name, public)
478
        except NotAllowedError:
479
            raise faults.Forbidden('Not allowed')
480
        except ItemNotExists:
481
            raise faults.ItemNotFound('Object does not exist')
482
    return version_id
483

    
484

    
485
def get_int_parameter(p):
486
    if p is not None:
487
        try:
488
            p = int(p)
489
        except ValueError:
490
            return None
491
        if p < 0:
492
            return None
493
    return p
494

    
495

    
496
def get_content_length(request):
497
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
498
    if content_length is None:
499
        raise faults.LengthRequired('Missing or invalid Content-Length header')
500
    return content_length
501

    
502

    
503
def get_range(request, size):
504
    """Parse a Range header from the request.
505

506
    Either returns None, when the header is not existent or should be ignored,
507
    or a list of (offset, length) tuples - should be further checked.
508
    """
509

    
510
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
511
    if not ranges.startswith('bytes='):
512
        return None
513

    
514
    ret = []
515
    for r in (x.strip() for x in ranges[6:].split(',')):
516
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
517
        m = p.match(r)
518
        if not m:
519
            return None
520
        offset = m.group('offset')
521
        upto = m.group('upto')
522
        if offset == '' and upto == '':
523
            return None
524

    
525
        if offset != '':
526
            offset = int(offset)
527
            if upto != '':
528
                upto = int(upto)
529
                if offset > upto:
530
                    return None
531
                ret.append((offset, upto - offset + 1))
532
            else:
533
                ret.append((offset, size - offset))
534
        else:
535
            length = int(upto)
536
            ret.append((size - length, length))
537

    
538
    return ret
539

    
540

    
541
def get_content_range(request):
542
    """Parse a Content-Range header from the request.
543

544
    Either returns None, when the header is not existent or should be ignored,
545
    or an (offset, length, total) tuple - check as length, total may be None.
546
    Returns (None, None, None) if the provided range is '*/*'.
547
    """
548

    
549
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
550
    if not ranges:
551
        return None
552

    
553
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
554
    m = p.match(ranges)
555
    if not m:
556
        if ranges == 'bytes */*':
557
            return (None, None, None)
558
        return None
559
    offset = int(m.group('offset'))
560
    upto = m.group('upto')
561
    total = m.group('total')
562
    if upto != '':
563
        upto = int(upto)
564
    else:
565
        upto = None
566
    if total != '*':
567
        total = int(total)
568
    else:
569
        total = None
570
    if (upto is not None and offset > upto) or \
571
        (total is not None and offset >= total) or \
572
            (total is not None and upto is not None and upto >= total):
573
        return None
574

    
575
    if upto is None:
576
        length = None
577
    else:
578
        length = upto - offset + 1
579
    return (offset, length, total)
580

    
581

    
582
def get_sharing(request):
583
    """Parse an X-Object-Sharing header from the request.
584

585
    Raises BadRequest on error.
586
    """
587

    
588
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
589
    if permissions is None:
590
        return None
591

    
592
    # TODO: Document or remove '~' replacing.
593
    permissions = permissions.replace('~', '')
594

    
595
    ret = {}
596
    permissions = permissions.replace(' ', '')
597
    if permissions == '':
598
        return ret
599
    for perm in (x for x in permissions.split(';')):
600
        if perm.startswith('read='):
601
            ret['read'] = list(set(
602
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
603
            if '' in ret['read']:
604
                ret['read'].remove('')
605
            if '*' in ret['read']:
606
                ret['read'] = ['*']
607
            if len(ret['read']) == 0:
608
                raise faults.BadRequest(
609
                    'Bad X-Object-Sharing header value: invalid length')
610
        elif perm.startswith('write='):
611
            ret['write'] = list(set(
612
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
613
            if '' in ret['write']:
614
                ret['write'].remove('')
615
            if '*' in ret['write']:
616
                ret['write'] = ['*']
617
            if len(ret['write']) == 0:
618
                raise faults.BadRequest(
619
                    'Bad X-Object-Sharing header value: invalid length')
620
        else:
621
            raise faults.BadRequest(
622
                'Bad X-Object-Sharing header value: missing prefix')
623

    
624
    # replace displayname with uuid
625
    if TRANSLATE_UUIDS:
626
        try:
627
            ret['read'] = [replace_permissions_displayname(
628
                getattr(request, 'token', None), x)
629
                for x in ret.get('read', [])]
630
            ret['write'] = [replace_permissions_displayname(
631
                getattr(request, 'token', None), x)
632
                for x in ret.get('write', [])]
633
        except ItemNotExists, e:
634
            raise faults.BadRequest(
635
                'Bad X-Object-Sharing header value: unknown account: %s' % e)
636

    
637
    # Keep duplicates only in write list.
638
    dups = [x for x in ret.get(
639
        'read', []) if x in ret.get('write', []) and x != '*']
640
    if dups:
641
        for x in dups:
642
            ret['read'].remove(x)
643
        if len(ret['read']) == 0:
644
            del(ret['read'])
645

    
646
    return ret
647

    
648

    
649
def get_public(request):
650
    """Parse an X-Object-Public header from the request.
651

652
    Raises BadRequest on error.
653
    """
654

    
655
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
656
    if public is None:
657
        return None
658

    
659
    public = public.replace(' ', '').lower()
660
    if public == 'true':
661
        return True
662
    elif public == 'false' or public == '':
663
        return False
664
    raise faults.BadRequest('Bad X-Object-Public header value')
665

    
666

    
667
def raw_input_socket(request):
668
    """Return the socket for reading the rest of the request."""
669

    
670
    server_software = request.META.get('SERVER_SOFTWARE')
671
    if server_software and server_software.startswith('mod_python'):
672
        return request._req
673
    if 'wsgi.input' in request.environ:
674
        return request.environ['wsgi.input']
675
    raise NotImplemented('Unknown server software')
676

    
677
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB
678

    
679

    
680
def socket_read_iterator(request, length=0, blocksize=4096):
681
    """Return a maximum of blocksize data read from the socket in each iteration
682

683
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
684
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
685
    """
686

    
687
    sock = raw_input_socket(request)
688
    if length < 0:  # Chunked transfers
689
        # Small version (server does the dechunking).
690
        if (request.environ.get('mod_wsgi.input_chunked', None)
691
                or request.META['SERVER_SOFTWARE'].startswith('gunicorn')):
692
            while length < MAX_UPLOAD_SIZE:
693
                data = sock.read(blocksize)
694
                if data == '':
695
                    return
696
                yield data
697
            raise faults.BadRequest('Maximum size is reached')
698

    
699
        # Long version (do the dechunking).
700
        data = ''
701
        while length < MAX_UPLOAD_SIZE:
702
            # Get chunk size.
703
            if hasattr(sock, 'readline'):
704
                chunk_length = sock.readline()
705
            else:
706
                chunk_length = ''
707
                while chunk_length[-1:] != '\n':
708
                    chunk_length += sock.read(1)
709
                chunk_length.strip()
710
            pos = chunk_length.find(';')
711
            if pos >= 0:
712
                chunk_length = chunk_length[:pos]
713
            try:
714
                chunk_length = int(chunk_length, 16)
715
            except Exception:
716
                raise faults.BadRequest('Bad chunk size')
717
                                 # TODO: Change to something more appropriate.
718
            # Check if done.
719
            if chunk_length == 0:
720
                if len(data) > 0:
721
                    yield data
722
                return
723
            # Get the actual data.
724
            while chunk_length > 0:
725
                chunk = sock.read(min(chunk_length, blocksize))
726
                chunk_length -= len(chunk)
727
                if length > 0:
728
                    length += len(chunk)
729
                data += chunk
730
                if len(data) >= blocksize:
731
                    ret = data[:blocksize]
732
                    data = data[blocksize:]
733
                    yield ret
734
            sock.read(2)  # CRLF
735
        raise faults.BadRequest('Maximum size is reached')
736
    else:
737
        if length > MAX_UPLOAD_SIZE:
738
            raise faults.BadRequest('Maximum size is reached')
739
        while length > 0:
740
            data = sock.read(min(length, blocksize))
741
            if not data:
742
                raise faults.BadRequest()
743
            length -= len(data)
744
            yield data
745

    
746

    
747
class SaveToBackendHandler(FileUploadHandler):
748
    """Handle a file from an HTML form the django way."""
749

    
750
    def __init__(self, request=None):
751
        super(SaveToBackendHandler, self).__init__(request)
752
        self.backend = request.backend
753

    
754
    def put_data(self, length):
755
        if len(self.data) >= length:
756
            block = self.data[:length]
757
            self.file.hashmap.append(self.backend.put_block(block))
758
            self.md5.update(block)
759
            self.data = self.data[length:]
760

    
761
    def new_file(self, field_name, file_name, content_type,
762
                 content_length, charset=None):
763
        self.md5 = hashlib.md5()
764
        self.data = ''
765
        self.file = UploadedFile(
766
            name=file_name, content_type=content_type, charset=charset)
767
        self.file.size = 0
768
        self.file.hashmap = []
769

    
770
    def receive_data_chunk(self, raw_data, start):
771
        self.data += raw_data
772
        self.file.size += len(raw_data)
773
        self.put_data(self.request.backend.block_size)
774
        return None
775

    
776
    def file_complete(self, file_size):
777
        l = len(self.data)
778
        if l > 0:
779
            self.put_data(l)
780
        self.file.etag = self.md5.hexdigest().lower()
781
        return self.file
782

    
783

    
784
class ObjectWrapper(object):
785
    """Return the object's data block-per-block in each iteration.
786

787
    Read from the object using the offset and length provided
788
    in each entry of the range list.
789
    """
790

    
791
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
792
        self.backend = backend
793
        self.ranges = ranges
794
        self.sizes = sizes
795
        self.hashmaps = hashmaps
796
        self.boundary = boundary
797
        self.size = sum(self.sizes)
798

    
799
        self.file_index = 0
800
        self.block_index = 0
801
        self.block_hash = -1
802
        self.block = ''
803

    
804
        self.range_index = -1
805
        self.offset, self.length = self.ranges[0]
806

    
807
    def __iter__(self):
808
        return self
809

    
810
    def part_iterator(self):
811
        if self.length > 0:
812
            # Get the file for the current offset.
813
            file_size = self.sizes[self.file_index]
814
            while self.offset >= file_size:
815
                self.offset -= file_size
816
                self.file_index += 1
817
                file_size = self.sizes[self.file_index]
818

    
819
            # Get the block for the current position.
820
            self.block_index = int(self.offset / self.backend.block_size)
821
            if self.block_hash != \
822
                    self.hashmaps[self.file_index][self.block_index]:
823
                self.block_hash = self.hashmaps[
824
                    self.file_index][self.block_index]
825
                try:
826
                    self.block = self.backend.get_block(self.block_hash)
827
                except ItemNotExists:
828
                    raise faults.ItemNotFound('Block does not exist')
829

    
830
            # Get the data from the block.
831
            bo = self.offset % self.backend.block_size
832
            bs = self.backend.block_size
833
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
834
                    self.sizes[self.file_index] % self.backend.block_size):
835
                bs = self.sizes[self.file_index] % self.backend.block_size
836
            bl = min(self.length, bs - bo)
837
            data = self.block[bo:bo + bl]
838
            self.offset += bl
839
            self.length -= bl
840
            return data
841
        else:
842
            raise StopIteration
843

    
844
    def next(self):
845
        if len(self.ranges) == 1:
846
            return self.part_iterator()
847
        if self.range_index == len(self.ranges):
848
            raise StopIteration
849
        try:
850
            if self.range_index == -1:
851
                raise StopIteration
852
            return self.part_iterator()
853
        except StopIteration:
854
            self.range_index += 1
855
            out = []
856
            if self.range_index < len(self.ranges):
857
                # Part header.
858
                self.offset, self.length = self.ranges[self.range_index]
859
                self.file_index = 0
860
                if self.range_index > 0:
861
                    out.append('')
862
                out.append('--' + self.boundary)
863
                out.append('Content-Range: bytes %d-%d/%d' % (
864
                    self.offset, self.offset + self.length - 1, self.size))
865
                out.append('Content-Transfer-Encoding: binary')
866
                out.append('')
867
                out.append('')
868
                return '\r\n'.join(out)
869
            else:
870
                # Footer.
871
                out.append('')
872
                out.append('--' + self.boundary + '--')
873
                out.append('')
874
                return '\r\n'.join(out)
875

    
876

    
877
def object_data_response(request, sizes, hashmaps, meta, public=False):
878
    """Get the HttpResponse object for replying with the object's data."""
879

    
880
    # Range handling.
881
    size = sum(sizes)
882
    ranges = get_range(request, size)
883
    if ranges is None:
884
        ranges = [(0, size)]
885
        ret = 200
886
    else:
887
        check = [True for offset, length in ranges if
888
                 length <= 0 or length > size or
889
                 offset < 0 or offset >= size or
890
                 offset + length > size]
891
        if len(check) > 0:
892
            raise faults.RangeNotSatisfiable(
893
                'Requested range exceeds object limits')
894
        ret = 206
895
        if_range = request.META.get('HTTP_IF_RANGE')
896
        if if_range:
897
            try:
898
                # Modification time has passed instead.
899
                last_modified = parse_http_date(if_range)
900
                if last_modified != meta['modified']:
901
                    ranges = [(0, size)]
902
                    ret = 200
903
            except ValueError:
904
                if if_range != meta['checksum']:
905
                    ranges = [(0, size)]
906
                    ret = 200
907

    
908
    if ret == 206 and len(ranges) > 1:
909
        boundary = uuid.uuid4().hex
910
    else:
911
        boundary = ''
912
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
913
    response = HttpResponse(wrapper, status=ret)
914
    put_object_headers(
915
        response, meta, restricted=public,
916
        token=getattr(request, 'token', None))
917
    if ret == 206:
918
        if len(ranges) == 1:
919
            offset, length = ranges[0]
920
            response[
921
                'Content-Length'] = length  # Update with the correct length.
922
            response['Content-Range'] = 'bytes %d-%d/%d' % (
923
                offset, offset + length - 1, size)
924
        else:
925
            del(response['Content-Length'])
926
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (
927
                boundary,)
928
    return response
929

    
930

    
931
def put_object_block(request, hashmap, data, offset):
932
    """Put one block of data at the given offset."""
933

    
934
    bi = int(offset / request.backend.block_size)
935
    bo = offset % request.backend.block_size
936
    bl = min(len(data), request.backend.block_size - bo)
937
    if bi < len(hashmap):
938
        hashmap[bi] = request.backend.update_block(hashmap[bi], data[:bl], bo)
939
    else:
940
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
941
    return bl  # Return ammount of data written.
942

    
943

    
944
def hashmap_md5(backend, hashmap, size):
945
    """Produce the MD5 sum from the data in the hashmap."""
946

    
947
    # TODO: Search backend for the MD5 of another object
948
    #       with the same hashmap and size...
949
    md5 = hashlib.md5()
950
    bs = backend.block_size
951
    for bi, hash in enumerate(hashmap):
952
        data = backend.get_block(hash)  # Blocks come in padded.
953
        if bi == len(hashmap) - 1:
954
            data = data[:size % bs]
955
        md5.update(data)
956
    return md5.hexdigest().lower()
957

    
958

    
959
def simple_list_response(request, l):
960
    if request.serialization == 'text':
961
        return '\n'.join(l) + '\n'
962
    if request.serialization == 'xml':
963
        return render_to_string('items.xml', {'items': l})
964
    if request.serialization == 'json':
965
        return json.dumps(l)
966

    
967

    
968
from pithos.backends.util import PithosBackendPool
969

    
970
if RADOS_STORAGE:
971
    BLOCK_PARAMS = {'mappool': RADOS_POOL_MAPS,
972
                    'blockpool': RADOS_POOL_BLOCKS, }
973
else:
974
    BLOCK_PARAMS = {'mappool': None,
975
                    'blockpool': None, }
976

    
977

    
978
_pithos_backend_pool = PithosBackendPool(
979
    size=BACKEND_POOL_SIZE,
980
    db_module=BACKEND_DB_MODULE,
981
    db_connection=BACKEND_DB_CONNECTION,
982
    block_module=BACKEND_BLOCK_MODULE,
983
    block_path=BACKEND_BLOCK_PATH,
984
    block_umask=BACKEND_BLOCK_UMASK,
985
    queue_module=BACKEND_QUEUE_MODULE,
986
    queue_hosts=BACKEND_QUEUE_HOSTS,
987
    queue_exchange=BACKEND_QUEUE_EXCHANGE,
988
    quotaholder_enabled=USE_QUOTAHOLDER,
989
    quotaholder_url=QUOTAHOLDER_URL,
990
    quotaholder_token=QUOTAHOLDER_TOKEN,
991
    quotaholder_client_poolsize=QUOTAHOLDER_POOLSIZE,
992
    free_versioning=BACKEND_FREE_VERSIONING,
993
    block_params=BLOCK_PARAMS,
994
    public_url_security=PUBLIC_URL_SECURITY,
995
    public_url_alphabet=PUBLIC_URL_ALPHABET,
996
    account_quota_policy=BACKEND_ACCOUNT_QUOTA,
997
    container_quota_policy=BACKEND_CONTAINER_QUOTA,
998
    container_versioning_policy=BACKEND_VERSIONING)
999

    
1000

    
1001
def get_backend():
1002
    backend = _pithos_backend_pool.pool_get()
1003
    backend.messages = []
1004
    return backend
1005

    
1006

    
1007
def update_request_headers(request):
1008
    # Handle URL-encoded keys and values.
1009
    meta = dict([(
1010
        k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
1011
    for k, v in meta.iteritems():
1012
        try:
1013
            k.decode('ascii')
1014
            v.decode('ascii')
1015
        except UnicodeDecodeError:
1016
            raise faults.BadRequest('Bad character in headers.')
1017
        if '%' in k or '%' in v:
1018
            del(request.META[k])
1019
            request.META[unquote(k)] = smart_unicode(unquote(
1020
                v), strings_only=True)
1021

    
1022

    
1023
def update_response_headers(request, response):
1024
    if (not response.has_header('Content-Length') and
1025
        not (response.has_header('Content-Type') and
1026
             response['Content-Type'].startswith('multipart/byteranges'))):
1027
        response['Content-Length'] = len(response.content)
1028

    
1029
    # URL-encode unicode in headers.
1030
    meta = response.items()
1031
    for k, v in meta:
1032
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
1033
                k.startswith('X-Object-') or k.startswith('Content-')):
1034
            del(response[k])
1035
            response[quote(k)] = quote(v, safe='/=,:@; ')
1036

    
1037

    
1038
def get_pithos_usage(token):
1039
    """Get Pithos Usage from astakos."""
1040
    astakos_url = ASTAKOS_URL + "im/authenticate"
1041
    user_info = user_for_token(token, astakos_url, usage=True)
1042
    usage = user_info.get("usage", [])
1043
    for u in usage:
1044
        if u.get('name') == 'pithos+.diskspace':
1045
            return u
1046

    
1047

    
1048
def api_method(http_method=None, user_required=True, logger=None,
1049
               format_allowed=False):
1050
    def decorator(func):
1051
        @api.api_method(http_method=http_method, user_required=user_required,
1052
                        logger=logger, format_allowed=format_allowed,
1053
                        astakos_url=ASTAKOS_URL)
1054
        @wraps(func)
1055
        def wrapper(request, *args, **kwargs):
1056
            # The args variable may contain up to (account, container, object).
1057
            if len(args) > 1 and len(args[1]) > 256:
1058
                raise faults.BadRequest("Container name too large")
1059
            if len(args) > 2 and len(args[2]) > 1024:
1060
                raise faults.BadRequest('Object name too large.')
1061

    
1062
            try:
1063
                # Add a PithosBackend as attribute of the request object
1064
                request.backend = get_backend()
1065
                # Many API method expect thet X-Auth-Token in request,token
1066
                request.token = request.x_auth_token
1067
                update_request_headers(request)
1068
                response = func(request, *args, **kwargs)
1069
                update_response_headers(request, response)
1070
                return response
1071
            finally:
1072
                # Always close PithosBackend connection
1073
                if getattr(request, "backend", None) is not None:
1074
                    request.backend.close()
1075
        return wrapper
1076
    return decorator