Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ a3fcee5b

History | View | Annotate | Download (39.2 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from functools import wraps
35
from datetime import datetime
36
from urllib import quote, unquote
37

    
38
from django.http import HttpResponse
39
from django.template.loader import render_to_string
40
from django.utils import simplejson as json
41
from django.utils.http import http_date, parse_etags
42
from django.utils.encoding import smart_unicode, smart_str
43
from django.core.files.uploadhandler import FileUploadHandler
44
from django.core.files.uploadedfile import UploadedFile
45
from django.core.urlresolvers import reverse
46

    
47
from snf_django.lib.api.parsedate import parse_http_date_safe, parse_http_date
48
from snf_django.lib import api
49
from snf_django.lib.api import faults, utils
50

    
51
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
52
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
53
                                 BACKEND_BLOCK_UMASK,
54
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
55
                                 BACKEND_QUEUE_EXCHANGE,
56
                                 ASTAKOSCLIENT_POOLSIZE,
57
                                 SERVICE_TOKEN,
58
                                 ASTAKOS_BASE_URL,
59
                                 BACKEND_ACCOUNT_QUOTA, BACKEND_CONTAINER_QUOTA,
60
                                 BACKEND_VERSIONING,
61
                                 BACKEND_FREE_VERSIONING, BACKEND_POOL_SIZE,
62
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
63
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
64
                                 PUBLIC_URL_SECURITY,
65
                                 PUBLIC_URL_ALPHABET,
66
                                 COOKIE_NAME, BASE_URL)
67
from pithos.api.resources import resources
68
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
69
                                  VersionNotExists)
70

    
71
from synnefo.lib import join_urls
72

    
73
from astakosclient import AstakosClient
74
from astakosclient.errors import NoUserName, NoUUID
75

    
76
import logging
77
import re
78
import hashlib
79
import uuid
80
import decimal
81

    
82
logger = logging.getLogger(__name__)
83

    
84

    
85
def json_encode_decimal(obj):
86
    if isinstance(obj, decimal.Decimal):
87
        return str(obj)
88
    raise TypeError(repr(obj) + " is not JSON serializable")
89

    
90

    
91
def rename_meta_key(d, old, new):
92
    if old not in d:
93
        return
94
    d[new] = d[old]
95
    del(d[old])
96

    
97

    
98
def printable_header_dict(d):
99
    """Format a meta dictionary for printing out json/xml.
100

101
    Convert all keys to lower case and replace dashes with underscores.
102
    Format 'last_modified' timestamp.
103
    """
104

    
105
    if 'last_modified' in d and d['last_modified']:
106
        d['last_modified'] = utils.isoformat(
107
            datetime.fromtimestamp(d['last_modified']))
108
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
109

    
110

    
111
def format_header_key(k):
112
    """Convert underscores to dashes and capitalize intra-dash strings."""
113
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
114

    
115

    
116
def get_header_prefix(request, prefix):
117
    """Get all prefix-* request headers in a dict.
118
       Reformat keys with format_header_key()."""
119

    
120
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
121
    # TODO: Document or remove '~' replacing.
122
    return dict([(format_header_key(k[5:]), v.replace('~', ''))
123
                for k, v in request.META.iteritems()
124
                if k.startswith(prefix) and len(k) > len(prefix)])
125

    
126

    
127
def check_meta_headers(meta):
128
    if len(meta) > 90:
129
        raise faults.BadRequest('Too many headers.')
130
    for k, v in meta.iteritems():
131
        if len(k) > 128:
132
            raise faults.BadRequest('Header name too large.')
133
        if len(v) > 256:
134
            raise faults.BadRequest('Header value too large.')
135

    
136

    
137
def get_account_headers(request):
138
    meta = get_header_prefix(request, 'X-Account-Meta-')
139
    check_meta_headers(meta)
140
    groups = {}
141
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
142
        n = k[16:].lower()
143
        if '-' in n or '_' in n:
144
            raise faults.BadRequest('Bad characters in group name')
145
        groups[n] = v.replace(' ', '').split(',')
146
        while '' in groups[n]:
147
            groups[n].remove('')
148
    return meta, groups
149

    
150

    
151
def put_account_headers(response, meta, groups, policy):
152
    if 'count' in meta:
153
        response['X-Account-Container-Count'] = meta['count']
154
    if 'bytes' in meta:
155
        response['X-Account-Bytes-Used'] = meta['bytes']
156
    response['Last-Modified'] = http_date(int(meta['modified']))
157
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
158
        response[smart_str(
159
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
160
    if 'until_timestamp' in meta:
161
        response['X-Account-Until-Timestamp'] = http_date(
162
            int(meta['until_timestamp']))
163
    for k, v in groups.iteritems():
164
        k = smart_str(k, strings_only=True)
165
        k = format_header_key('X-Account-Group-' + k)
166
        v = smart_str(','.join(v), strings_only=True)
167
        response[k] = v
168
    for k, v in policy.iteritems():
169
        response[smart_str(format_header_key('X-Account-Policy-' + k),
170
                 strings_only=True)] = smart_str(v, strings_only=True)
171

    
172

    
173
def get_container_headers(request):
174
    meta = get_header_prefix(request, 'X-Container-Meta-')
175
    check_meta_headers(meta)
176
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in
177
                  get_header_prefix(request,
178
                                    'X-Container-Policy-').iteritems()])
179
    return meta, policy
180

    
181

    
182
def put_container_headers(request, response, meta, policy):
183
    if 'count' in meta:
184
        response['X-Container-Object-Count'] = meta['count']
185
    if 'bytes' in meta:
186
        response['X-Container-Bytes-Used'] = meta['bytes']
187
    response['Last-Modified'] = http_date(int(meta['modified']))
188
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
189
        response[smart_str(
190
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
191
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
192
         if x.startswith('X-Object-Meta-')]
193
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
194
    response['X-Container-Block-Size'] = request.backend.block_size
195
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
196
    if 'until_timestamp' in meta:
197
        response['X-Container-Until-Timestamp'] = http_date(
198
            int(meta['until_timestamp']))
199
    for k, v in policy.iteritems():
200
        response[smart_str(format_header_key('X-Container-Policy-' + k),
201
                           strings_only=True)] = smart_str(v, strings_only=True)
202

    
203

    
204
def get_object_headers(request):
205
    content_type = request.META.get('CONTENT_TYPE', None)
206
    meta = get_header_prefix(request, 'X-Object-Meta-')
207
    check_meta_headers(meta)
208
    if request.META.get('HTTP_CONTENT_ENCODING'):
209
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
210
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
211
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
212
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
213
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
214
    return content_type, meta, get_sharing(request), get_public(request)
215

    
216

    
217
def put_object_headers(response, meta, restricted=False, token=None):
218
    response['ETag'] = meta['checksum']
219
    response['Content-Length'] = meta['bytes']
220
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
221
    response['Last-Modified'] = http_date(int(meta['modified']))
222
    if not restricted:
223
        response['X-Object-Hash'] = meta['hash']
224
        response['X-Object-UUID'] = meta['uuid']
225
        if TRANSLATE_UUIDS:
226
            meta['modified_by'] = \
227
                retrieve_displayname(token, meta['modified_by'])
228
        response['X-Object-Modified-By'] = smart_str(
229
            meta['modified_by'], strings_only=True)
230
        response['X-Object-Version'] = meta['version']
231
        response['X-Object-Version-Timestamp'] = http_date(
232
            int(meta['version_timestamp']))
233
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
234
            response[smart_str(
235
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
236
        for k in (
237
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
238
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
239
                'X-Object-Public'):
240
            if k in meta:
241
                response[k] = smart_str(meta[k], strings_only=True)
242
    else:
243
        for k in ('Content-Encoding', 'Content-Disposition'):
244
            if k in meta:
245
                response[k] = smart_str(meta[k], strings_only=True)
246

    
247

    
248
def update_manifest_meta(request, v_account, meta):
249
    """Update metadata if the object has an X-Object-Manifest."""
250

    
251
    if 'X-Object-Manifest' in meta:
252
        etag = ''
253
        bytes = 0
254
        try:
255
            src_container, src_name = split_container_object_string(
256
                '/' + meta['X-Object-Manifest'])
257
            objects = request.backend.list_objects(
258
                request.user_uniq, v_account,
259
                src_container, prefix=src_name, virtual=False)
260
            for x in objects:
261
                src_meta = request.backend.get_object_meta(request.user_uniq,
262
                                                           v_account,
263
                                                           src_container,
264
                                                           x[0], 'pithos', x[1])
265
                etag += src_meta['checksum']
266
                bytes += src_meta['bytes']
267
        except:
268
            # Ignore errors.
269
            return
270
        meta['bytes'] = bytes
271
        md5 = hashlib.md5()
272
        md5.update(etag)
273
        meta['checksum'] = md5.hexdigest().lower()
274

    
275

    
276
def is_uuid(str):
277
    if str is None:
278
        return False
279
    try:
280
        uuid.UUID(str)
281
    except ValueError:
282
        return False
283
    else:
284
        return True
285

    
286

    
287
##########################
288
# USER CATALOG utilities #
289
##########################
290

    
291
def retrieve_displayname(token, uuid, fail_silently=True):
292
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
293
                            logger=logger)
294
    try:
295
        displayname = astakos.get_username(token, uuid)
296
    except NoUserName:
297
        if not fail_silently:
298
            raise ItemNotExists(uuid)
299
        else:
300
            # just return the uuid
301
            return uuid
302
    return displayname
303

    
304

    
305
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
306
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
307
                            logger=logger)
308
    catalog = astakos.get_usernames(token, uuids) or {}
309
    missing = list(set(uuids) - set(catalog))
310
    if missing and not fail_silently:
311
        raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
312
    return catalog if return_dict else [catalog.get(i) for i in uuids]
313

    
314

    
315
def retrieve_uuid(token, displayname):
316
    if is_uuid(displayname):
317
        return displayname
318

    
319
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
320
                            logger=logger)
321
    try:
322
        uuid = astakos.get_uuid(token, displayname)
323
    except NoUUID:
324
        raise ItemNotExists(displayname)
325
    return uuid
326

    
327

    
328
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
329
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
330
                            logger=logger)
331
    catalog = astakos.get_uuids(token, displaynames) or {}
332
    missing = list(set(displaynames) - set(catalog))
333
    if missing and not fail_silently:
334
        raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
335
    return catalog if return_dict else [catalog.get(i) for i in displaynames]
336

    
337

    
338
def replace_permissions_displayname(token, holder):
339
    if holder == '*':
340
        return holder
341
    try:
342
        # check first for a group permission
343
        account, group = holder.split(':', 1)
344
    except ValueError:
345
        return retrieve_uuid(token, holder)
346
    else:
347
        return ':'.join([retrieve_uuid(token, account), group])
348

    
349

    
350
def replace_permissions_uuid(token, holder):
351
    if holder == '*':
352
        return holder
353
    try:
354
        # check first for a group permission
355
        account, group = holder.split(':', 1)
356
    except ValueError:
357
        return retrieve_displayname(token, holder)
358
    else:
359
        return ':'.join([retrieve_displayname(token, account), group])
360

    
361

    
362
def update_sharing_meta(request, permissions, v_account,
363
                        v_container, v_object, meta):
364
    if permissions is None:
365
        return
366
    allowed, perm_path, perms = permissions
367
    if len(perms) == 0:
368
        return
369

    
370
    # replace uuid with displayname
371
    if TRANSLATE_UUIDS:
372
        perms['read'] = [replace_permissions_uuid(
373
            getattr(request, 'token', None), x)
374
            for x in perms.get('read', [])]
375
        perms['write'] = [replace_permissions_uuid(
376
            getattr(request, 'token', None), x)
377
            for x in perms.get('write', [])]
378

    
379
    ret = []
380

    
381
    r = ','.join(perms.get('read', []))
382
    if r:
383
        ret.append('read=' + r)
384
    w = ','.join(perms.get('write', []))
385
    if w:
386
        ret.append('write=' + w)
387
    meta['X-Object-Sharing'] = '; '.join(ret)
388
    if '/'.join((v_account, v_container, v_object)) != perm_path:
389
        meta['X-Object-Shared-By'] = perm_path
390
    if request.user_uniq != v_account:
391
        meta['X-Object-Allowed-To'] = allowed
392

    
393

    
394
def update_public_meta(public, meta):
395
    if not public:
396
        return
397
    meta['X-Object-Public'] = join_urls(
398
        BASE_URL, reverse('pithos.api.public.public_demux', args=(public,)))
399

    
400

    
401
def validate_modification_preconditions(request, meta):
402
    """Check that the modified timestamp conforms with the preconditions set."""
403

    
404
    if 'modified' not in meta:
405
        return  # TODO: Always return?
406

    
407
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
408
    if if_modified_since is not None:
409
        if_modified_since = parse_http_date_safe(if_modified_since)
410
    if (if_modified_since is not None
411
            and int(meta['modified']) <= if_modified_since):
412
        raise faults.NotModified('Resource has not been modified')
413

    
414
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
415
    if if_unmodified_since is not None:
416
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
417
    if (if_unmodified_since is not None
418
            and int(meta['modified']) > if_unmodified_since):
419
        raise faults.PreconditionFailed('Resource has been modified')
420

    
421

    
422
def validate_matching_preconditions(request, meta):
423
    """Check that the ETag conforms with the preconditions set."""
424

    
425
    etag = meta['checksum']
426
    if not etag:
427
        etag = None
428

    
429
    if_match = request.META.get('HTTP_IF_MATCH')
430
    if if_match is not None:
431
        if etag is None:
432
            raise faults.PreconditionFailed('Resource does not exist')
433
        if (if_match != '*'
434
                and etag not in [x.lower() for x in parse_etags(if_match)]):
435
            raise faults.PreconditionFailed('Resource ETag does not match')
436

    
437
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
438
    if if_none_match is not None:
439
        # TODO: If this passes, must ignore If-Modified-Since header.
440
        if etag is not None:
441
            if (if_none_match == '*'
442
                    or etag in [x.lower() for x in parse_etags(if_none_match)]):
443
                # TODO: Continue if an If-Modified-Since header is present.
444
                if request.method in ('HEAD', 'GET'):
445
                    raise faults.NotModified('Resource ETag matches')
446
                raise faults.PreconditionFailed(
447
                    'Resource exists or ETag matches')
448

    
449

    
450
def split_container_object_string(s):
451
    if not len(s) > 0 or s[0] != '/':
452
        raise ValueError
453
    s = s[1:]
454
    pos = s.find('/')
455
    if pos == -1 or pos == len(s) - 1:
456
        raise ValueError
457
    return s[:pos], s[(pos + 1):]
458

    
459

    
460
def copy_or_move_object(request, src_account, src_container, src_name,
461
                        dest_account, dest_container, dest_name,
462
                        move=False, delimiter=None):
463
    """Copy or move an object."""
464

    
465
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
466
        del(request.META['CONTENT_TYPE'])
467
    content_type, meta, permissions, public = get_object_headers(request)
468
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
469
    try:
470
        if move:
471
            version_id = request.backend.move_object(
472
                request.user_uniq, src_account, src_container, src_name,
473
                dest_account, dest_container, dest_name,
474
                content_type, 'pithos', meta, False, permissions, delimiter)
475
        else:
476
            version_id = request.backend.copy_object(
477
                request.user_uniq, src_account, src_container, src_name,
478
                dest_account, dest_container, dest_name,
479
                content_type, 'pithos', meta, False, permissions,
480
                src_version, delimiter)
481
    except NotAllowedError:
482
        raise faults.Forbidden('Not allowed')
483
    except (ItemNotExists, VersionNotExists):
484
        raise faults.ItemNotFound('Container or object does not exist')
485
    except ValueError:
486
        raise faults.BadRequest('Invalid sharing header')
487
    except QuotaError, e:
488
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
489
    if public is not None:
490
        try:
491
            request.backend.update_object_public(
492
                request.user_uniq, dest_account,
493
                dest_container, dest_name, public)
494
        except NotAllowedError:
495
            raise faults.Forbidden('Not allowed')
496
        except ItemNotExists:
497
            raise faults.ItemNotFound('Object does not exist')
498
    return version_id
499

    
500

    
501
def get_int_parameter(p):
502
    if p is not None:
503
        try:
504
            p = int(p)
505
        except ValueError:
506
            return None
507
        if p < 0:
508
            return None
509
    return p
510

    
511

    
512
def get_content_length(request):
513
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
514
    if content_length is None:
515
        raise faults.LengthRequired('Missing or invalid Content-Length header')
516
    return content_length
517

    
518

    
519
def get_range(request, size):
520
    """Parse a Range header from the request.
521

522
    Either returns None, when the header is not existent or should be ignored,
523
    or a list of (offset, length) tuples - should be further checked.
524
    """
525

    
526
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
527
    if not ranges.startswith('bytes='):
528
        return None
529

    
530
    ret = []
531
    for r in (x.strip() for x in ranges[6:].split(',')):
532
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
533
        m = p.match(r)
534
        if not m:
535
            return None
536
        offset = m.group('offset')
537
        upto = m.group('upto')
538
        if offset == '' and upto == '':
539
            return None
540

    
541
        if offset != '':
542
            offset = int(offset)
543
            if upto != '':
544
                upto = int(upto)
545
                if offset > upto:
546
                    return None
547
                ret.append((offset, upto - offset + 1))
548
            else:
549
                ret.append((offset, size - offset))
550
        else:
551
            length = int(upto)
552
            ret.append((size - length, length))
553

    
554
    return ret
555

    
556

    
557
def get_content_range(request):
558
    """Parse a Content-Range header from the request.
559

560
    Either returns None, when the header is not existent or should be ignored,
561
    or an (offset, length, total) tuple - check as length, total may be None.
562
    Returns (None, None, None) if the provided range is '*/*'.
563
    """
564

    
565
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
566
    if not ranges:
567
        return None
568

    
569
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
570
    m = p.match(ranges)
571
    if not m:
572
        if ranges == 'bytes */*':
573
            return (None, None, None)
574
        return None
575
    offset = int(m.group('offset'))
576
    upto = m.group('upto')
577
    total = m.group('total')
578
    if upto != '':
579
        upto = int(upto)
580
    else:
581
        upto = None
582
    if total != '*':
583
        total = int(total)
584
    else:
585
        total = None
586
    if (upto is not None and offset > upto) or \
587
        (total is not None and offset >= total) or \
588
            (total is not None and upto is not None and upto >= total):
589
        return None
590

    
591
    if upto is None:
592
        length = None
593
    else:
594
        length = upto - offset + 1
595
    return (offset, length, total)
596

    
597

    
598
def get_sharing(request):
599
    """Parse an X-Object-Sharing header from the request.
600

601
    Raises BadRequest on error.
602
    """
603

    
604
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
605
    if permissions is None:
606
        return None
607

    
608
    # TODO: Document or remove '~' replacing.
609
    permissions = permissions.replace('~', '')
610

    
611
    ret = {}
612
    permissions = permissions.replace(' ', '')
613
    if permissions == '':
614
        return ret
615
    for perm in (x for x in permissions.split(';')):
616
        if perm.startswith('read='):
617
            ret['read'] = list(set(
618
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
619
            if '' in ret['read']:
620
                ret['read'].remove('')
621
            if '*' in ret['read']:
622
                ret['read'] = ['*']
623
            if len(ret['read']) == 0:
624
                raise faults.BadRequest(
625
                    'Bad X-Object-Sharing header value: invalid length')
626
        elif perm.startswith('write='):
627
            ret['write'] = list(set(
628
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
629
            if '' in ret['write']:
630
                ret['write'].remove('')
631
            if '*' in ret['write']:
632
                ret['write'] = ['*']
633
            if len(ret['write']) == 0:
634
                raise faults.BadRequest(
635
                    'Bad X-Object-Sharing header value: invalid length')
636
        else:
637
            raise faults.BadRequest(
638
                'Bad X-Object-Sharing header value: missing prefix')
639

    
640
    # replace displayname with uuid
641
    if TRANSLATE_UUIDS:
642
        try:
643
            ret['read'] = [replace_permissions_displayname(
644
                getattr(request, 'token', None), x)
645
                for x in ret.get('read', [])]
646
            ret['write'] = [replace_permissions_displayname(
647
                getattr(request, 'token', None), x)
648
                for x in ret.get('write', [])]
649
        except ItemNotExists, e:
650
            raise faults.BadRequest(
651
                'Bad X-Object-Sharing header value: unknown account: %s' % e)
652

    
653
    # Keep duplicates only in write list.
654
    dups = [x for x in ret.get(
655
        'read', []) if x in ret.get('write', []) and x != '*']
656
    if dups:
657
        for x in dups:
658
            ret['read'].remove(x)
659
        if len(ret['read']) == 0:
660
            del(ret['read'])
661

    
662
    return ret
663

    
664

    
665
def get_public(request):
666
    """Parse an X-Object-Public header from the request.
667

668
    Raises BadRequest on error.
669
    """
670

    
671
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
672
    if public is None:
673
        return None
674

    
675
    public = public.replace(' ', '').lower()
676
    if public == 'true':
677
        return True
678
    elif public == 'false' or public == '':
679
        return False
680
    raise faults.BadRequest('Bad X-Object-Public header value')
681

    
682

    
683
def raw_input_socket(request):
684
    """Return the socket for reading the rest of the request."""
685

    
686
    server_software = request.META.get('SERVER_SOFTWARE')
687
    if server_software and server_software.startswith('mod_python'):
688
        return request._req
689
    if 'wsgi.input' in request.environ:
690
        return request.environ['wsgi.input']
691
    raise NotImplemented('Unknown server software')
692

    
693
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB
694

    
695

    
696
def socket_read_iterator(request, length=0, blocksize=4096):
697
    """Return a maximum of blocksize data read from the socket in each iteration
698

699
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
700
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
701
    """
702

    
703
    sock = raw_input_socket(request)
704
    if length < 0:  # Chunked transfers
705
        # Small version (server does the dechunking).
706
        if (request.environ.get('mod_wsgi.input_chunked', None)
707
                or request.META['SERVER_SOFTWARE'].startswith('gunicorn')):
708
            while length < MAX_UPLOAD_SIZE:
709
                data = sock.read(blocksize)
710
                if data == '':
711
                    return
712
                yield data
713
            raise faults.BadRequest('Maximum size is reached')
714

    
715
        # Long version (do the dechunking).
716
        data = ''
717
        while length < MAX_UPLOAD_SIZE:
718
            # Get chunk size.
719
            if hasattr(sock, 'readline'):
720
                chunk_length = sock.readline()
721
            else:
722
                chunk_length = ''
723
                while chunk_length[-1:] != '\n':
724
                    chunk_length += sock.read(1)
725
                chunk_length.strip()
726
            pos = chunk_length.find(';')
727
            if pos >= 0:
728
                chunk_length = chunk_length[:pos]
729
            try:
730
                chunk_length = int(chunk_length, 16)
731
            except Exception:
732
                raise faults.BadRequest('Bad chunk size')
733
                                 # TODO: Change to something more appropriate.
734
            # Check if done.
735
            if chunk_length == 0:
736
                if len(data) > 0:
737
                    yield data
738
                return
739
            # Get the actual data.
740
            while chunk_length > 0:
741
                chunk = sock.read(min(chunk_length, blocksize))
742
                chunk_length -= len(chunk)
743
                if length > 0:
744
                    length += len(chunk)
745
                data += chunk
746
                if len(data) >= blocksize:
747
                    ret = data[:blocksize]
748
                    data = data[blocksize:]
749
                    yield ret
750
            sock.read(2)  # CRLF
751
        raise faults.BadRequest('Maximum size is reached')
752
    else:
753
        if length > MAX_UPLOAD_SIZE:
754
            raise faults.BadRequest('Maximum size is reached')
755
        while length > 0:
756
            data = sock.read(min(length, blocksize))
757
            if not data:
758
                raise faults.BadRequest()
759
            length -= len(data)
760
            yield data
761

    
762

    
763
class SaveToBackendHandler(FileUploadHandler):
764
    """Handle a file from an HTML form the django way."""
765

    
766
    def __init__(self, request=None):
767
        super(SaveToBackendHandler, self).__init__(request)
768
        self.backend = request.backend
769

    
770
    def put_data(self, length):
771
        if len(self.data) >= length:
772
            block = self.data[:length]
773
            self.file.hashmap.append(self.backend.put_block(block))
774
            self.md5.update(block)
775
            self.data = self.data[length:]
776

    
777
    def new_file(self, field_name, file_name, content_type,
778
                 content_length, charset=None):
779
        self.md5 = hashlib.md5()
780
        self.data = ''
781
        self.file = UploadedFile(
782
            name=file_name, content_type=content_type, charset=charset)
783
        self.file.size = 0
784
        self.file.hashmap = []
785

    
786
    def receive_data_chunk(self, raw_data, start):
787
        self.data += raw_data
788
        self.file.size += len(raw_data)
789
        self.put_data(self.request.backend.block_size)
790
        return None
791

    
792
    def file_complete(self, file_size):
793
        l = len(self.data)
794
        if l > 0:
795
            self.put_data(l)
796
        self.file.etag = self.md5.hexdigest().lower()
797
        return self.file
798

    
799

    
800
class ObjectWrapper(object):
801
    """Return the object's data block-per-block in each iteration.
802

803
    Read from the object using the offset and length provided
804
    in each entry of the range list.
805
    """
806

    
807
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
808
        self.backend = backend
809
        self.ranges = ranges
810
        self.sizes = sizes
811
        self.hashmaps = hashmaps
812
        self.boundary = boundary
813
        self.size = sum(self.sizes)
814

    
815
        self.file_index = 0
816
        self.block_index = 0
817
        self.block_hash = -1
818
        self.block = ''
819

    
820
        self.range_index = -1
821
        self.offset, self.length = self.ranges[0]
822

    
823
    def __iter__(self):
824
        return self
825

    
826
    def part_iterator(self):
827
        if self.length > 0:
828
            # Get the file for the current offset.
829
            file_size = self.sizes[self.file_index]
830
            while self.offset >= file_size:
831
                self.offset -= file_size
832
                self.file_index += 1
833
                file_size = self.sizes[self.file_index]
834

    
835
            # Get the block for the current position.
836
            self.block_index = int(self.offset / self.backend.block_size)
837
            if self.block_hash != \
838
                    self.hashmaps[self.file_index][self.block_index]:
839
                self.block_hash = self.hashmaps[
840
                    self.file_index][self.block_index]
841
                try:
842
                    self.block = self.backend.get_block(self.block_hash)
843
                except ItemNotExists:
844
                    raise faults.ItemNotFound('Block does not exist')
845

    
846
            # Get the data from the block.
847
            bo = self.offset % self.backend.block_size
848
            bs = self.backend.block_size
849
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
850
                    self.sizes[self.file_index] % self.backend.block_size):
851
                bs = self.sizes[self.file_index] % self.backend.block_size
852
            bl = min(self.length, bs - bo)
853
            data = self.block[bo:bo + bl]
854
            self.offset += bl
855
            self.length -= bl
856
            return data
857
        else:
858
            raise StopIteration
859

    
860
    def next(self):
861
        if len(self.ranges) == 1:
862
            return self.part_iterator()
863
        if self.range_index == len(self.ranges):
864
            raise StopIteration
865
        try:
866
            if self.range_index == -1:
867
                raise StopIteration
868
            return self.part_iterator()
869
        except StopIteration:
870
            self.range_index += 1
871
            out = []
872
            if self.range_index < len(self.ranges):
873
                # Part header.
874
                self.offset, self.length = self.ranges[self.range_index]
875
                self.file_index = 0
876
                if self.range_index > 0:
877
                    out.append('')
878
                out.append('--' + self.boundary)
879
                out.append('Content-Range: bytes %d-%d/%d' % (
880
                    self.offset, self.offset + self.length - 1, self.size))
881
                out.append('Content-Transfer-Encoding: binary')
882
                out.append('')
883
                out.append('')
884
                return '\r\n'.join(out)
885
            else:
886
                # Footer.
887
                out.append('')
888
                out.append('--' + self.boundary + '--')
889
                out.append('')
890
                return '\r\n'.join(out)
891

    
892

    
893
def object_data_response(request, sizes, hashmaps, meta, public=False):
894
    """Get the HttpResponse object for replying with the object's data."""
895

    
896
    # Range handling.
897
    size = sum(sizes)
898
    ranges = get_range(request, size)
899
    if ranges is None:
900
        ranges = [(0, size)]
901
        ret = 200
902
    else:
903
        check = [True for offset, length in ranges if
904
                 length <= 0 or length > size or
905
                 offset < 0 or offset >= size or
906
                 offset + length > size]
907
        if len(check) > 0:
908
            raise faults.RangeNotSatisfiable(
909
                'Requested range exceeds object limits')
910
        ret = 206
911
        if_range = request.META.get('HTTP_IF_RANGE')
912
        if if_range:
913
            try:
914
                # Modification time has passed instead.
915
                last_modified = parse_http_date(if_range)
916
                if last_modified != meta['modified']:
917
                    ranges = [(0, size)]
918
                    ret = 200
919
            except ValueError:
920
                if if_range != meta['checksum']:
921
                    ranges = [(0, size)]
922
                    ret = 200
923

    
924
    if ret == 206 and len(ranges) > 1:
925
        boundary = uuid.uuid4().hex
926
    else:
927
        boundary = ''
928
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
929
    response = HttpResponse(wrapper, status=ret)
930
    response.override_serialization = True
931
    put_object_headers(
932
        response, meta, restricted=public,
933
        token=getattr(request, 'token', None))
934
    if ret == 206:
935
        if len(ranges) == 1:
936
            offset, length = ranges[0]
937
            response[
938
                'Content-Length'] = length  # Update with the correct length.
939
            response['Content-Range'] = 'bytes %d-%d/%d' % (
940
                offset, offset + length - 1, size)
941
        else:
942
            del(response['Content-Length'])
943
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (
944
                boundary,)
945
    return response
946

    
947

    
948
def put_object_block(request, hashmap, data, offset):
949
    """Put one block of data at the given offset."""
950

    
951
    bi = int(offset / request.backend.block_size)
952
    bo = offset % request.backend.block_size
953
    bl = min(len(data), request.backend.block_size - bo)
954
    if bi < len(hashmap):
955
        hashmap[bi] = request.backend.update_block(hashmap[bi], data[:bl], bo)
956
    else:
957
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
958
    return bl  # Return ammount of data written.
959

    
960

    
961
def hashmap_md5(backend, hashmap, size):
962
    """Produce the MD5 sum from the data in the hashmap."""
963

    
964
    # TODO: Search backend for the MD5 of another object
965
    #       with the same hashmap and size...
966
    md5 = hashlib.md5()
967
    bs = backend.block_size
968
    for bi, hash in enumerate(hashmap):
969
        data = backend.get_block(hash)  # Blocks come in padded.
970
        if bi == len(hashmap) - 1:
971
            data = data[:size % bs]
972
        md5.update(data)
973
    return md5.hexdigest().lower()
974

    
975

    
976
def simple_list_response(request, l):
977
    if request.serialization == 'text':
978
        return '\n'.join(l) + '\n'
979
    if request.serialization == 'xml':
980
        return render_to_string('items.xml', {'items': l})
981
    if request.serialization == 'json':
982
        return json.dumps(l)
983

    
984

    
985
from pithos.backends.util import PithosBackendPool
986

    
987
if RADOS_STORAGE:
988
    BLOCK_PARAMS = {'mappool': RADOS_POOL_MAPS,
989
                    'blockpool': RADOS_POOL_BLOCKS, }
990
else:
991
    BLOCK_PARAMS = {'mappool': None,
992
                    'blockpool': None, }
993

    
994

    
995
_pithos_backend_pool = PithosBackendPool(
996
        size=BACKEND_POOL_SIZE,
997
        db_module=BACKEND_DB_MODULE,
998
        db_connection=BACKEND_DB_CONNECTION,
999
        block_module=BACKEND_BLOCK_MODULE,
1000
        block_path=BACKEND_BLOCK_PATH,
1001
        block_umask=BACKEND_BLOCK_UMASK,
1002
        queue_module=BACKEND_QUEUE_MODULE,
1003
        queue_hosts=BACKEND_QUEUE_HOSTS,
1004
        queue_exchange=BACKEND_QUEUE_EXCHANGE,
1005
        astakos_url=ASTAKOS_BASE_URL,
1006
        service_token=SERVICE_TOKEN,
1007
        astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
1008
        free_versioning=BACKEND_FREE_VERSIONING,
1009
        block_params=BLOCK_PARAMS,
1010
        public_url_security=PUBLIC_URL_SECURITY,
1011
        public_url_alphabet=PUBLIC_URL_ALPHABET,
1012
        account_quota_policy=BACKEND_ACCOUNT_QUOTA,
1013
        container_quota_policy=BACKEND_CONTAINER_QUOTA,
1014
        container_versioning_policy=BACKEND_VERSIONING)
1015

    
1016

    
1017
def get_backend():
1018
    backend = _pithos_backend_pool.pool_get()
1019
    backend.messages = []
1020
    return backend
1021

    
1022

    
1023
def update_request_headers(request):
1024
    # Handle URL-encoded keys and values.
1025
    meta = dict([(
1026
        k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
1027
    for k, v in meta.iteritems():
1028
        try:
1029
            k.decode('ascii')
1030
            v.decode('ascii')
1031
        except UnicodeDecodeError:
1032
            raise faults.BadRequest('Bad character in headers.')
1033
        if '%' in k or '%' in v:
1034
            del(request.META[k])
1035
            request.META[unquote(k)] = smart_unicode(unquote(
1036
                v), strings_only=True)
1037

    
1038

    
1039
def update_response_headers(request, response):
1040
    if (not response.has_header('Content-Length') and
1041
        not (response.has_header('Content-Type') and
1042
             response['Content-Type'].startswith('multipart/byteranges'))):
1043
        response['Content-Length'] = len(response.content)
1044

    
1045
    # URL-encode unicode in headers.
1046
    meta = response.items()
1047
    for k, v in meta:
1048
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
1049
                k.startswith('X-Object-') or k.startswith('Content-')):
1050
            del(response[k])
1051
            response[quote(k)] = quote(v, safe='/=,:@; ')
1052

    
1053

    
1054
def get_pithos_usage(token):
1055
    """Get Pithos Usage from astakos."""
1056
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
1057
                            logger=logger)
1058
    quotas = astakos.get_quotas(token)['system']
1059
    pithos_resources = [r['name'] for r in resources]
1060
    map(quotas.pop, filter(lambda k: k not in pithos_resources, quotas.keys()))
1061
    return quotas.popitem()[-1] # assume only one resource
1062

    
1063

    
1064
def api_method(http_method=None, user_required=True, logger=None,
1065
               format_allowed=False, default_serialization="json"):
1066
    def decorator(func):
1067
        @api.api_method(http_method=http_method, user_required=user_required,
1068
                        logger=logger, format_allowed=format_allowed,
1069
                        astakos_url=ASTAKOS_BASE_URL,
1070
                        default_serialization=default_serialization)
1071
        @wraps(func)
1072
        def wrapper(request, *args, **kwargs):
1073
            # The args variable may contain up to (account, container, object).
1074
            if len(args) > 1 and len(args[1]) > 256:
1075
                raise faults.BadRequest("Container name too large")
1076
            if len(args) > 2 and len(args[2]) > 1024:
1077
                raise faults.BadRequest('Object name too large.')
1078

    
1079
            try:
1080
                # Add a PithosBackend as attribute of the request object
1081
                request.backend = get_backend()
1082
                # Many API method expect thet X-Auth-Token in request,token
1083
                request.token = request.x_auth_token
1084
                update_request_headers(request)
1085
                response = func(request, *args, **kwargs)
1086
                update_response_headers(request, response)
1087
                return response
1088
            finally:
1089
                # Always close PithosBackend connection
1090
                if getattr(request, "backend", None) is not None:
1091
                    request.backend.close()
1092
        return wrapper
1093
    return decorator