Statistics
| Branch: | Tag: | Revision:

root / snf-pithos-app / pithos / api / util.py @ 87835e94

History | View | Annotate | Download (41.7 kB)

1
# Copyright 2011-2012 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from functools import wraps
35
from datetime import datetime
36
from urllib import quote, unquote
37

    
38
from django.http import (HttpResponse, HttpResponseRedirect, Http404,
39
                         HttpResponseForbidden)
40
from django.template.loader import render_to_string
41
from django.utils import simplejson as json
42
from django.utils.http import http_date, parse_etags
43
from django.utils.encoding import smart_unicode, smart_str
44
from django.core.files.uploadhandler import FileUploadHandler
45
from django.core.files.uploadedfile import UploadedFile
46
from django.core.urlresolvers import reverse
47

    
48
from snf_django.lib.api.parsedate import parse_http_date_safe, parse_http_date
49
from snf_django.lib import api
50
from snf_django.lib.api import faults, utils
51

    
52
from pithos.api.settings import (BACKEND_DB_MODULE, BACKEND_DB_CONNECTION,
53
                                 BACKEND_BLOCK_MODULE, BACKEND_BLOCK_PATH,
54
                                 BACKEND_BLOCK_UMASK,
55
                                 BACKEND_QUEUE_MODULE, BACKEND_QUEUE_HOSTS,
56
                                 BACKEND_QUEUE_EXCHANGE,
57
                                 ASTAKOSCLIENT_POOLSIZE,
58
                                 SERVICE_TOKEN,
59
                                 ASTAKOS_BASE_URL,
60
                                 BACKEND_ACCOUNT_QUOTA,
61
                                 BACKEND_CONTAINER_QUOTA,
62
                                 BACKEND_VERSIONING, BACKEND_FREE_VERSIONING,
63
                                 BACKEND_POOL_ENABLED, BACKEND_POOL_SIZE,
64
                                 BACKEND_BLOCK_SIZE, BACKEND_HASH_ALGORITHM,
65
                                 RADOS_STORAGE, RADOS_POOL_BLOCKS,
66
                                 RADOS_POOL_MAPS, TRANSLATE_UUIDS,
67
                                 PUBLIC_URL_SECURITY, PUBLIC_URL_ALPHABET,
68
                                 COOKIE_NAME, BASE_HOST, UPDATE_MD5, LOGIN_URL)
69

    
70
from pithos.api.resources import resources
71
from pithos.backends import connect_backend
72
from pithos.backends.base import (NotAllowedError, QuotaError, ItemNotExists,
73
                                  VersionNotExists)
74

    
75
from synnefo.lib import join_urls
76

    
77
from astakosclient import AstakosClient
78
from astakosclient.errors import NoUserName, NoUUID
79

    
80
import logging
81
import re
82
import hashlib
83
import uuid
84
import decimal
85

    
86
logger = logging.getLogger(__name__)
87

    
88

    
89
def json_encode_decimal(obj):
90
    if isinstance(obj, decimal.Decimal):
91
        return str(obj)
92
    raise TypeError(repr(obj) + " is not JSON serializable")
93

    
94

    
95
def rename_meta_key(d, old, new):
96
    if old not in d:
97
        return
98
    d[new] = d[old]
99
    del(d[old])
100

    
101

    
102
def printable_header_dict(d):
103
    """Format a meta dictionary for printing out json/xml.
104

105
    Convert all keys to lower case and replace dashes with underscores.
106
    Format 'last_modified' timestamp.
107
    """
108

    
109
    timestamps = ('last_modified', 'x_container_until_timestamp',
110
                  'x_acount_until_timestamp')
111
    for timestamp in timestamps:
112
        if timestamp in d and d[timestamp]:
113
            d[timestamp] = utils.isoformat(
114
                    datetime.fromtimestamp(d[timestamp]))
115
    return dict([(k.lower().replace('-', '_'), v) for k, v in d.iteritems()])
116

    
117

    
118
def format_header_key(k):
119
    """Convert underscores to dashes and capitalize intra-dash strings."""
120
    return '-'.join([x.capitalize() for x in k.replace('_', '-').split('-')])
121

    
122

    
123
def get_header_prefix(request, prefix):
124
    """Get all prefix-* request headers in a dict.
125
       Reformat keys with format_header_key()."""
126

    
127
    prefix = 'HTTP_' + prefix.upper().replace('-', '_')
128
    # TODO: Document or remove '~' replacing.
129
    return dict([(format_header_key(k[5:]), v.replace('~', ''))
130
                for k, v in request.META.iteritems()
131
                if k.startswith(prefix) and len(k) > len(prefix)])
132

    
133

    
134
def check_meta_headers(meta):
135
    if len(meta) > 90:
136
        raise faults.BadRequest('Too many headers.')
137
    for k, v in meta.iteritems():
138
        if len(k) > 128:
139
            raise faults.BadRequest('Header name too large.')
140
        if len(v) > 256:
141
            raise faults.BadRequest('Header value too large.')
142

    
143

    
144
def get_account_headers(request):
145
    meta = get_header_prefix(request, 'X-Account-Meta-')
146
    check_meta_headers(meta)
147
    groups = {}
148
    for k, v in get_header_prefix(request, 'X-Account-Group-').iteritems():
149
        n = k[16:].lower()
150
        if '-' in n or '_' in n:
151
            raise faults.BadRequest('Bad characters in group name')
152
        groups[n] = v.replace(' ', '').split(',')
153
        while '' in groups[n]:
154
            groups[n].remove('')
155
    return meta, groups
156

    
157

    
158
def put_account_headers(response, meta, groups, policy):
159
    if 'count' in meta:
160
        response['X-Account-Container-Count'] = meta['count']
161
    if 'bytes' in meta:
162
        response['X-Account-Bytes-Used'] = meta['bytes']
163
    response['Last-Modified'] = http_date(int(meta['modified']))
164
    for k in [x for x in meta.keys() if x.startswith('X-Account-Meta-')]:
165
        response[smart_str(
166
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
167
    if 'until_timestamp' in meta:
168
        response['X-Account-Until-Timestamp'] = http_date(
169
            int(meta['until_timestamp']))
170
    for k, v in groups.iteritems():
171
        k = smart_str(k, strings_only=True)
172
        k = format_header_key('X-Account-Group-' + k)
173
        v = smart_str(','.join(v), strings_only=True)
174
        response[k] = v
175
    for k, v in policy.iteritems():
176
        response[smart_str(format_header_key('X-Account-Policy-' + k),
177
                 strings_only=True)] = smart_str(v, strings_only=True)
178

    
179

    
180
def get_container_headers(request):
181
    meta = get_header_prefix(request, 'X-Container-Meta-')
182
    check_meta_headers(meta)
183
    policy = dict([(k[19:].lower(), v.replace(' ', '')) for k, v in
184
                  get_header_prefix(request,
185
                                    'X-Container-Policy-').iteritems()])
186
    return meta, policy
187

    
188

    
189
def put_container_headers(request, response, meta, policy):
190
    if 'count' in meta:
191
        response['X-Container-Object-Count'] = meta['count']
192
    if 'bytes' in meta:
193
        response['X-Container-Bytes-Used'] = meta['bytes']
194
    response['Last-Modified'] = http_date(int(meta['modified']))
195
    for k in [x for x in meta.keys() if x.startswith('X-Container-Meta-')]:
196
        response[smart_str(
197
            k, strings_only=True)] = smart_str(meta[k], strings_only=True)
198
    l = [smart_str(x, strings_only=True) for x in meta['object_meta']
199
         if x.startswith('X-Object-Meta-')]
200
    response['X-Container-Object-Meta'] = ','.join([x[14:] for x in l])
201
    response['X-Container-Block-Size'] = request.backend.block_size
202
    response['X-Container-Block-Hash'] = request.backend.hash_algorithm
203
    if 'until_timestamp' in meta:
204
        response['X-Container-Until-Timestamp'] = http_date(
205
            int(meta['until_timestamp']))
206
    for k, v in policy.iteritems():
207
        response[smart_str(format_header_key('X-Container-Policy-' + k),
208
                           strings_only=True)] = smart_str(v,
209
                                                           strings_only=True)
210

    
211

    
212
def get_object_headers(request):
213
    content_type = request.META.get('CONTENT_TYPE', None)
214
    meta = get_header_prefix(request, 'X-Object-Meta-')
215
    check_meta_headers(meta)
216
    if request.META.get('HTTP_CONTENT_ENCODING'):
217
        meta['Content-Encoding'] = request.META['HTTP_CONTENT_ENCODING']
218
    if request.META.get('HTTP_CONTENT_DISPOSITION'):
219
        meta['Content-Disposition'] = request.META['HTTP_CONTENT_DISPOSITION']
220
    if request.META.get('HTTP_X_OBJECT_MANIFEST'):
221
        meta['X-Object-Manifest'] = request.META['HTTP_X_OBJECT_MANIFEST']
222
    return content_type, meta, get_sharing(request), get_public(request)
223

    
224

    
225
def put_object_headers(response, meta, restricted=False, token=None):
226
    response['ETag'] = meta['hash'] if not UPDATE_MD5 else meta['checksum']
227
    response['Content-Length'] = meta['bytes']
228
    response.override_serialization = True
229
    response['Content-Type'] = meta.get('type', 'application/octet-stream')
230
    response['Last-Modified'] = http_date(int(meta['modified']))
231
    if not restricted:
232
        response['X-Object-Hash'] = meta['hash']
233
        response['X-Object-UUID'] = meta['uuid']
234
        if TRANSLATE_UUIDS:
235
            meta['modified_by'] = \
236
                retrieve_displayname(token, meta['modified_by'])
237
        response['X-Object-Modified-By'] = smart_str(
238
            meta['modified_by'], strings_only=True)
239
        response['X-Object-Version'] = meta['version']
240
        response['X-Object-Version-Timestamp'] = http_date(
241
            int(meta['version_timestamp']))
242
        for k in [x for x in meta.keys() if x.startswith('X-Object-Meta-')]:
243
            response[smart_str(
244
                k, strings_only=True)] = smart_str(meta[k], strings_only=True)
245
        for k in (
246
            'Content-Encoding', 'Content-Disposition', 'X-Object-Manifest',
247
            'X-Object-Sharing', 'X-Object-Shared-By', 'X-Object-Allowed-To',
248
                'X-Object-Public'):
249
            if k in meta:
250
                response[k] = smart_str(meta[k], strings_only=True)
251
    else:
252
        for k in ('Content-Encoding', 'Content-Disposition'):
253
            if k in meta:
254
                response[k] = smart_str(meta[k], strings_only=True)
255

    
256

    
257
def update_manifest_meta(request, v_account, meta):
258
    """Update metadata if the object has an X-Object-Manifest."""
259

    
260
    if 'X-Object-Manifest' in meta:
261
        etag = ''
262
        bytes = 0
263
        try:
264
            src_container, src_name = split_container_object_string(
265
                '/' + meta['X-Object-Manifest'])
266
            objects = request.backend.list_objects(
267
                request.user_uniq, v_account,
268
                src_container, prefix=src_name, virtual=False)
269
            for x in objects:
270
                src_meta = request.backend.get_object_meta(
271
                    request.user_uniq, v_account, src_container, x[0],
272
                    'pithos', x[1])
273
                etag += (src_meta['hash'] if not UPDATE_MD5 else
274
                         src_meta['checksum'])
275
                bytes += src_meta['bytes']
276
        except:
277
            # Ignore errors.
278
            return
279
        meta['bytes'] = bytes
280
        md5 = hashlib.md5()
281
        md5.update(etag)
282
        meta['checksum'] = md5.hexdigest().lower()
283

    
284

    
285
def is_uuid(str):
286
    if str is None:
287
        return False
288
    try:
289
        uuid.UUID(str)
290
    except ValueError:
291
        return False
292
    else:
293
        return True
294

    
295

    
296
##########################
297
# USER CATALOG utilities #
298
##########################
299

    
300
def retrieve_displayname(token, uuid, fail_silently=True):
301
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
302
                            logger=logger)
303
    try:
304
        displayname = astakos.get_username(token, uuid)
305
    except NoUserName:
306
        if not fail_silently:
307
            raise ItemNotExists(uuid)
308
        else:
309
            # just return the uuid
310
            return uuid
311
    return displayname
312

    
313

    
314
def retrieve_displaynames(token, uuids, return_dict=False, fail_silently=True):
315
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
316
                            logger=logger)
317
    catalog = astakos.get_usernames(token, uuids) or {}
318
    missing = list(set(uuids) - set(catalog))
319
    if missing and not fail_silently:
320
        raise ItemNotExists('Unknown displaynames: %s' % ', '.join(missing))
321
    return catalog if return_dict else [catalog.get(i) for i in uuids]
322

    
323

    
324
def retrieve_uuid(token, displayname):
325
    if is_uuid(displayname):
326
        return displayname
327

    
328
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
329
                            logger=logger)
330
    try:
331
        uuid = astakos.get_uuid(token, displayname)
332
    except NoUUID:
333
        raise ItemNotExists(displayname)
334
    return uuid
335

    
336

    
337
def retrieve_uuids(token, displaynames, return_dict=False, fail_silently=True):
338
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
339
                            logger=logger)
340
    catalog = astakos.get_uuids(token, displaynames) or {}
341
    missing = list(set(displaynames) - set(catalog))
342
    if missing and not fail_silently:
343
        raise ItemNotExists('Unknown uuids: %s' % ', '.join(missing))
344
    return catalog if return_dict else [catalog.get(i) for i in displaynames]
345

    
346

    
347
def replace_permissions_displayname(token, holder):
348
    if holder == '*':
349
        return holder
350
    try:
351
        # check first for a group permission
352
        account, group = holder.split(':', 1)
353
    except ValueError:
354
        return retrieve_uuid(token, holder)
355
    else:
356
        return ':'.join([retrieve_uuid(token, account), group])
357

    
358

    
359
def replace_permissions_uuid(token, holder):
360
    if holder == '*':
361
        return holder
362
    try:
363
        # check first for a group permission
364
        account, group = holder.split(':', 1)
365
    except ValueError:
366
        return retrieve_displayname(token, holder)
367
    else:
368
        return ':'.join([retrieve_displayname(token, account), group])
369

    
370

    
371
def update_sharing_meta(request, permissions, v_account,
372
                        v_container, v_object, meta):
373
    if permissions is None:
374
        return
375
    allowed, perm_path, perms = permissions
376
    if len(perms) == 0:
377
        return
378

    
379
    # replace uuid with displayname
380
    if TRANSLATE_UUIDS:
381
        perms['read'] = [replace_permissions_uuid(
382
            getattr(request, 'token', None), x)
383
            for x in perms.get('read', [])]
384
        perms['write'] = [replace_permissions_uuid(
385
            getattr(request, 'token', None), x)
386
            for x in perms.get('write', [])]
387

    
388
    ret = []
389

    
390
    r = ','.join(perms.get('read', []))
391
    if r:
392
        ret.append('read=' + r)
393
    w = ','.join(perms.get('write', []))
394
    if w:
395
        ret.append('write=' + w)
396
    meta['X-Object-Sharing'] = '; '.join(ret)
397
    if '/'.join((v_account, v_container, v_object)) != perm_path:
398
        meta['X-Object-Shared-By'] = perm_path
399
    if request.user_uniq != v_account:
400
        meta['X-Object-Allowed-To'] = allowed
401

    
402

    
403
def update_public_meta(public, meta):
404
    if not public:
405
        return
406
    meta['X-Object-Public'] = join_urls(
407
        BASE_HOST, reverse('pithos.api.public.public_demux', args=(public,)))
408

    
409

    
410
def validate_modification_preconditions(request, meta):
411
    """Check the modified timestamp conforms with the preconditions set."""
412

    
413
    if 'modified' not in meta:
414
        return  # TODO: Always return?
415

    
416
    if_modified_since = request.META.get('HTTP_IF_MODIFIED_SINCE')
417
    if if_modified_since is not None:
418
        if_modified_since = parse_http_date_safe(if_modified_since)
419
    if (if_modified_since is not None
420
            and int(meta['modified']) <= if_modified_since):
421
        raise faults.NotModified('Resource has not been modified')
422

    
423
    if_unmodified_since = request.META.get('HTTP_IF_UNMODIFIED_SINCE')
424
    if if_unmodified_since is not None:
425
        if_unmodified_since = parse_http_date_safe(if_unmodified_since)
426
    if (if_unmodified_since is not None
427
            and int(meta['modified']) > if_unmodified_since):
428
        raise faults.PreconditionFailed('Resource has been modified')
429

    
430

    
431
def validate_matching_preconditions(request, meta):
432
    """Check that the ETag conforms with the preconditions set."""
433

    
434
    etag = meta['hash'] if not UPDATE_MD5 else meta['checksum']
435
    if not etag:
436
        etag = None
437

    
438
    if_match = request.META.get('HTTP_IF_MATCH')
439
    if if_match is not None:
440
        if etag is None:
441
            raise faults.PreconditionFailed('Resource does not exist')
442
        if (if_match != '*'
443
                and etag not in [x.lower() for x in parse_etags(if_match)]):
444
            raise faults.PreconditionFailed('Resource ETag does not match')
445

    
446
    if_none_match = request.META.get('HTTP_IF_NONE_MATCH')
447
    if if_none_match is not None:
448
        # TODO: If this passes, must ignore If-Modified-Since header.
449
        if etag is not None:
450
            if (if_none_match == '*' or etag in [x.lower() for x in
451
                                                 parse_etags(if_none_match)]):
452
                # TODO: Continue if an If-Modified-Since header is present.
453
                if request.method in ('HEAD', 'GET'):
454
                    raise faults.NotModified('Resource ETag matches')
455
                raise faults.PreconditionFailed(
456
                    'Resource exists or ETag matches')
457

    
458

    
459
def split_container_object_string(s):
460
    if not len(s) > 0 or s[0] != '/':
461
        raise ValueError
462
    s = s[1:]
463
    pos = s.find('/')
464
    if pos == -1 or pos == len(s) - 1:
465
        raise ValueError
466
    return s[:pos], s[(pos + 1):]
467

    
468

    
469
def copy_or_move_object(request, src_account, src_container, src_name,
470
                        dest_account, dest_container, dest_name,
471
                        move=False, delimiter=None):
472
    """Copy or move an object."""
473

    
474
    if 'ignore_content_type' in request.GET and 'CONTENT_TYPE' in request.META:
475
        del(request.META['CONTENT_TYPE'])
476
    content_type, meta, permissions, public = get_object_headers(request)
477
    src_version = request.META.get('HTTP_X_SOURCE_VERSION')
478
    try:
479
        if move:
480
            version_id = request.backend.move_object(
481
                request.user_uniq, src_account, src_container, src_name,
482
                dest_account, dest_container, dest_name,
483
                content_type, 'pithos', meta, False, permissions, delimiter)
484
        else:
485
            version_id = request.backend.copy_object(
486
                request.user_uniq, src_account, src_container, src_name,
487
                dest_account, dest_container, dest_name,
488
                content_type, 'pithos', meta, False, permissions,
489
                src_version, delimiter)
490
    except NotAllowedError:
491
        raise faults.Forbidden('Not allowed')
492
    except (ItemNotExists, VersionNotExists):
493
        raise faults.ItemNotFound('Container or object does not exist')
494
    except ValueError:
495
        raise faults.BadRequest('Invalid sharing header')
496
    except QuotaError, e:
497
        raise faults.RequestEntityTooLarge('Quota error: %s' % e)
498
    if public is not None:
499
        try:
500
            request.backend.update_object_public(
501
                request.user_uniq, dest_account,
502
                dest_container, dest_name, public)
503
        except NotAllowedError:
504
            raise faults.Forbidden('Not allowed')
505
        except ItemNotExists:
506
            raise faults.ItemNotFound('Object does not exist')
507
    return version_id
508

    
509

    
510
def get_int_parameter(p):
511
    if p is not None:
512
        try:
513
            p = int(p)
514
        except ValueError:
515
            return None
516
        if p < 0:
517
            return None
518
    return p
519

    
520

    
521
def get_content_length(request):
522
    content_length = get_int_parameter(request.META.get('CONTENT_LENGTH'))
523
    if content_length is None:
524
        raise faults.LengthRequired('Missing or invalid Content-Length header')
525
    return content_length
526

    
527

    
528
def get_range(request, size):
529
    """Parse a Range header from the request.
530

531
    Either returns None, when the header is not existent or should be ignored,
532
    or a list of (offset, length) tuples - should be further checked.
533
    """
534

    
535
    ranges = request.META.get('HTTP_RANGE', '').replace(' ', '')
536
    if not ranges.startswith('bytes='):
537
        return None
538

    
539
    ret = []
540
    for r in (x.strip() for x in ranges[6:].split(',')):
541
        p = re.compile('^(?P<offset>\d*)-(?P<upto>\d*)$')
542
        m = p.match(r)
543
        if not m:
544
            return None
545
        offset = m.group('offset')
546
        upto = m.group('upto')
547
        if offset == '' and upto == '':
548
            return None
549

    
550
        if offset != '':
551
            offset = int(offset)
552
            if upto != '':
553
                upto = int(upto)
554
                if offset > upto:
555
                    return None
556
                ret.append((offset, upto - offset + 1))
557
            else:
558
                ret.append((offset, size - offset))
559
        else:
560
            length = int(upto)
561
            ret.append((size - length, length))
562

    
563
    return ret
564

    
565

    
566
def get_content_range(request):
567
    """Parse a Content-Range header from the request.
568

569
    Either returns None, when the header is not existent or should be ignored,
570
    or an (offset, length, total) tuple - check as length, total may be None.
571
    Returns (None, None, None) if the provided range is '*/*'.
572
    """
573

    
574
    ranges = request.META.get('HTTP_CONTENT_RANGE', '')
575
    if not ranges:
576
        return None
577

    
578
    p = re.compile('^bytes (?P<offset>\d+)-(?P<upto>\d*)/(?P<total>(\d+|\*))$')
579
    m = p.match(ranges)
580
    if not m:
581
        if ranges == 'bytes */*':
582
            return (None, None, None)
583
        return None
584
    offset = int(m.group('offset'))
585
    upto = m.group('upto')
586
    total = m.group('total')
587
    if upto != '':
588
        upto = int(upto)
589
    else:
590
        upto = None
591
    if total != '*':
592
        total = int(total)
593
    else:
594
        total = None
595
    if (upto is not None and offset > upto) or \
596
        (total is not None and offset >= total) or \
597
            (total is not None and upto is not None and upto >= total):
598
        return None
599

    
600
    if upto is None:
601
        length = None
602
    else:
603
        length = upto - offset + 1
604
    return (offset, length, total)
605

    
606

    
607
def get_sharing(request):
608
    """Parse an X-Object-Sharing header from the request.
609

610
    Raises BadRequest on error.
611
    """
612

    
613
    permissions = request.META.get('HTTP_X_OBJECT_SHARING')
614
    if permissions is None:
615
        return None
616

    
617
    # TODO: Document or remove '~' replacing.
618
    permissions = permissions.replace('~', '')
619

    
620
    ret = {}
621
    permissions = permissions.replace(' ', '')
622
    if permissions == '':
623
        return ret
624
    for perm in (x for x in permissions.split(';')):
625
        if perm.startswith('read='):
626
            ret['read'] = list(set(
627
                [v.replace(' ', '').lower() for v in perm[5:].split(',')]))
628
            if '' in ret['read']:
629
                ret['read'].remove('')
630
            if '*' in ret['read']:
631
                ret['read'] = ['*']
632
            if len(ret['read']) == 0:
633
                raise faults.BadRequest(
634
                    'Bad X-Object-Sharing header value: invalid length')
635
        elif perm.startswith('write='):
636
            ret['write'] = list(set(
637
                [v.replace(' ', '').lower() for v in perm[6:].split(',')]))
638
            if '' in ret['write']:
639
                ret['write'].remove('')
640
            if '*' in ret['write']:
641
                ret['write'] = ['*']
642
            if len(ret['write']) == 0:
643
                raise faults.BadRequest(
644
                    'Bad X-Object-Sharing header value: invalid length')
645
        else:
646
            raise faults.BadRequest(
647
                'Bad X-Object-Sharing header value: missing prefix')
648

    
649
    # replace displayname with uuid
650
    if TRANSLATE_UUIDS:
651
        try:
652
            ret['read'] = [replace_permissions_displayname(
653
                getattr(request, 'token', None), x)
654
                for x in ret.get('read', [])]
655
            ret['write'] = [replace_permissions_displayname(
656
                getattr(request, 'token', None), x)
657
                for x in ret.get('write', [])]
658
        except ItemNotExists, e:
659
            raise faults.BadRequest(
660
                'Bad X-Object-Sharing header value: unknown account: %s' % e)
661

    
662
    # Keep duplicates only in write list.
663
    dups = [x for x in ret.get(
664
        'read', []) if x in ret.get('write', []) and x != '*']
665
    if dups:
666
        for x in dups:
667
            ret['read'].remove(x)
668
        if len(ret['read']) == 0:
669
            del(ret['read'])
670

    
671
    return ret
672

    
673

    
674
def get_public(request):
675
    """Parse an X-Object-Public header from the request.
676

677
    Raises BadRequest on error.
678
    """
679

    
680
    public = request.META.get('HTTP_X_OBJECT_PUBLIC')
681
    if public is None:
682
        return None
683

    
684
    public = public.replace(' ', '').lower()
685
    if public == 'true':
686
        return True
687
    elif public == 'false' or public == '':
688
        return False
689
    raise faults.BadRequest('Bad X-Object-Public header value')
690

    
691

    
692
def raw_input_socket(request):
693
    """Return the socket for reading the rest of the request."""
694

    
695
    server_software = request.META.get('SERVER_SOFTWARE')
696
    if server_software and server_software.startswith('mod_python'):
697
        return request._req
698
    if 'wsgi.input' in request.environ:
699
        return request.environ['wsgi.input']
700
    raise NotImplemented('Unknown server software')
701

    
702
MAX_UPLOAD_SIZE = 5 * (1024 * 1024 * 1024)  # 5GB
703

    
704

    
705
def socket_read_iterator(request, length=0, blocksize=4096):
706
    """Return maximum of blocksize data read from the socket in each iteration
707

708
    Read up to 'length'. If 'length' is negative, will attempt a chunked read.
709
    The maximum ammount of data read is controlled by MAX_UPLOAD_SIZE.
710
    """
711

    
712
    sock = raw_input_socket(request)
713
    if length < 0:  # Chunked transfers
714
        # Small version (server does the dechunking).
715
        if (request.environ.get('mod_wsgi.input_chunked', None)
716
                or request.META['SERVER_SOFTWARE'].startswith('gunicorn')):
717
            while length < MAX_UPLOAD_SIZE:
718
                data = sock.read(blocksize)
719
                if data == '':
720
                    return
721
                yield data
722
            raise faults.BadRequest('Maximum size is reached')
723

    
724
        # Long version (do the dechunking).
725
        data = ''
726
        while length < MAX_UPLOAD_SIZE:
727
            # Get chunk size.
728
            if hasattr(sock, 'readline'):
729
                chunk_length = sock.readline()
730
            else:
731
                chunk_length = ''
732
                while chunk_length[-1:] != '\n':
733
                    chunk_length += sock.read(1)
734
                chunk_length.strip()
735
            pos = chunk_length.find(';')
736
            if pos >= 0:
737
                chunk_length = chunk_length[:pos]
738
            try:
739
                chunk_length = int(chunk_length, 16)
740
            except Exception:
741
                raise faults.BadRequest('Bad chunk size')
742
                                 # TODO: Change to something more appropriate.
743
            # Check if done.
744
            if chunk_length == 0:
745
                if len(data) > 0:
746
                    yield data
747
                return
748
            # Get the actual data.
749
            while chunk_length > 0:
750
                chunk = sock.read(min(chunk_length, blocksize))
751
                chunk_length -= len(chunk)
752
                if length > 0:
753
                    length += len(chunk)
754
                data += chunk
755
                if len(data) >= blocksize:
756
                    ret = data[:blocksize]
757
                    data = data[blocksize:]
758
                    yield ret
759
            sock.read(2)  # CRLF
760
        raise faults.BadRequest('Maximum size is reached')
761
    else:
762
        if length > MAX_UPLOAD_SIZE:
763
            raise faults.BadRequest('Maximum size is reached')
764
        while length > 0:
765
            data = sock.read(min(length, blocksize))
766
            if not data:
767
                raise faults.BadRequest()
768
            length -= len(data)
769
            yield data
770

    
771

    
772
class SaveToBackendHandler(FileUploadHandler):
773
    """Handle a file from an HTML form the django way."""
774

    
775
    def __init__(self, request=None):
776
        super(SaveToBackendHandler, self).__init__(request)
777
        self.backend = request.backend
778

    
779
    def put_data(self, length):
780
        if len(self.data) >= length:
781
            block = self.data[:length]
782
            self.file.hashmap.append(self.backend.put_block(block))
783
            self.checksum_compute.update(block)
784
            self.data = self.data[length:]
785

    
786
    def new_file(self, field_name, file_name, content_type,
787
                 content_length, charset=None):
788
        self.checksum_compute = NoChecksum() if not UPDATE_MD5 else Checksum()
789
        self.data = ''
790
        self.file = UploadedFile(
791
            name=file_name, content_type=content_type, charset=charset)
792
        self.file.size = 0
793
        self.file.hashmap = []
794

    
795
    def receive_data_chunk(self, raw_data, start):
796
        self.data += raw_data
797
        self.file.size += len(raw_data)
798
        self.put_data(self.request.backend.block_size)
799
        return None
800

    
801
    def file_complete(self, file_size):
802
        l = len(self.data)
803
        if l > 0:
804
            self.put_data(l)
805
        self.file.etag = self.checksum_compute.hexdigest()
806
        return self.file
807

    
808

    
809
class ObjectWrapper(object):
810
    """Return the object's data block-per-block in each iteration.
811

812
    Read from the object using the offset and length provided
813
    in each entry of the range list.
814
    """
815

    
816
    def __init__(self, backend, ranges, sizes, hashmaps, boundary):
817
        self.backend = backend
818
        self.ranges = ranges
819
        self.sizes = sizes
820
        self.hashmaps = hashmaps
821
        self.boundary = boundary
822
        self.size = sum(self.sizes)
823

    
824
        self.file_index = 0
825
        self.block_index = 0
826
        self.block_hash = -1
827
        self.block = ''
828

    
829
        self.range_index = -1
830
        self.offset, self.length = self.ranges[0]
831

    
832
    def __iter__(self):
833
        return self
834

    
835
    def part_iterator(self):
836
        if self.length > 0:
837
            # Get the file for the current offset.
838
            file_size = self.sizes[self.file_index]
839
            while self.offset >= file_size:
840
                self.offset -= file_size
841
                self.file_index += 1
842
                file_size = self.sizes[self.file_index]
843

    
844
            # Get the block for the current position.
845
            self.block_index = int(self.offset / self.backend.block_size)
846
            if self.block_hash != \
847
                    self.hashmaps[self.file_index][self.block_index]:
848
                self.block_hash = self.hashmaps[
849
                    self.file_index][self.block_index]
850
                try:
851
                    self.block = self.backend.get_block(self.block_hash)
852
                except ItemNotExists:
853
                    raise faults.ItemNotFound('Block does not exist')
854

    
855
            # Get the data from the block.
856
            bo = self.offset % self.backend.block_size
857
            bs = self.backend.block_size
858
            if (self.block_index == len(self.hashmaps[self.file_index]) - 1 and
859
                    self.sizes[self.file_index] % self.backend.block_size):
860
                bs = self.sizes[self.file_index] % self.backend.block_size
861
            bl = min(self.length, bs - bo)
862
            data = self.block[bo:bo + bl]
863
            self.offset += bl
864
            self.length -= bl
865
            return data
866
        else:
867
            raise StopIteration
868

    
869
    def next(self):
870
        if len(self.ranges) == 1:
871
            return self.part_iterator()
872
        if self.range_index == len(self.ranges):
873
            raise StopIteration
874
        try:
875
            if self.range_index == -1:
876
                raise StopIteration
877
            return self.part_iterator()
878
        except StopIteration:
879
            self.range_index += 1
880
            out = []
881
            if self.range_index < len(self.ranges):
882
                # Part header.
883
                self.offset, self.length = self.ranges[self.range_index]
884
                self.file_index = 0
885
                if self.range_index > 0:
886
                    out.append('')
887
                out.append('--' + self.boundary)
888
                out.append('Content-Range: bytes %d-%d/%d' % (
889
                    self.offset, self.offset + self.length - 1, self.size))
890
                out.append('Content-Transfer-Encoding: binary')
891
                out.append('')
892
                out.append('')
893
                return '\r\n'.join(out)
894
            else:
895
                # Footer.
896
                out.append('')
897
                out.append('--' + self.boundary + '--')
898
                out.append('')
899
                return '\r\n'.join(out)
900

    
901

    
902
def object_data_response(request, sizes, hashmaps, meta, public=False):
903
    """Get the HttpResponse object for replying with the object's data."""
904

    
905
    # Range handling.
906
    size = sum(sizes)
907
    ranges = get_range(request, size)
908
    if ranges is None:
909
        ranges = [(0, size)]
910
        ret = 200
911
    else:
912
        check = [True for offset, length in ranges if
913
                 length <= 0 or length > size or
914
                 offset < 0 or offset >= size or
915
                 offset + length > size]
916
        if len(check) > 0:
917
            raise faults.RangeNotSatisfiable(
918
                'Requested range exceeds object limits')
919
        ret = 206
920
        if_range = request.META.get('HTTP_IF_RANGE')
921
        if if_range:
922
            try:
923
                # Modification time has passed instead.
924
                last_modified = parse_http_date(if_range)
925
                if last_modified != meta['modified']:
926
                    ranges = [(0, size)]
927
                    ret = 200
928
            except ValueError:
929
                if if_range != meta['checksum']:
930
                    ranges = [(0, size)]
931
                    ret = 200
932

    
933
    if ret == 206 and len(ranges) > 1:
934
        boundary = uuid.uuid4().hex
935
    else:
936
        boundary = ''
937
    wrapper = ObjectWrapper(request.backend, ranges, sizes, hashmaps, boundary)
938
    response = HttpResponse(wrapper, status=ret)
939
    put_object_headers(
940
        response, meta, restricted=public,
941
        token=getattr(request, 'token', None))
942
    if ret == 206:
943
        if len(ranges) == 1:
944
            offset, length = ranges[0]
945
            response[
946
                'Content-Length'] = length  # Update with the correct length.
947
            response['Content-Range'] = 'bytes %d-%d/%d' % (
948
                offset, offset + length - 1, size)
949
        else:
950
            del(response['Content-Length'])
951
            response['Content-Type'] = 'multipart/byteranges; boundary=%s' % (
952
                boundary,)
953
    return response
954

    
955

    
956
def put_object_block(request, hashmap, data, offset):
957
    """Put one block of data at the given offset."""
958

    
959
    bi = int(offset / request.backend.block_size)
960
    bo = offset % request.backend.block_size
961
    bl = min(len(data), request.backend.block_size - bo)
962
    if bi < len(hashmap):
963
        hashmap[bi] = request.backend.update_block(hashmap[bi], data[:bl], bo)
964
    else:
965
        hashmap.append(request.backend.put_block(('\x00' * bo) + data[:bl]))
966
    return bl  # Return ammount of data written.
967

    
968

    
969
def hashmap_md5(backend, hashmap, size):
970
    """Produce the MD5 sum from the data in the hashmap."""
971

    
972
    # TODO: Search backend for the MD5 of another object
973
    #       with the same hashmap and size...
974
    md5 = hashlib.md5()
975
    bs = backend.block_size
976
    for bi, hash in enumerate(hashmap):
977
        data = backend.get_block(hash)  # Blocks come in padded.
978
        if bi == len(hashmap) - 1:
979
            data = data[:size % bs]
980
        md5.update(data)
981
    return md5.hexdigest().lower()
982

    
983

    
984
def simple_list_response(request, l):
985
    if request.serialization == 'text':
986
        return '\n'.join(l) + '\n'
987
    if request.serialization == 'xml':
988
        return render_to_string('items.xml', {'items': l})
989
    if request.serialization == 'json':
990
        return json.dumps(l)
991

    
992

    
993
from pithos.backends.util import PithosBackendPool
994

    
995
if RADOS_STORAGE:
996
    BLOCK_PARAMS = {'mappool': RADOS_POOL_MAPS,
997
                    'blockpool': RADOS_POOL_BLOCKS, }
998
else:
999
    BLOCK_PARAMS = {'mappool': None,
1000
                    'blockpool': None, }
1001

    
1002
BACKEND_KWARGS = dict(
1003
    db_module=BACKEND_DB_MODULE,
1004
    db_connection=BACKEND_DB_CONNECTION,
1005
    block_module=BACKEND_BLOCK_MODULE,
1006
    block_path=BACKEND_BLOCK_PATH,
1007
    block_umask=BACKEND_BLOCK_UMASK,
1008
    block_size=BACKEND_BLOCK_SIZE,
1009
    hash_algorithm=BACKEND_HASH_ALGORITHM,
1010
    queue_module=BACKEND_QUEUE_MODULE,
1011
    queue_hosts=BACKEND_QUEUE_HOSTS,
1012
    queue_exchange=BACKEND_QUEUE_EXCHANGE,
1013
    astakos_url=ASTAKOS_BASE_URL,
1014
    service_token=SERVICE_TOKEN,
1015
    astakosclient_poolsize=ASTAKOSCLIENT_POOLSIZE,
1016
    free_versioning=BACKEND_FREE_VERSIONING,
1017
    block_params=BLOCK_PARAMS,
1018
    public_url_security=PUBLIC_URL_SECURITY,
1019
    public_url_alphabet=PUBLIC_URL_ALPHABET,
1020
    account_quota_policy=BACKEND_ACCOUNT_QUOTA,
1021
    container_quota_policy=BACKEND_CONTAINER_QUOTA,
1022
    container_versioning_policy=BACKEND_VERSIONING)
1023

    
1024
_pithos_backend_pool = PithosBackendPool(size=BACKEND_POOL_SIZE,
1025
                                         **BACKEND_KWARGS)
1026

    
1027

    
1028
def get_backend():
1029
    if BACKEND_POOL_ENABLED:
1030
        backend = _pithos_backend_pool.pool_get()
1031
    else:
1032
        backend = connect_backend(**BACKEND_KWARGS)
1033
    backend.serials = []
1034
    backend.messages = []
1035
    return backend
1036

    
1037

    
1038
def update_request_headers(request):
1039
    # Handle URL-encoded keys and values.
1040
    meta = dict([(
1041
        k, v) for k, v in request.META.iteritems() if k.startswith('HTTP_')])
1042
    for k, v in meta.iteritems():
1043
        try:
1044
            k.decode('ascii')
1045
            v.decode('ascii')
1046
        except UnicodeDecodeError:
1047
            raise faults.BadRequest('Bad character in headers.')
1048
        if '%' in k or '%' in v:
1049
            del(request.META[k])
1050
            request.META[unquote(k)] = smart_unicode(unquote(
1051
                v), strings_only=True)
1052

    
1053

    
1054
def update_response_headers(request, response):
1055
    # URL-encode unicode in headers.
1056
    meta = response.items()
1057
    for k, v in meta:
1058
        if (k.startswith('X-Account-') or k.startswith('X-Container-') or
1059
                k.startswith('X-Object-') or k.startswith('Content-')):
1060
            del(response[k])
1061
            response[quote(k)] = quote(v, safe='/=,:@; ')
1062

    
1063

    
1064
def get_pithos_usage(token):
1065
    """Get Pithos Usage from astakos."""
1066
    astakos = AstakosClient(ASTAKOS_BASE_URL, retry=2, use_pool=True,
1067
                            logger=logger)
1068
    quotas = astakos.get_quotas(token)['system']
1069
    pithos_resources = [r['name'] for r in resources]
1070
    map(quotas.pop, filter(lambda k: k not in pithos_resources, quotas.keys()))
1071
    return quotas.popitem()[-1]  # assume only one resource
1072

    
1073

    
1074
def api_method(http_method=None, token_required=True, user_required=True,
1075
               logger=None, format_allowed=False, serializations=None,
1076
               strict_serlization=False, lock_container_path=False):
1077
    serializations = serializations or ['json', 'xml']
1078

    
1079
    def decorator(func):
1080
        @api.api_method(http_method=http_method, token_required=token_required,
1081
                        user_required=user_required,
1082
                        logger=logger, format_allowed=format_allowed,
1083
                        astakos_url=ASTAKOS_BASE_URL,
1084
                        serializations=serializations,
1085
                        strict_serlization=strict_serlization)
1086
        @wraps(func)
1087
        def wrapper(request, *args, **kwargs):
1088
            # The args variable may contain up to (account, container, object).
1089
            if len(args) > 1 and len(args[1]) > 256:
1090
                raise faults.BadRequest("Container name too large")
1091
            if len(args) > 2 and len(args[2]) > 1024:
1092
                raise faults.BadRequest('Object name too large.')
1093

    
1094
            success_status = False
1095
            try:
1096
                # Add a PithosBackend as attribute of the request object
1097
                request.backend = get_backend()
1098
                request.backend.pre_exec(lock_container_path)
1099

    
1100
                # Many API method expect thet X-Auth-Token in request,token
1101
                request.token = request.x_auth_token
1102
                update_request_headers(request)
1103
                response = func(request, *args, **kwargs)
1104
                update_response_headers(request, response)
1105

    
1106
                success_status = True
1107
                return response
1108
            finally:
1109
                # Always close PithosBackend connection
1110
                if getattr(request, "backend", None) is not None:
1111
                    request.backend.post_exec(success_status)
1112
                    request.backend.close()
1113
        return wrapper
1114
    return decorator
1115

    
1116

    
1117
def get_token_from_cookie(request):
1118
    assert(request.method == 'GET'),\
1119
        "Cookie based authentication is only allowed to GET requests"
1120
    token = None
1121
    if COOKIE_NAME in request.COOKIES:
1122
        cookie_value = unquote(request.COOKIES.get(COOKIE_NAME, ''))
1123
        account, sep, token = cookie_value.partition('|')
1124
    return token
1125

    
1126

    
1127
def view_method():
1128
    """Decorator function for views."""
1129

    
1130
    def decorator(func):
1131
        @wraps(func)
1132
        def wrapper(request, *args, **kwargs):
1133
            token = get_token_from_cookie(request)
1134
            if token is None:
1135
                return HttpResponseRedirect('%s?next=%s' % (
1136
                    LOGIN_URL, join_urls(BASE_HOST, request.path)))
1137
            request.META['HTTP_X_AUTH_TOKEN'] = token
1138
            # Get the response object
1139
            response = func(request, *args, **kwargs)
1140
            if response.status_code in [200, 206, 304, 412, 416]:
1141
                return response
1142
            elif response.status_code == 404:
1143
                raise Http404()
1144
            elif response.status_code in [401, 403]:
1145
                return HttpResponseForbidden()
1146
            else:
1147
                # unexpected response status
1148
                raise Exception(response.status_code)
1149
        return wrapper
1150
    return decorator
1151

    
1152

    
1153
class Checksum:
1154
    def __init__(self):
1155
        self.md5 = hashlib.md5()
1156

    
1157
    def update(self, data):
1158
        self.md5.update(data)
1159

    
1160
    def hexdigest(self):
1161
        return self.md5.hexdigest().lower()
1162

    
1163

    
1164
class NoChecksum:
1165
    def update(self, data):
1166
        pass
1167

    
1168
    def hexdigest(self):
1169
        return ''