Revision bb4eafc6

b/pithos/api/short_url.py
1
# Copyright (C) 2009 by Michael Fogleman
2
# 
3
# Permission is hereby granted, free of charge, to any person obtaining a copy
4
# of this software and associated documentation files (the "Software"), to deal
5
# in the Software without restriction, including without limitation the rights
6
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
# copies of the Software, and to permit persons to whom the Software is
8
# furnished to do so, subject to the following conditions:
9
# 
10
# The above copyright notice and this permission notice shall be included in
11
# all copies or substantial portions of the Software.
12
# 
13
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
# THE SOFTWARE.
20

  
21
'''
22
Short URL Generator
23
===================
24

  
25
Python implementation for generating Tiny URL- and bit.ly-like URLs.
26

  
27
A bit-shuffling approach is used to avoid generating consecutive, predictable 
28
URLs.  However, the algorithm is deterministic and will guarantee that no 
29
collisions will occur.
30

  
31
The URL alphabet is fully customizable and may contain any number of 
32
characters.  By default, digits and lower-case letters are used, with 
33
some removed to avoid confusion between characters like o, O and 0.  The 
34
default alphabet is shuffled and has a prime number of characters to further 
35
improve the results of the algorithm.
36

  
37
The block size specifies how many bits will be shuffled.  The lower BLOCK_SIZE 
38
bits are reversed.  Any bits higher than BLOCK_SIZE will remain as is.
39
BLOCK_SIZE of 0 will leave all bits unaffected and the algorithm will simply 
40
be converting your integer to a different base.
41

  
42
The intended use is that incrementing, consecutive integers will be used as 
43
keys to generate the short URLs.  For example, when creating a new URL, the 
44
unique integer ID assigned by a database could be used to generate the URL 
45
by using this module.  Or a simple counter may be used.  As long as the same 
46
integer is not used twice, the same short URL will not be generated twice.
47

  
48
The module supports both encoding and decoding of URLs. The min_length 
49
parameter allows you to pad the URL if you want it to be a specific length.
50

  
51
Sample Usage:
52

  
53
>>> import short_url
54
>>> url = short_url.encode_url(12)
55
>>> print url
56
LhKA
57
>>> key = short_url.decode_url(url)
58
>>> print key
59
12
60

  
61
Use the functions in the top-level of the module to use the default encoder. 
62
Otherwise, you may create your own UrlEncoder object and use its encode_url 
63
and decode_url methods.
64

  
65
Author: Michael Fogleman
66
License: MIT
67
Link: http://code.activestate.com/recipes/576918/
68
'''
69

  
70
DEFAULT_ALPHABET = 'mn6j2c4rv8bpygw95z7hsdaetxuk3fq'
71
DEFAULT_BLOCK_SIZE = 24
72
MIN_LENGTH = 5
73

  
74
class UrlEncoder(object):
75
    def __init__(self, alphabet=DEFAULT_ALPHABET, block_size=DEFAULT_BLOCK_SIZE):
76
        self.alphabet = alphabet
77
        self.block_size = block_size
78
        self.mask = (1 << block_size) - 1
79
        self.mapping = range(block_size)
80
        self.mapping.reverse()
81
    def encode_url(self, n, min_length=MIN_LENGTH):
82
        return self.enbase(self.encode(n), min_length)
83
    def decode_url(self, n):
84
        return self.decode(self.debase(n))
85
    def encode(self, n):
86
        return (n & ~self.mask) | self._encode(n & self.mask)
87
    def _encode(self, n):
88
        result = 0
89
        for i, b in enumerate(self.mapping):
90
            if n & (1 << i):
91
                result |= (1 << b)
92
        return result
93
    def decode(self, n):
94
        return (n & ~self.mask) | self._decode(n & self.mask)
95
    def _decode(self, n):
96
        result = 0
97
        for i, b in enumerate(self.mapping):
98
            if n & (1 << b):
99
                result |= (1 << i)
100
        return result
101
    def enbase(self, x, min_length=MIN_LENGTH):
102
        result = self._enbase(x)
103
        padding = self.alphabet[0] * (min_length - len(result))
104
        return '%s%s' % (padding, result)
105
    def _enbase(self, x):
106
        n = len(self.alphabet)
107
        if x < n:
108
            return self.alphabet[x]
109
        return self._enbase(x / n) + self.alphabet[x % n]
110
    def debase(self, x):
111
        n = len(self.alphabet)
112
        result = 0
113
        for i, c in enumerate(reversed(x)):
114
            result += self.alphabet.index(c) * (n ** i)
115
        return result
116

  
117
DEFAULT_ENCODER = UrlEncoder()
118

  
119
def encode(n):
120
    return DEFAULT_ENCODER.encode(n)
121

  
122
def decode(n):
123
    return DEFAULT_ENCODER.decode(n)
124

  
125
def enbase(n, min_length=MIN_LENGTH):
126
    return DEFAULT_ENCODER.enbase(n, min_length)
127

  
128
def debase(n):
129
    return DEFAULT_ENCODER.debase(n)
130

  
131
def encode_url(n, min_length=MIN_LENGTH):
132
    return DEFAULT_ENCODER.encode_url(n, min_length)
133

  
134
def decode_url(n):
135
    return DEFAULT_ENCODER.decode_url(n)
136

  
137
if __name__ == '__main__':
138
    for a in range(0, 200000, 37):
139
        b = encode(a)
140
        c = enbase(b)
141
        d = debase(c)
142
        e = decode(d)
143
        assert a == e
144
        assert b == d
145
        c = (' ' * (7 - len(c))) + c
146
        print '%6d %12d %s %12d %6d' % (a, b, c, d, e)
b/pithos/api/util.py
50 50
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound,
51 51
                                Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge,
52 52
                                RangeNotSatisfiable, ServiceUnavailable)
53
from pithos.api.short_url import encode_url
53 54
from pithos.backends import connect_backend
54 55
from pithos.backends.base import NotAllowedError, QuotaError
55 56

  
......
241 242
def update_public_meta(public, meta):
242 243
    if not public:
243 244
        return
244
    meta['X-Object-Public'] = public
245
    meta['X-Object-Public'] = '/public/' + encode_url(public)
245 246

  
246 247
def validate_modification_preconditions(request, meta):
247 248
    """Check that the modified timestamp conforms with the preconditions set."""
b/pithos/backends/base.py
388 388
        return
389 389
    
390 390
    def get_object_public(self, user, account, container, name):
391
        """Return the public URL of the object if applicable.
391
        """Return the public id of the object if applicable.
392 392
        
393 393
        Raises:
394 394
            NotAllowedError: Operation not permitted
......
513 513
        """
514 514
        return []
515 515
    
516
    def get_public(self, user, public):
517
        """Return the (account, container, name) for the public id given.
518
        
519
        Raises:
520
            NotAllowedError: Operation not permitted
521
            
522
            NameError: Public id does not exist
523
        """
524
        return None
525
    
516 526
    def get_block(self, hash):
517 527
        """Return a block's data.
518 528
        
b/pithos/backends/lib/sqlalchemy/permissions.py
79 79
    def access_check(self, path, access, member):
80 80
        """Return true if the member has this access to the path."""
81 81
        
82
        if access == READ and self.public_check(path):
82
        if access == READ and self.public_get(path) is not None:
83 83
            return True
84 84
        
85 85
        r = self.xfeature_inherit(path)
b/pithos/backends/lib/sqlalchemy/public.py
32 32
# or implied, of GRNET S.A.
33 33

  
34 34
from dbworker import DBWorker
35
from sqlalchemy import Table, Column, String, MetaData
35
from sqlalchemy import Table, Column, String, Integer, MetaData
36 36
from sqlalchemy.sql import select
37
from sqlalchemy.schema import Index
38

  
37 39

  
38 40
class Public(DBWorker):
39 41
    """Paths can be marked as public."""
......
42 44
        DBWorker.__init__(self, **params)
43 45
        metadata = MetaData()
44 46
        columns=[]
45
        columns.append(Column('path', String(2048), index=True))
46
        self.public = Table('public', metadata, *columns, mysql_engine='InnoDB')
47
        columns.append(Column('public_id', Integer, primary_key=True))
48
        columns.append(Column('path', String(2048)))
49
        self.public = Table('public', metadata, *columns, mysql_engine='InnoDB', sqlite_autoincrement=True)
50
        # place an index on path
51
        Index('idx_public_path', self.public.c.path)
47 52
        metadata.create_all(self.engine)
48 53
    
49
    
50 54
    def public_set(self, path):
51 55
        s = self.public.select()
52 56
        s = s.where(self.public.c.path == path)
......
63 67
        r = self.conn.execute(s)
64 68
        r.close()
65 69
    
66
    def public_check(self, path):
67
        s = select([self.public.c.path], self.public.c.path == path)
70
    def public_get(self, path):
71
        s = select([self.public.c.public_id], self.public.c.path == path)
72
        r = self.conn.execute(s)
73
        row = r.fetchone()
74
        r.close()
75
        if row:
76
            return row[0]
77
        return None
78
    
79
    def public_path(self, public):
80
        s = select([self.public.c.path], self.public.c.public_id == public)
68 81
        r = self.conn.execute(s)
69
        l = r.fetchone()
82
        row = r.fetchone()
70 83
        r.close()
71
        return bool(l)
84
        if row:
85
            return row[0]
86
        return None
b/pithos/backends/lib/sqlite/permissions.py
76 76
    def access_check(self, path, access, member):
77 77
        """Return true if the member has this access to the path."""
78 78
        
79
        if access == READ and self.public_check(path):
79
        if access == READ and self.public_get(path) is not None:
80 80
            return True
81 81
        
82 82
        r = self.xfeature_inherit(path)
b/pithos/backends/lib/sqlite/public.py
42 42
        execute = self.execute
43 43
        
44 44
        execute(""" create table if not exists public
45
                          ( path text primary key ) """)
45
                          ( public_id integer primary key autoincrement,
46
                            path      text ) """)
47
        execute(""" create unique index if not exists idx_public_path
48
                    on public(path) """)
46 49
    
47 50
    def public_set(self, path):
48 51
        q = "insert or ignore into public (path) values (?)"
......
52 55
        q = "delete from public where path = ?"
53 56
        self.execute(q, (path,))
54 57
    
55
    def public_check(self, path):
56
        q = "select 1 from public where path = ?"
58
    def public_get(self, path):
59
        q = "select public_id from public where path = ?"
57 60
        self.execute(q, (path,))
58
        return bool(self.fetchone())
61
        row = self.fetchone()
62
        if row:
63
            return row[0]
64
        return None
65
    
66
    def public_path(self, public):
67
        q = "select path from public where public_id = ?"
68
        self.execute(q, (public,))
69
        row = self.fetchone()
70
        if row:
71
            return row[0]
72
        return None
b/pithos/backends/modular.py
44 44

  
45 45
inf = float('inf')
46 46

  
47
ULTIMATE_ANSWER = 42
48

  
47 49

  
48 50
logger = logging.getLogger(__name__)
49 51

  
......
492 494
    
493 495
    @backend_method
494 496
    def get_object_public(self, user, account, container, name):
495
        """Return the public URL of the object if applicable."""
497
        """Return the public id of the object if applicable."""
496 498
        
497 499
        logger.debug("get_object_public: %s %s %s", account, container, name)
498 500
        self._can_read(user, account, container, name)
499 501
        path = self._lookup_object(account, container, name)[0]
500
        if self.permissions.public_check(path):
501
            return '/public/' + path
502
        return None
502
        p = self.permissions.public_get(path)
503
        if p is not None:
504
            p += ULTIMATE_ANSWER
505
        return p
503 506
    
504 507
    @backend_method
505 508
    def update_object_public(self, user, account, container, name, public):
......
651 654
        versions = self.node.node_get_versions(node)
652 655
        return [[x[self.SERIAL], x[self.MTIME]] for x in versions if x[self.CLUSTER] != CLUSTER_DELETED]
653 656
    
657
    @backend_method
658
    def get_public(self, user, public):
659
        """Return the (account, container, name) for the public id given."""
660
        logger.debug("get_public: %s", public)
661
        if public is None or public < ULTIMATE_ANSWER:
662
            raise NameError
663
        path = self.permissions.public_path(public - ULTIMATE_ANSWER)
664
        account, container, name = path.split('/', 2)
665
        self._can_read(user, account, container, name)
666
        return (account, container, name)
667
    
654 668
    @backend_method(autocommit=0)
655 669
    def get_block(self, hash):
656 670
        """Return a block's data."""
b/pithos/public/functions.py
39 39
from pithos.api.util import (put_object_headers, update_manifest_meta,
40 40
    validate_modification_preconditions, validate_matching_preconditions,
41 41
    object_data_response, api_method)
42
from pithos.api.short_url import decode_url
42 43

  
43 44

  
44 45
logger = logging.getLogger(__name__)
45 46

  
46 47

  
47
def object_demux(request, v_account, v_container, v_object):
48
def public_demux(request, v_public):
48 49
    if request.method == 'HEAD':
49
        return object_meta(request, v_account, v_container, v_object)
50
        return public_meta(request, v_public)
50 51
    elif request.method == 'GET':
51
        return object_read(request, v_account, v_container, v_object)
52
        return public_read(request, v_public)
52 53
    else:
53 54
        return method_not_allowed(request)
54 55

  
55 56
@api_method('HEAD', user_required=False)
56
def object_meta(request, v_account, v_container, v_object):
57
def public_meta(request, v_public):
57 58
    # Normal Response Codes: 204
58 59
    # Error Response Codes: serviceUnavailable (503),
59 60
    #                       itemNotFound (404),
60 61
    #                       badRequest (400)
61 62
    
62 63
    try:
64
        v_account, v_container, v_object = request.backend.get_public(request.user_uniq,
65
                                                    decode_url(v_public))
63 66
        meta = request.backend.get_object_meta(request.user_uniq, v_account,
64
                                                v_container, v_object)
67
                                                    v_container, v_object)
65 68
        public = request.backend.get_object_public(request.user_uniq, v_account,
66 69
                                                    v_container, v_object)
67 70
    except:
......
76 79
    return response
77 80

  
78 81
@api_method('GET', user_required=False)
79
def object_read(request, v_account, v_container, v_object):
82
def public_read(request, v_public):
80 83
    # Normal Response Codes: 200, 206
81 84
    # Error Response Codes: serviceUnavailable (503),
82 85
    #                       rangeNotSatisfiable (416),
......
86 89
    #                       notModified (304)
87 90
    
88 91
    try:
92
        v_account, v_container, v_object = request.backend.get_public(request.user_uniq,
93
                                                    decode_url(v_public))
89 94
        meta = request.backend.get_object_meta(request.user_uniq, v_account,
90
                                                v_container, v_object)
95
                                                    v_container, v_object)
91 96
        public = request.backend.get_object_public(request.user_uniq, v_account,
92 97
                                                    v_container, v_object)
93 98
    except:
......
133 138
        except:
134 139
            raise ItemNotFound('Object does not exist')
135 140
    
141
    if 'Content-Disposition' not in meta:
142
        name = v_object.rstrip('/').split('/')[-1]
143
        if not name:
144
            name = v_public
145
        meta['Content-Disposition'] = 'attachment; filename=%s' % (name,)
146
    
136 147
    return object_data_response(request, sizes, hashmaps, meta, True)
137 148

  
138 149
@api_method(user_required=False)
b/pithos/public/urls.py
33 33

  
34 34
from django.conf.urls.defaults import *
35 35

  
36
# TODO: This only works when in this order.
37 36
urlpatterns = patterns('pithos.public.functions',
38 37
    (r'^$', 'method_not_allowed'),
39
    (r'^(?P<v_account>.+?)/(?P<v_container>.+?)/(?P<v_object>.+?)$', 'object_demux'),
40
    (r'^(?P<v_account>.+?)/(?P<v_container>.+?)/?$', 'method_not_allowed'),
41
    (r'^(?P<v_account>.+?)/?$', 'method_not_allowed')
38
    (r'^(?P<v_public>.+?)/?$', 'public_demux')
42 39
)

Also available in: Unified diff